Skip to content

Commit 5a37e74

Browse files
AlenkaFraulcd
authored andcommitted
GH-38431: [Python][CI] Update fs.type_name checks for s3fs tests (#38455)
### Rationale for this change Appveyor CI is failing https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow/builds/48347810. It seems the reason for the test failures is a change in the `type_name` (from `"py::fsspec+s3"` to `"py::fsspec+('s3', 's3a')"`) and due to it tests are not being skipped. ### What changes are included in this PR? Update the check for `type_name` in case of `PyFileSystem(FSSpecHandler(s3fs.S3FileSystem()))`. * Closes: #38431 Authored-by: AlenkaF <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent 2dcee3f commit 5a37e74

File tree

2 files changed

+7
-7
lines changed

2 files changed

+7
-7
lines changed

ci/conda_env_python.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,6 @@ numpy>=1.16.6
2626
pytest
2727
pytest-faulthandler
2828
pytest-lazy-fixture
29-
s3fs>=2021.8.0
29+
s3fs>=2023.10.0
3030
setuptools
3131
setuptools_scm<8.0.0

python/pyarrow/tests/test_fs.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -463,7 +463,7 @@ def check_mtime_or_absent(file_info):
463463

464464

465465
def skip_fsspec_s3fs(fs):
466-
if fs.type_name == "py::fsspec+s3":
466+
if fs.type_name == "py::fsspec+('s3', 's3a')":
467467
pytest.xfail(reason="Not working with fsspec's s3fs")
468468

469469

@@ -631,7 +631,7 @@ def test_get_file_info(fs, pathfn):
631631
assert aaa_info.path == aaa
632632
assert 'aaa' in repr(aaa_info)
633633
assert aaa_info.extension == ''
634-
if fs.type_name == "py::fsspec+s3":
634+
if fs.type_name == "py::fsspec+('s3', 's3a')":
635635
# s3fs doesn't create empty directories
636636
assert aaa_info.type == FileType.NotFound
637637
else:
@@ -646,7 +646,7 @@ def test_get_file_info(fs, pathfn):
646646
assert bb_info.type == FileType.File
647647
assert 'FileType.File' in repr(bb_info)
648648
assert bb_info.size == 0
649-
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
649+
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+('s3', 's3a')"]:
650650
check_mtime(bb_info)
651651

652652
assert c_info.path == str(c)
@@ -655,7 +655,7 @@ def test_get_file_info(fs, pathfn):
655655
assert c_info.type == FileType.File
656656
assert 'FileType.File' in repr(c_info)
657657
assert c_info.size == 4
658-
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+s3"]:
658+
if fs.type_name not in ["py::fsspec+memory", "py::fsspec+('s3', 's3a')"]:
659659
check_mtime(c_info)
660660

661661
assert zzz_info.path == str(zzz)
@@ -698,7 +698,7 @@ def test_get_file_info_with_selector(fs, pathfn):
698698
assert selector.base_dir == base_dir
699699

700700
infos = fs.get_file_info(selector)
701-
if fs.type_name == "py::fsspec+s3":
701+
if fs.type_name == "py::fsspec+('s3', 's3a')":
702702
# s3fs only lists directories if they are not empty
703703
len(infos) == 4
704704
else:
@@ -719,7 +719,7 @@ def test_get_file_info_with_selector(fs, pathfn):
719719
selector = FileSelector(base_dir, recursive=False)
720720

721721
infos = fs.get_file_info(selector)
722-
if fs.type_name == "py::fsspec+s3":
722+
if fs.type_name == "py::fsspec+('s3', 's3a')":
723723
# s3fs only lists directories if they are not empty
724724
assert len(infos) == 3
725725
else:

0 commit comments

Comments
 (0)