support fsspec 2025.10.0 (#7844)

lhoestq · web-flow · commit fc7f97c442b3 · 2025-11-03T15:51:32.000+01:00
fsspec 2025.10.0
diff --git a/setup.py b/setup.py
@@ -127,7 +127,7 @@
     "multiprocess<0.70.19",  # to align with dill<0.3.9 (see above)
     # to save datasets locally or on any filesystem
     # minimum 2023.1.0 to support protocol=kwargs in fsspec's `open`, `get_fs_token_paths`, etc.: see https://github.com/fsspec/filesystem_spec/pull/1143
-    "fsspec[http]>=2023.1.0,<=2025.9.0",
+    "fsspec[http]>=2023.1.0,<=2025.10.0",
     # To get datasets from the Datasets Hub on huggingface.co
     "huggingface-hub>=0.25.0,<2.0",
     # Utilities from PyPA to e.g., compare versions
diff --git a/src/datasets/data_files.py b/src/datasets/data_files.py
@@ -349,14 +349,18 @@ def resolve_pattern(
     pattern, storage_options = _prepare_path_and_storage_options(pattern, download_config=download_config)
     fs, fs_pattern = url_to_fs(pattern, **storage_options)
     files_to_ignore = set(FILES_TO_IGNORE) - {xbasename(pattern)}
-    protocol = fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]
+    protocol = (
+        pattern.split("://")[0]
+        if "://" in pattern
+        else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0])
+    )
     protocol_prefix = protocol + "://" if protocol != "file" else ""
     glob_kwargs = {}
     if protocol == "hf":
         # 10 times faster glob with detail=True (ignores costly info like lastCommit)
         glob_kwargs["expand_info"] = False
     matched_paths = [
-        filepath if filepath.startswith(protocol_prefix) else protocol_prefix + filepath
+        filepath if "://" in filepath else protocol_prefix + filepath
         for filepath, info in fs.glob(pattern, detail=True, **glob_kwargs).items()
         if (info["type"] == "file" or (info.get("islink") and os.path.isfile(os.path.realpath(filepath))))
         and (xbasename(filepath) not in files_to_ignore)
