update_lib todo also shows test todo (#6859)

youknowone · web-flow · commit 691d2816f901 · 2026-01-24T16:04:36.000+09:00
also tracking untracked files considered in #6775
diff --git a/scripts/update_lib/deps.py b/scripts/update_lib/deps.py
@@ -68,24 +68,9 @@ def clear_import_graph_caches() -> None:
         "lib": [],  # No Python lib (Rust implementation)
         "hard_deps": ["_pylong.py"],
     },
-    # Pure Python implementations
-    "abc": {
-        "hard_deps": ["_py_abc.py"],
-    },
-    "codecs": {
-        "hard_deps": ["_pycodecs.py"],
-    },
-    "datetime": {
-        "hard_deps": ["_pydatetime.py"],
-    },
-    "decimal": {
-        "hard_deps": ["_pydecimal.py"],
-    },
-    "io": {
-        "hard_deps": ["_pyio.py"],
-    },
-    "warnings": {
-        "hard_deps": ["_py_warnings.py"],
+    # Non-pattern hard_deps (can't be auto-detected)
+    "ast": {
+        "hard_deps": ["_ast_unparse.py"],
     },
     # Data directories
     "pydoc": {
@@ -102,29 +87,52 @@ def clear_import_graph_caches() -> None:
 }
 
 
-def resolve_hard_dep_parent(name: str) -> str | None:
+def resolve_hard_dep_parent(name: str, cpython_prefix: str = "cpython") -> str | None:
     """Resolve a hard_dep name to its parent module.
 
-    If 'name' is listed as a hard_dep of another module, return that module's name.
-    E.g., 'pydoc_data' -> 'pydoc', '_pydatetime' -> 'datetime'
+    Only returns a parent if the file is actually tracked:
+    - Explicitly listed in DEPENDENCIES as a hard_dep
+    - Or auto-detected _py{module}.py pattern where the parent module exists
 
     Args:
         name: Module or file name (with or without .py extension)
+        cpython_prefix: CPython directory prefix
 
     Returns:
-        Parent module name if found, None otherwise
+        Parent module name if found and tracked, None otherwise
     """
     # Normalize: remove .py extension if present
     if name.endswith(".py"):
         name = name[:-3]
 
+    # Check DEPENDENCIES table first (explicit hard_deps)
     for module_name, dep_info in DEPENDENCIES.items():
         hard_deps = dep_info.get("hard_deps", [])
         for dep in hard_deps:
             # Normalize dep: remove .py extension
             dep_normalized = dep[:-3] if dep.endswith(".py") else dep
             if dep_normalized == name:
                 return module_name
+
+    # Auto-detect _py{module} or _py_{module} patterns
+    # Only if the parent module actually exists
+    if name.startswith("_py"):
+        if name.startswith("_py_"):
+            # _py_abc -> abc
+            parent = name[4:]
+        else:
+            # _pydatetime -> datetime
+            parent = name[3:]
+
+        # Verify the parent module exists
+        lib_dir = pathlib.Path(cpython_prefix) / "Lib"
+        parent_file = lib_dir / f"{parent}.py"
+        parent_dir = lib_dir / parent
+        if parent_file.exists() or (
+            parent_dir.exists() and (parent_dir / "__init__.py").exists()
+        ):
+            return parent
+
     return None
 
 
@@ -234,10 +242,16 @@ def get_lib_paths(
         # Default: try file first, then directory
         paths = [resolve_module_path(name, cpython_prefix, prefer="file")]
 
-    # Add hard_deps
+    # Add hard_deps from DEPENDENCIES
     for dep in dep_info.get("hard_deps", []):
         paths.append(construct_lib_path(cpython_prefix, dep))
 
+    # Auto-detect _py{module}.py or _py_{module}.py patterns
+    for pattern in [f"_py{name}.py", f"_py_{name}.py"]:
+        auto_path = construct_lib_path(cpython_prefix, pattern)
+        if auto_path.exists() and auto_path not in paths:
+            paths.append(auto_path)
+
     return tuple(paths)
 
 
diff --git a/scripts/update_lib/show_deps.py b/scripts/update_lib/show_deps.py
@@ -19,26 +19,42 @@
 def get_all_modules(cpython_prefix: str = "cpython") -> list[str]:
     """Get all top-level module names from cpython/Lib/.
 
+    Includes private modules (_*) that are not hard_deps of other modules.
+
     Returns:
         Sorted list of module names (without .py extension)
     """
+    from update_lib.deps import resolve_hard_dep_parent
+
     lib_dir = pathlib.Path(cpython_prefix) / "Lib"
     if not lib_dir.exists():
         return []
 
     modules = set()
     for entry in lib_dir.iterdir():
-        # Skip private/internal modules and special directories
-        if entry.name.startswith(("_", ".")):
+        # Skip hidden files
+        if entry.name.startswith("."):
             continue
         # Skip test directory
         if entry.name == "test":
             continue
 
         if entry.is_file() and entry.suffix == ".py":
-            modules.add(entry.stem)
+            name = entry.stem
         elif entry.is_dir() and (entry / "__init__.py").exists():
-            modules.add(entry.name)
+            name = entry.name
+        else:
+            continue
+
+        # Skip private modules that are hard_deps of other modules
+        # e.g., _pydatetime is a hard_dep of datetime, so skip it
+        if (
+            name.startswith("_")
+            and resolve_hard_dep_parent(name, cpython_prefix) is not None
+        ):
+            continue
+
+        modules.add(name)
 
     return sorted(modules)
 
diff --git a/scripts/update_lib/show_todo.py b/scripts/update_lib/show_todo.py
