Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion .github/workflows/lib-deps-check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ jobs:
run: |
git fetch origin ${{ github.event.pull_request.head.sha }}
- name: Checkout PR Lib files
run: |
# Checkout only Lib/ directory from PR head for accurate comparison
git checkout ${{ github.event.pull_request.head.sha }} -- Lib/
- name: Checkout CPython
run: |
git clone --depth 1 --branch v3.14.2 https://github.com/python/cpython.git cpython
Expand Down Expand Up @@ -104,7 +109,7 @@ jobs:
</details>
**Legend:**
- `[+]` path exists, `[-]` path missing
- `[+]` path exists in CPython
- `[x]` up-to-date, `[ ]` outdated
- `native:` Rust/C extension modules
Expand Down
10 changes: 10 additions & 0 deletions scripts/update_lib/__main__.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ def main(argv: list[str] | None = None) -> int:
help="Show dependency information for a module",
add_help=False,
)
subparsers.add_parser(
"todo",
help="Show prioritized list of modules to update",
add_help=False,
)

args, remaining = parser.parse_known_args(argv)

Expand Down Expand Up @@ -87,6 +92,11 @@ def main(argv: list[str] | None = None) -> int:

return show_deps_main(remaining)

if args.command == "todo":
from update_lib.show_todo import main as show_todo_main

return show_todo_main(remaining)

return 0


Expand Down
64 changes: 39 additions & 25 deletions scripts/update_lib/deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- Test dependencies (auto-detected from 'from test import ...')
"""

import functools
import pathlib

from update_lib.io_utils import read_python_files, safe_parse_ast, safe_read_text
Expand Down Expand Up @@ -145,15 +146,18 @@
}


def get_lib_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]:
@functools.cache
def get_lib_paths(
name: str, cpython_prefix: str = "cpython"
) -> tuple[pathlib.Path, ...]:
"""Get all library paths for a module.

Args:
name: Module name (e.g., "datetime", "libregrtest")
cpython_prefix: CPython directory prefix

Returns:
List of paths to copy
Tuple of paths to copy
"""
dep_info = DEPENDENCIES.get(name, {})

Expand All @@ -168,43 +172,49 @@ def get_lib_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Pa
for dep in dep_info.get("hard_deps", []):
paths.append(construct_lib_path(cpython_prefix, dep))

return paths
return tuple(paths)


def get_test_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]:
@functools.cache
def get_test_paths(
name: str, cpython_prefix: str = "cpython"
) -> tuple[pathlib.Path, ...]:
"""Get all test paths for a module.

Args:
name: Module name (e.g., "datetime", "libregrtest")
cpython_prefix: CPython directory prefix

Returns:
List of test paths
Tuple of test paths
"""
if name in DEPENDENCIES and "test" in DEPENDENCIES[name]:
return [
return tuple(
construct_lib_path(cpython_prefix, p) for p in DEPENDENCIES[name]["test"]
]
)

# Default: try directory first, then file
return [resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir")]
return (resolve_module_path(f"test/test_{name}", cpython_prefix, prefer="dir"),)


def get_data_paths(name: str, cpython_prefix: str = "cpython") -> list[pathlib.Path]:
@functools.cache
def get_data_paths(
name: str, cpython_prefix: str = "cpython"
) -> tuple[pathlib.Path, ...]:
"""Get additional data paths for a module.

Args:
name: Module name
cpython_prefix: CPython directory prefix

Returns:
List of data paths (may be empty)
Tuple of data paths (may be empty)
"""
if name in DEPENDENCIES and "data" in DEPENDENCIES[name]:
return [
return tuple(
construct_lib_path(cpython_prefix, p) for p in DEPENDENCIES[name]["data"]
]
return []
)
return ()


def parse_test_imports(content: str) -> set[str]:
Expand Down Expand Up @@ -272,15 +282,16 @@ def parse_lib_imports(content: str) -> set[str]:
return imports


def get_all_imports(name: str, cpython_prefix: str = "cpython") -> set[str]:
@functools.cache
def get_all_imports(name: str, cpython_prefix: str = "cpython") -> frozenset[str]:
"""Get all imports from a library file.

Args:
name: Module name
cpython_prefix: CPython directory prefix

Returns:
Set of all imported module names
Frozenset of all imported module names
"""
all_imports = set()
for lib_path in get_lib_paths(name, cpython_prefix):
Expand All @@ -290,18 +301,19 @@ def get_all_imports(name: str, cpython_prefix: str = "cpython") -> set[str]:

# Remove self
all_imports.discard(name)
return all_imports
return frozenset(all_imports)


def get_soft_deps(name: str, cpython_prefix: str = "cpython") -> set[str]:
@functools.cache
def get_soft_deps(name: str, cpython_prefix: str = "cpython") -> frozenset[str]:
"""Get soft dependencies by parsing imports from library file.

Args:
name: Module name
cpython_prefix: CPython directory prefix

Returns:
Set of imported stdlib module names (those that exist in cpython/Lib/)
Frozenset of imported stdlib module names (those that exist in cpython/Lib/)
"""
all_imports = get_all_imports(name, cpython_prefix)

Expand All @@ -312,22 +324,23 @@ def get_soft_deps(name: str, cpython_prefix: str = "cpython") -> set[str]:
if module_path.exists():
stdlib_deps.add(imp)

return stdlib_deps
return frozenset(stdlib_deps)


def get_rust_deps(name: str, cpython_prefix: str = "cpython") -> set[str]:
@functools.cache
def get_rust_deps(name: str, cpython_prefix: str = "cpython") -> frozenset[str]:
"""Get Rust/C dependencies (imports that don't exist in cpython/Lib/).

Args:
name: Module name
cpython_prefix: CPython directory prefix

Returns:
Set of imported module names that are built-in or C extensions
Frozenset of imported module names that are built-in or C extensions
"""
all_imports = get_all_imports(name, cpython_prefix)
soft_deps = get_soft_deps(name, cpython_prefix)
return all_imports - soft_deps
return frozenset(all_imports - soft_deps)


def _dircmp_is_same(dcmp) -> bool:
Expand All @@ -350,6 +363,7 @@ def _dircmp_is_same(dcmp) -> bool:
return True


@functools.cache
def is_up_to_date(
name: str, cpython_prefix: str = "cpython", lib_prefix: str = "Lib"
) -> bool:
Expand Down Expand Up @@ -472,9 +486,9 @@ def resolve_all_paths(
Dict with "lib", "test", "data", "test_deps" keys
"""
result = {
"lib": get_lib_paths(name, cpython_prefix),
"test": get_test_paths(name, cpython_prefix),
"data": get_data_paths(name, cpython_prefix),
"lib": list(get_lib_paths(name, cpython_prefix)),
"test": list(get_test_paths(name, cpython_prefix)),
"data": list(get_data_paths(name, cpython_prefix)),
"test_deps": [],
}

Expand Down
12 changes: 6 additions & 6 deletions scripts/update_lib/show_deps.py
Original file line number Diff line number Diff line change
Expand Up @@ -169,17 +169,17 @@ def format_deps(

lines = []

# lib paths
# lib paths (only show existing)
lib_paths = get_lib_paths(name, cpython_prefix)
for p in lib_paths:
exists = "+" if p.exists() else "-"
lines.append(f"[{exists}] lib: {p}")
if p.exists():
lines.append(f"[+] lib: {p}")

# test paths
# test paths (only show existing)
test_paths = get_test_paths(name, cpython_prefix)
for p in test_paths:
exists = "+" if p.exists() else "-"
lines.append(f"[{exists}] test: {p}")
if p.exists():
lines.append(f"[+] test: {p}")

# hard_deps (from DEPENDENCIES table)
dep_info = DEPENDENCIES.get(name, {})
Expand Down
Loading
Loading