diff --git a/.gitignore b/.gitignore index 92ebe6e..fd7f403 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ __pycache__/ .pytest_cache/ .ruff_cache/ /.codex -/profile/cache/bazel_registry_checkout/ -/profile/cache/reference_integration_checkout/ +/profile/cache/ /.cache /_site/ diff --git a/docs/repo-overview/architecture.md b/docs/repo-overview/architecture.md index d2b5abf..e2ea8a0 100644 --- a/docs/repo-overview/architecture.md +++ b/docs/repo-overview/architecture.md @@ -53,7 +53,8 @@ rather than Markdown- or HTML-specific values. - `collector/signal_detection.py` derives repository-local content signals. - `collector/platform_docs.py` discovers and associates platform Sphinx declarations after repository collection. -- `collector/git_checkout.py` owns shallow checkout synchronization and reads. +- `collector/git_checkout.py` delegates checkout synchronization to the shared + `repo_cache` package and provides repository reads and release-ref helpers. - `collector/snapshot_io.py` serializes the normalized snapshot. - `models.py` defines the collection/rendering boundary. - `profile_readme.py` renders the organization profile. diff --git a/docs/repo-overview/collection-and-cache.md b/docs/repo-overview/collection-and-cache.md index 5d9a709..e491e89 100644 --- a/docs/repo-overview/collection-and-cache.md +++ b/docs/repo-overview/collection-and-cache.md @@ -3,7 +3,7 @@ ## Snapshot Cache The default cache file is `.cache/repo_overview.json`. -Repository checkouts share the SCORE repository-policy cache: +Repository checkouts share the SCORE `repo_cache` cache: `${XDG_CACHE_HOME:-~/.cache}/repo-cache//`. The cache is used in two ways: @@ -17,8 +17,9 @@ Changing a renderer or template therefore requires no GitHub refresh. ## Incremental Collection Collection still fetches current high-level repository state, including the -default branch. Git synchronizes each shallow partial checkout and supplies the -current commit SHA. The collector then chooses one of these paths: +default branch. The shared `repo_cache` package synchronizes each shallow +checkout through the authenticated GitHub CLI and supplies the current commit +SHA. The collector then chooses one of these paths: - unchanged SHA and fresh volatile metrics: reuse the cached repository entry - unchanged SHA and stale volatile metrics: refresh activity metrics only @@ -50,10 +51,10 @@ Git supplies repository content and identity: - Bazel and Sphinx declarations - release `MODULE.bazel` and `.bazelversion` -Checkouts are disposable, shallow, single-branch partial clones. The generic -cache is shared with other repository tools such as `score-repo-policy-sync`. -Authentication is passed through a transient Git HTTP header and is not written -into the remote URL. +Checkouts are disposable, shallow, single-branch clones. The cache is shared +with other repository tools such as `score-repo-policy-sync` and +`score-repo-cache`. Checkout authentication is handled by `gh`; use +`GITHUB_TOKEN` or `gh auth login` before collecting. For a repository without any commits, GitHub may report a default-branch name even though that branch cannot be resolved. After a failed checkout, the diff --git a/docs/repo-overview/usage-and-extension.md b/docs/repo-overview/usage-and-extension.md index 21248c3..0136fae 100644 --- a/docs/repo-overview/usage-and-extension.md +++ b/docs/repo-overview/usage-and-extension.md @@ -37,6 +37,8 @@ uv sync --all-groups --frozen Collection reads `GITHUB_TOKEN` and falls back to `gh auth token`. The token must be able to read every configured organization and platform repository. +The shared `repo_cache` dependency uses the authenticated `gh` CLI for Git +checkout synchronization; a custom `--token-env` value is forwarded to it. The policy report fetch uses `GH_TOKEN`/`GITHUB_TOKEN` through the installed GitHub CLI (`gh auth login` is sufficient for local use). diff --git a/pyproject.toml b/pyproject.toml index 2c3f26b..32ded9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,7 @@ description = "Collect cached GitHub organization overviews and render Markdown requires-python = ">=3.12" dependencies = [ "PyGithub", + "repo-cache @ git+https://github.com/eclipse-score/tools#subdirectory=repo_cache", "tqdm", ] @@ -26,6 +27,9 @@ dev = [ [tool.hatch.build.targets.wheel] packages = ["src/generate_repo_overview"] +[tool.hatch.metadata] +allow-direct-references = true + [tool.uv] package = true diff --git a/src/generate_repo_overview/collector/__init__.py b/src/generate_repo_overview/collector/__init__.py index 781d390..ef370a8 100644 --- a/src/generate_repo_overview/collector/__init__.py +++ b/src/generate_repo_overview/collector/__init__.py @@ -170,6 +170,11 @@ def collect_snapshot( when="before collection", status_prefix=status_prefix, ) + # repo_cache starts gh subprocesses for checkout synchronization. GH_TOKEN + # gives those subprocesses the same credential used by PyGithub, including + # when the caller selected a custom --token-env variable. + previous_gh_token = os.environ.get("GH_TOKEN") + os.environ["GH_TOKEN"] = token try: organization = github.get_organization(org_name) @@ -197,7 +202,6 @@ def collect_snapshot( repos, github=github, platform_repos=config.platform_repos, - github_token=token, status_prefix=status_prefix, ) @@ -228,6 +232,10 @@ def collect_snapshot( print_status(f"Wrote snapshot to {cache_path}", prefix=status_prefix) return snapshot finally: + if previous_gh_token is None: + os.environ.pop("GH_TOKEN", None) + else: + os.environ["GH_TOKEN"] = previous_gh_token print_rest_api_rate_limit( github, when="after collection", @@ -240,7 +248,6 @@ def enrich_repositories_with_platform_docs( *, github: RepositoryResolverLike, platform_repos: tuple[str, ...], - github_token: str | None = None, status_prefix: str, ) -> list[RepoEntry]: enriched = [ @@ -260,16 +267,14 @@ def enrich_repositories_with_platform_docs( prefix=status_prefix, ) repository = github.get_repo(full_name) - clone_url = cast("str | None", getattr(repository, "clone_url", None)) default_branch = cast("str | None", getattr(repository, "default_branch", None)) - if clone_url is None or default_branch is None: + if default_branch is None: raise RuntimeError( - f"Configured platform repository {full_name} has no clone metadata." + f"Configured platform repository {full_name} has no default branch." ) checkout_path = sync_repository_checkout( - clone_url=clone_url, + repository=full_name, default_branch=default_branch, - github_token=github_token, checkout_path=DEFAULT_REPOSITORY_CHECKOUTS / full_name, ) if checkout_path is None: @@ -360,6 +365,10 @@ def fetch_repositories( f"{repositories_with_custom_properties} repositories", prefix=status_prefix, ) + resolved_registry_repository = cast( + "str | None", getattr(registry_repository, "full_name", None) + ) + registry_repository_name = resolved_registry_repository or config.registry_repo bazel_registry_metadata_by_repo: dict[ str, registry_metadata.RegistrySignalsPayload @@ -373,7 +382,6 @@ def fetch_repositories( registry_metadata.fetch_bazel_registry_metadata_by_repo( bazel_registry_repository=registry_repository, active_repository_names=set(active_repositories), - github_token=github_token, ) ) print_status( @@ -398,7 +406,7 @@ def fetch_repositories( else None ), active_repository_names=set(active_repositories), - github_token=github_token, + registry_repository=registry_repository_name, org_name=config.org_name, ) ) diff --git a/src/generate_repo_overview/collector/git_checkout.py b/src/generate_repo_overview/collector/git_checkout.py index ff61825..cb4743d 100644 --- a/src/generate_repo_overview/collector/git_checkout.py +++ b/src/generate_repo_overview/collector/git_checkout.py @@ -2,109 +2,33 @@ import base64 import os -import shutil import subprocess from typing import TYPE_CHECKING +from repo_cache import RepoCacheError, sync_default_branch + if TYPE_CHECKING: from pathlib import Path def sync_repository_checkout( *, - clone_url: str, + repository: str, default_branch: str, - github_token: str | None, checkout_path: Path, ) -> Path | None: - authenticated_url = build_authenticated_clone_url(clone_url, github_token) - checkout_path.parent.mkdir(parents=True, exist_ok=True) - - if update_existing_checkout( - checkout_path, - default_branch, - github_token=github_token, - ): - return checkout_path - - if not clone_fresh_checkout( - authenticated_url=authenticated_url, - default_branch=default_branch, - checkout_path=checkout_path, - github_token=github_token, - ): + """Synchronize a GitHub checkout through the shared ``repo_cache`` package.""" + try: + sync_default_branch( + repository=repository, + branch=default_branch, + destination=checkout_path, + ) + except (OSError, RepoCacheError): return None - return checkout_path -def update_existing_checkout( - checkout_path: Path, - default_branch: str, - *, - github_token: str | None = None, -) -> bool: - git_dir = checkout_path / ".git" - if not git_dir.exists(): - return False - - fetch_ok = run_git_command( - [ - "git", - "-C", - str(checkout_path), - "fetch", - "--depth", - "1", - "origin", - default_branch, - ], - github_token=github_token, - ) - checkout_ok = run_git_command( - [ - "git", - "-C", - str(checkout_path), - "checkout", - "--force", - "--detach", - "FETCH_HEAD", - ] - ) - if not (fetch_ok and checkout_ok): - return False - - run_git_command(["git", "-C", str(checkout_path), "clean", "-fdx"]) - return True - - -def clone_fresh_checkout( - *, - authenticated_url: str, - default_branch: str, - checkout_path: Path, - github_token: str | None = None, -) -> bool: - shutil.rmtree(checkout_path, ignore_errors=True) - return run_git_command( - [ - "git", - "clone", - "--depth", - "1", - "--filter=blob:none", - "--single-branch", - "--no-tags", - "--branch", - default_branch, - authenticated_url, - str(checkout_path), - ], - github_token=github_token, - ) - - def get_checkout_head_sha(checkout_path: Path) -> str | None: try: result = subprocess.run( @@ -215,10 +139,6 @@ def run_git_command( return True -def build_authenticated_clone_url(clone_url: str, github_token: str | None) -> str: - return clone_url - - def _run_git_for_text(command: list[str]) -> str | None: try: result = subprocess.run( diff --git a/src/generate_repo_overview/collector/reference_integration.py b/src/generate_repo_overview/collector/reference_integration.py index debe9f5..7244d6f 100644 --- a/src/generate_repo_overview/collector/reference_integration.py +++ b/src/generate_repo_overview/collector/reference_integration.py @@ -5,19 +5,17 @@ from typing import TYPE_CHECKING, cast from urllib.parse import urlsplit +from repo_cache import default_cache_directory + +from generate_repo_overview.org_config import is_safe_repository_path + from .git_checkout import sync_repository_checkout -from .registry_metadata import ( - BAZEL_REGISTRY_LOCAL_CHECKOUT, - parse_bazel_registry_metadata, -) +from .registry_metadata import parse_bazel_registry_metadata from .signal_detection import dedupe_preserving_order if TYPE_CHECKING: from collections.abc import Iterable -REFERENCE_INTEGRATION_LOCAL_CHECKOUT = Path( - "profile/cache/reference_integration_checkout" -) ROOT_MODULE_PATH = Path("MODULE.bazel") INCLUDE_PATTERN = re.compile(r'\binclude\s*\(\s*"(?P