From 14e09777633c99d372f19ee6808019e8e0b0324e Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 26 Feb 2026 21:53:05 +0000 Subject: [PATCH 1/2] Add to example --- example/dfetch.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/example/dfetch.yaml b/example/dfetch.yaml index 7f950abd..82d4b075 100644 --- a/example/dfetch.yaml +++ b/example/dfetch.yaml @@ -42,3 +42,15 @@ manifest: dst: Tests/cpputest-git-rev-only revision: d14505cc9191fcf17ccbd92af1c3409eb3969890 repo-path: cpputest/cpputest.git # Use external git directly + + - name: TF-PSA-Crypto + url: https://github.com/Mbed-TLS/TF-PSA-Crypto.git + tag: v1.0.0 + dst: ext/TF-PSA-Crypto + ignore: + - tests + - scripts + - programs + - drivers + - doxygen + - docs From 78711d12dd9a7237d09456c40ce06a74c64fa5b8 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 23 Feb 2026 22:09:41 +0000 Subject: [PATCH 2/2] Fetching any submodule in a subproject with submodules Fixes #1013 --- CHANGELOG.rst | 8 +- dfetch/log.py | 5 +- dfetch/project/gitsubproject.py | 30 +++++-- dfetch/project/metadata.py | 33 ++++++- dfetch/project/subproject.py | 30 ++++++- dfetch/project/svnsubproject.py | 6 +- dfetch/reporting/stdout_reporter.py | 34 ++++--- dfetch/util/util.py | 9 +- dfetch/vcs/git.py | 13 ++- .../fetch-git-repo-with-submodule.feature | 89 +++++++++++++++++++ features/steps/generic_steps.py | 5 ++ features/steps/git_steps.py | 27 ++++-- tests/test_subproject.py | 6 +- 13 files changed, 250 insertions(+), 45 deletions(-) create mode 100644 features/fetch-git-repo-with-submodule.feature diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f71b9787..3622fb60 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,8 +1,14 @@ +Release 0.13.0 (unreleased) +==================================== + +* Rename child-manifests to sub-manifests in documentation and code (#1027) +* Fetch git submodules in git subproject at pinned revision (#1015) +* Add nested projects in subprojects to project report (#1015) + Release 0.12.1 (released 2026-02-24) ==================================== * Fix missing unicode data in standalone binaries (#1014) -* Rename child-manifests to sub-manifests in documentation and code (#1027) Release 0.12.0 (released 2026-02-21) ==================================== diff --git a/dfetch/log.py b/dfetch/log.py index 52476ffa..d5f43797 100644 --- a/dfetch/log.py +++ b/dfetch/log.py @@ -62,8 +62,9 @@ def print_info_line(self, name: str, info: str) -> None: self.info(f" [bold][bright_green]{name}:[/bright_green][/bold]") DLogger._printed_projects.add(name) - line = info.replace("\n", "\n ") - self.info(f" [bold blue]> {line}[/bold blue]") + if info: + line = info.replace("\n", "\n ") + self.info(f" [bold blue]> {line}[/bold blue]") def print_warning_line(self, name: str, info: str) -> None: """Print a warning line: green name, yellow value.""" diff --git a/dfetch/project/gitsubproject.py b/dfetch/project/gitsubproject.py index c52f3208..8b207413 100644 --- a/dfetch/project/gitsubproject.py +++ b/dfetch/project/gitsubproject.py @@ -7,8 +7,8 @@ from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version -from dfetch.project.subproject import SubProject -from dfetch.util.util import safe_rmtree +from dfetch.project.subproject import SubProject, VcsDependency +from dfetch.util.util import safe_rm, safe_rmtree from dfetch.vcs.git import GitLocalRepo, GitRemote, get_git_version logger = get_logger(__name__) @@ -57,7 +57,7 @@ def list_tool_info() -> None: ) SubProject._log_tool("git", "") - def _fetch_impl(self, version: Version) -> Version: + def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]: """Get the revision of the remote and place it at the local path.""" rev_or_branch_or_tag = self._determine_what_to_fetch(version) @@ -69,17 +69,35 @@ def _fetch_impl(self, version: Version) -> Version: ] local_repo = GitLocalRepo(self.local_path) - fetched_sha = local_repo.checkout_version( + fetched_sha, submodules = local_repo.checkout_version( remote=self.remote, version=rev_or_branch_or_tag, src=self.source, - must_keeps=license_globs, + must_keeps=license_globs + [".gitmodules"], ignore=self.ignore, ) + vcs_deps = [] + for submodule in submodules: + self._log_project( + f'Found & fetched submodule "./{submodule.path}" ' + f" ({submodule.url} @ {Version(tag=submodule.tag, branch=submodule.branch, revision=submodule.sha)})", + ) + vcs_deps.append( + VcsDependency( + remote_url=submodule.url, + destination=submodule.path, + branch=submodule.branch, + tag=submodule.tag, + revision=submodule.sha, + source_type="git-submodule", + ) + ) + safe_rmtree(os.path.join(self.local_path, local_repo.METADATA_DIR)) + safe_rm(os.path.join(self.local_path, local_repo.GIT_MODULES_FILE)) - return self._determine_fetched_version(version, fetched_sha) + return self._determine_fetched_version(version, fetched_sha), vcs_deps def _determine_what_to_fetch(self, version: Version) -> str: """Based on asked version, target to fetch.""" diff --git a/dfetch/project/metadata.py b/dfetch/project/metadata.py index 0f611c81..2b3eba47 100644 --- a/dfetch/project/metadata.py +++ b/dfetch/project/metadata.py @@ -16,6 +16,17 @@ """ +class Dependency(TypedDict): + """Argument types for dependency class construction.""" + + branch: str + tag: str + revision: str + remote_url: str + destination: str + source_type: str + + class Options(TypedDict): # pylint: disable=too-many-ancestors """Argument types for Metadata class construction.""" @@ -27,6 +38,7 @@ class Options(TypedDict): # pylint: disable=too-many-ancestors destination: str hash: str patch: str | list[str] + dependencies: list["Dependency"] class Metadata: @@ -54,6 +66,8 @@ def __init__(self, kwargs: Options) -> None: # Historically only a single patch was allowed self._patch: list[str] = always_str_list(kwargs.get("patch", [])) + self._dependencies: list[Dependency] = kwargs.get("dependencies", []) + @classmethod def from_project_entry(cls, project: ProjectEntry) -> "Metadata": """Create a metadata object from a project entry.""" @@ -66,6 +80,7 @@ def from_project_entry(cls, project: ProjectEntry) -> "Metadata": "last_fetch": datetime.datetime(2000, 1, 1, 0, 0, 0), "hash": "", "patch": project.patch, + "dependencies": [], } return cls(data) @@ -77,13 +92,18 @@ def from_file(cls, path: str) -> "Metadata": return cls(data) def fetched( - self, version: Version, hash_: str = "", patch_: list[str] | None = None + self, + version: Version, + hash_: str = "", + patch_: list[str] | None = None, + dependencies: list[Dependency] | None = None, ) -> None: """Update metadata.""" self._last_fetch = datetime.datetime.now() self._version = version self._hash = hash_ self._patch = patch_ or [] + self._dependencies = dependencies or [] @property def version(self) -> Version: @@ -129,6 +149,11 @@ def patch(self) -> list[str]: """The list of applied patches as stored in the metadata.""" return self._patch + @property + def dependencies(self) -> list[Dependency]: + """The list of dependency projects as stored in the metadata.""" + return self._dependencies + @property def path(self) -> str: """Path to metadata file.""" @@ -152,12 +177,13 @@ def __eq__(self, other: object) -> bool: other._version.revision == self._version.revision, other.hash == self.hash, other.patch == self.patch, + other.dependencies == self.dependencies, ] ) def dump(self) -> None: """Dump metadata file to correct path.""" - metadata = { + metadata: dict[str, dict[str, str | list[str] | list[Dependency]]] = { "dfetch": { "remote_url": self.remote_url, "branch": self._version.branch, @@ -169,6 +195,9 @@ def dump(self) -> None: } } + if self.dependencies: + metadata["dfetch"]["dependencies"] = self.dependencies + with open(self.path, "w+", encoding="utf-8") as metadata_file: metadata_file.write(DONT_EDIT_WARNING) yaml.dump(metadata, metadata_file) diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py index 20f685e8..9690c344 100644 --- a/dfetch/project/subproject.py +++ b/dfetch/project/subproject.py @@ -5,12 +5,13 @@ import pathlib from abc import ABC, abstractmethod from collections.abc import Sequence +from typing import NamedTuple from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version from dfetch.project.abstract_check_reporter import AbstractCheckReporter -from dfetch.project.metadata import Metadata +from dfetch.project.metadata import Dependency, Metadata from dfetch.util.util import hash_directory, safe_rm from dfetch.util.versions import latest_tag_from_list from dfetch.vcs.patch import Patch @@ -18,6 +19,28 @@ logger = get_logger(__name__) +class VcsDependency(NamedTuple): + """Information about a vcs dependency.""" + + destination: str + remote_url: str + branch: str + tag: str + revision: str + source_type: str + + def to_dependency(self) -> Dependency: + """Convert this vcs dependency to a Dependency object.""" + return Dependency( + destination=self.destination, + remote_url=self.remote_url, + branch=self.branch, + tag=self.tag, + revision=self.revision, + source_type=self.source_type, + ) + + class SubProject(ABC): """Abstract SubProject object. @@ -125,7 +148,7 @@ def update( f"Fetching {to_fetch}", enabled=self._show_animations, ): - actually_fetched = self._fetch_impl(to_fetch) + actually_fetched, dependency = self._fetch_impl(to_fetch) self._log_project(f"Fetched {actually_fetched}") applied_patches = self._apply_patches(patch_count) @@ -134,6 +157,7 @@ def update( actually_fetched, hash_=hash_directory(self.local_path, skiplist=[self.__metadata.FILENAME]), patch_=applied_patches, + dependencies=[dependency.to_dependency() for dependency in dependency], ) logger.debug(f"Writing repo metadata to: {self.__metadata.path}") @@ -381,7 +405,7 @@ def _are_there_local_changes(self, files_to_ignore: Sequence[str]) -> bool: ) @abstractmethod - def _fetch_impl(self, version: Version) -> Version: + def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]: """Fetch the given version of the subproject, should be implemented by the child class.""" @abstractmethod diff --git a/dfetch/project/svnsubproject.py b/dfetch/project/svnsubproject.py index 6284daaf..e5f1c2d2 100644 --- a/dfetch/project/svnsubproject.py +++ b/dfetch/project/svnsubproject.py @@ -7,7 +7,7 @@ from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version -from dfetch.project.subproject import SubProject +from dfetch.project.subproject import SubProject, VcsDependency from dfetch.util.util import ( find_matching_files, find_non_matching_files, @@ -106,7 +106,7 @@ def _remove_ignored_files(self) -> None: if not (file_or_dir.is_file() and self.is_license_file(file_or_dir.name)): safe_rm(file_or_dir) - def _fetch_impl(self, version: Version) -> Version: + def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]: """Get the revision of the remote and place it at the local path.""" branch, branch_path, revision = self._determine_what_to_fetch(version) rev_arg = f"--revision {revision}" if revision else "" @@ -147,7 +147,7 @@ def _fetch_impl(self, version: Version) -> Version: if self.ignore: self._remove_ignored_files() - return Version(tag=version.tag, branch=branch, revision=revision) + return Version(tag=version.tag, branch=branch, revision=revision), [] @staticmethod def _parse_file_pattern(complete_path: str) -> tuple[str, str]: diff --git a/dfetch/reporting/stdout_reporter.py b/dfetch/reporting/stdout_reporter.py index 4982088c..4c4ac521 100644 --- a/dfetch/reporting/stdout_reporter.py +++ b/dfetch/reporting/stdout_reporter.py @@ -26,22 +26,36 @@ def add_project( ) -> None: """Add a project to the report.""" del version - logger.print_info_field("project", project.name) - logger.print_info_field(" remote", project.remote) + logger.print_info_line(project.name, "") + logger.print_info_field("- remote", project.remote) try: metadata = Metadata.from_file(Metadata.from_project_entry(project).path) - logger.print_info_field(" remote url", metadata.remote_url) - logger.print_info_field(" branch", metadata.branch) - logger.print_info_field(" tag", metadata.tag) - logger.print_info_field(" last fetch", str(metadata.last_fetch)) - logger.print_info_field(" revision", metadata.revision) - logger.print_info_field(" patch", ", ".join(metadata.patch)) + logger.print_info_field(" remote url", metadata.remote_url) + logger.print_info_field(" branch", metadata.branch) + logger.print_info_field(" tag", metadata.tag) + logger.print_info_field(" last fetch", str(metadata.last_fetch)) + logger.print_info_field(" revision", metadata.revision) + logger.print_info_field(" patch", ", ".join(metadata.patch)) logger.print_info_field( - " licenses", ",".join(license.name for license in licenses) + " licenses", ",".join(license.name for license in licenses) ) + if metadata.dependencies: + logger.info("") + logger.print_report_line(" dependencies", "") + for dependency in metadata.dependencies: + logger.print_info_field(" - path", dependency.get("destination", "")) + logger.print_info_field(" url", dependency.get("remote_url", "")) + logger.print_info_field(" branch", dependency.get("branch", "")) + logger.print_info_field(" tag", dependency.get("tag", "")) + logger.print_info_field(" revision", dependency.get("revision", "")) + logger.print_info_field( + " source-type", dependency.get("source_type", "") + ) + logger.info("") + except FileNotFoundError: - logger.print_info_field(" last fetch", "never") + logger.print_info_field(" last fetch", "never") def dump_to_file(self, outfile: str) -> bool: """Do nothing.""" diff --git a/dfetch/util/util.py b/dfetch/util/util.py index b5f83b20..2840a2e9 100644 --- a/dfetch/util/util.py +++ b/dfetch/util/util.py @@ -44,10 +44,11 @@ def find_matching_files(directory: str, patterns: Sequence[str]) -> Iterator[Pat def safe_rm(path: str | Path) -> None: """Delete an file or directory safely.""" - if os.path.isdir(path): - safe_rmtree(str(path)) - else: - os.remove(path) + if os.path.exists(path): + if os.path.isdir(path): + safe_rmtree(str(path)) + else: + os.remove(path) def safe_rmtree(path: str) -> None: diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index 01315732..a8dda379 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -233,6 +233,7 @@ class GitLocalRepo: """A git repository.""" METADATA_DIR = ".git" + GIT_MODULES_FILE = ".gitmodules" def __init__(self, path: str | Path = ".") -> None: """Create a local git repo.""" @@ -258,7 +259,7 @@ def checkout_version( # pylint: disable=too-many-arguments src: str | None = None, must_keeps: list[str] | None = None, ignore: Sequence[str] | None = None, - ) -> str: + ) -> tuple[str, list[Submodule]]: """Checkout a specific version from a given remote. Args: @@ -295,6 +296,14 @@ def checkout_version( # pylint: disable=too-many-arguments ) run_on_cmdline(logger, ["git", "reset", "--hard", "FETCH_HEAD"]) + run_on_cmdline( + logger, + ["git", "submodule", "update", "--init", "--recursive"], + env=_extend_env_for_non_interactive_mode(), + ) + + submodules = self.submodules() + current_sha = ( run_on_cmdline(logger, ["git", "rev-parse", "HEAD"]) .stdout.decode() @@ -304,7 +313,7 @@ def checkout_version( # pylint: disable=too-many-arguments if src: self.move_src_folder_up(remote, src) - return str(current_sha) + return str(current_sha), submodules def move_src_folder_up(self, remote: str, src: str) -> None: """Move the files from the src folder into the root of the project. diff --git a/features/fetch-git-repo-with-submodule.feature b/features/fetch-git-repo-with-submodule.feature new file mode 100644 index 00000000..75555274 --- /dev/null +++ b/features/fetch-git-repo-with-submodule.feature @@ -0,0 +1,89 @@ +Feature: Fetch projects with nested VCS dependencies + + Some projects include nested version control dependencies + such as Git submodules or other externals + These dependencies must be fetched at the exact revision + pinned by the parent repository to ensure reproducibility + + Background: + Given a git-repository "SomeInterestingProject.git" with the following submodules + | path | url | revision | + | ext/test-repo1 | https://github.com/dfetch-org/test-repo | e1fda19a57b873eb8e6ae37780594cbb77b70f1a | + | ext/test-repo2 | https://github.com/dfetch-org/test-repo | 8df389d0524863b85f484f15a91c5f2c40aefda1 | + + Scenario: A project with a git submodule is fetched at the pinned revision + Given the manifest 'dfetch.yaml' in MyProject + """ + manifest: + version: 0.0 + projects: + - name: my-project-with-submodules + url: some-remote-server/SomeInterestingProject.git + """ + When I run "dfetch update" + Then the output shows + """ + Dfetch (0.12.1) + my-project-with-submodules: + > Found & fetched submodule "./ext/test-repo1" (https://github.com/dfetch-org/test-repo @ main - e1fda19a57b873eb8e6ae37780594cbb77b70f1a) + > Found & fetched submodule "./ext/test-repo2" (https://github.com/dfetch-org/test-repo @ v1) + > Fetched master - 79698c99152e4a4b7b759c9def50a130bc91a2ff + """ + Then 'MyProject' looks like: + """ + MyProject/ + dfetch.yaml + my-project-with-submodules/ + .dfetch_data.yaml + README.md + ext/ + test-repo1/ + .git/ + .gitignore + LICENSE + README.md + test-repo2/ + .git/ + .gitignore + LICENSE + README.md + """ + + Scenario: Submodule changes are reported in the project report + Given a fetched and committed MyProject with the manifest + """ + manifest: + version: 0.0 + projects: + - name: my-project-with-submodules + url: some-remote-server/SomeInterestingProject.git + """ + When I run "dfetch report" in MyProject + Then the output shows + """ + Dfetch (0.12.1) + my-project-with-submodules: + - remote : + remote url : some-remote-server/SomeInterestingProject.git + branch : master + tag : + last fetch : 26/02/2026, 20:28:24 + revision : 79698c99152e4a4b7b759c9def50a130bc91a2ff + patch : + licenses : + + dependencies : + - path : ext/test-repo1 + url : https://github.com/dfetch-org/test-repo + branch : main + tag : + revision : e1fda19a57b873eb8e6ae37780594cbb77b70f1a + source-type : git-submodule + + - path : ext/test-repo2 + url : https://github.com/dfetch-org/test-repo + branch : + tag : v1 + revision : 8df389d0524863b85f484f15a91c5f2c40aefda1 + source-type : git-submodule + """ diff --git a/features/steps/generic_steps.py b/features/steps/generic_steps.py index 8d35a44f..fe3772d2 100644 --- a/features/steps/generic_steps.py +++ b/features/steps/generic_steps.py @@ -330,6 +330,11 @@ def step_impl(context, name): check_file(name, context.text) +@then("'{name}' exists") +def step_impl(_, name): + check_file_exists(name) + + def multisub(patterns: List[Tuple[Pattern[str], str]], text: str) -> str: """Apply a list of tuples that each contain a regex + replace string.""" for pattern, replace in patterns: diff --git a/features/steps/git_steps.py b/features/steps/git_steps.py index 73d61318..ec3359a2 100644 --- a/features/steps/git_steps.py +++ b/features/steps/git_steps.py @@ -39,18 +39,27 @@ def tag(name: str): subprocess.check_call(["git", "tag", "-a", name, "-m", "'Some tag'"]) +@given('a git-repository "{name}" with the following submodules') @given("a git repo with the following submodules") -def step_impl(context): - create_repo() +def step_impl(context, name=None): - for submodule in context.table: - subprocess.check_call( - ["git", "submodule", "add", submodule["url"], submodule["path"]] - ) + path = os.getcwd() + if name: + path = os.path.join(context.remotes_dir, name) + os.makedirs(path, exist_ok=True) + + with in_directory(path): + create_repo() + generate_file("README.md", "some content") + + for submodule in context.table: + subprocess.check_call( + ["git", "submodule", "add", submodule["url"], submodule["path"]] + ) - with in_directory(submodule["path"]): - subprocess.check_call(["git", "checkout", submodule["revision"]]) - commit_all("Added submodules") + with in_directory(submodule["path"]): + subprocess.check_call(["git", "checkout", submodule["revision"]]) + commit_all("Added submodules") @given('a new tag "{tagname}" is added to git-repository "{name}"') diff --git a/tests/test_subproject.py b/tests/test_subproject.py index b3503c29..3e5c5189 100644 --- a/tests/test_subproject.py +++ b/tests/test_subproject.py @@ -10,14 +10,14 @@ from dfetch.manifest.project import ProjectEntry from dfetch.manifest.version import Version -from dfetch.project.subproject import SubProject +from dfetch.project.subproject import SubProject, VcsDependency class ConcreteSubProject(SubProject): _wanted_version: Version - def _fetch_impl(self, version: Version) -> Version: - return Version() + def _fetch_impl(self, version: Version) -> tuple[Version, list[VcsDependency]]: + return Version(), [] def _latest_revision_on_branch(self, branch): return "latest"