26951957092

Committed 04 Jun 2026 05:51AM UTC coverage: 77.164% (+0.9%) from 76.298%

Build # 26951957092

Build Type

Pull #180

github

Committed by

web-flow

Commit Message

Merge branch 'main' into wenqil/sync-operator-names

Pull Request Pull Request #180: Sync post-consolidation HoloHub CLI changes (#1576, #1582, #1587, #1596)

Coverage Stats

63 of 68 new or added lines in 4 files covered. (92.65%)

1 existing line in 1 file now uncovered.

2960 of 3836 relevant lines covered (77.16%)

0.77 hits per line

Source File
Press 'n' to go to next uncovered line, 'b' for previous

95.93

/src/holoscan_cli/utils/external_resolver.py

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Parser for external Holoscan Module dependencies declared in metadata.json.

Produces :class:`ModuleDep` records consumed by :mod:`holoscan_cli.utils.cmake_manifest`
(the CMake backend) and any future resolver backends.

Layered architecture:

* Package identity / dependencies: metadata.json:dependencies[] (module schema v2).
* Workspace materialization: :mod:`holoscan_cli.utils.cmake_manifest` emits
  ``FetchContent_Declare`` entries; CMake performs the actual fetch.
* Build resolution: CMake. Reads the manifest's ``FetchContent_Declare`` entries
  plus ``HOLOHUB_EXT_OP_<op>_PROVIDER`` lookup table, and the consuming project's
  root ``CMakeLists.txt`` post-step calls ``FetchContent_MakeAvailable`` for any
  module whose operators ended up enabled (``OP_<x>=ON``).
"""

from __future__ import annotations

import json
import os
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional

_SHA_RE = re.compile(r"^[0-9a-f]{40}$")


def _ref_is_immutable(ref: str) -> bool:
    """True if ``ref`` looks like a full commit SHA (40 hex chars)."""
    return bool(_SHA_RE.match(ref or ""))


@dataclass
class ModuleDep:
    """A parsed module dependency from a consumer's metadata.json.

    ``is_internal=True`` means the module lives inside the active source tree
    under ``modules/<name>/`` and requires no FetchContent fetch; its operators
    are already part of that tree's normal CMake graph.
    """

    name: str
    git_url: Optional[str] = None
    ref: Optional[str] = None
    provides_operators: list[str] = field(default_factory=list)
    override_path: Optional[Path] = None
    is_internal: bool = False


def _override_env_name(module_name: str) -> str:
    """Translate a module name into the local-override env-var key.

    ``holoscan-example-utils`` -> ``HOLOSCAN_CLI_LOCAL_HOLOSCAN_EXAMPLE_UTILS``
    """
    sanitized = re.sub(r"[^A-Za-z0-9]+", "_", module_name).strip("_").upper()
    return f"HOLOSCAN_CLI_LOCAL_{sanitized}"


def _read_metadata(path: Path) -> dict:
    with path.open() as f:
        return json.load(f)


def _module_dependencies_raw(metadata: dict) -> list[dict]:
    """Pull the raw module dependency dicts out of either an app or module
    metadata.json. Returns ``[]`` if no external deps are declared."""
    if "module" in metadata:
        return list(metadata["module"].get("dependencies", []) or [])
    for key in ("application", "workflow", "benchmark"):
        if key in metadata:
            deps = metadata[key].get("dependencies") or {}
            if isinstance(deps, dict) and "modules" in deps:
                return list(deps["modules"] or [])
    return []


def parse_module_dependencies(
    metadata_path: Path,
    source_root: Optional[Path] = None,
    env: Optional[dict] = None,
) -> list[ModuleDep]:
    """Parse a metadata.json's external module dependency list into
    :class:`ModuleDep` records.

    Honors ``HOLOSCAN_CLI_LOCAL_<NAME>`` env-var overrides by populating
    ``override_path``. Pass ``env`` to override the process environment for
    override lookups (defaults to ``os.environ``). When ``source_root`` is
    provided, dependencies with no ``source`` block are checked against
    ``source_root/modules/<name>/``. If a metadata.json exists there, the
    dependency is treated as an in-tree module instead of an error.

    Does not fetch anything. A missing metadata.json is
    treated as "no external deps" rather than an error — unifies the
    file-doesn't-exist path with the file-vanished-between-exists-and-open
    race window.
    """
    try:
        metadata = _read_metadata(metadata_path)
    except FileNotFoundError:
        return []
    except json.JSONDecodeError as e:
        raise ValueError(f"Malformed JSON in {metadata_path}: {e}") from e
    _env = env if env is not None else os.environ
    raw = _module_dependencies_raw(metadata)
    out: list[ModuleDep] = []
    for entry in raw:
        name = entry.get("name")
        if not name:
            continue
        source = entry.get("source") or {}
        provides = list(entry.get("provides_operators") or [])

        override = _env.get(_override_env_name(name))
        override_path: Optional[Path] = None
        if override:
            p = Path(override).expanduser().resolve()
            if not (p / "metadata.json").exists():
                raise FileNotFoundError(
                    f"{_override_env_name(name)}={override} does not contain a "
                    "metadata.json — point it at the root of a Holoscan Module "
                    "project."
                )
            override_path = p

        ref = source.get("ref")
        git_url = source.get("git_url")
        if override_path is None:
            if not (git_url and ref):
                if source_root is not None:
                    in_tree_path = source_root / "modules" / name
                    if (in_tree_path / "metadata.json").exists():
                        out.append(
                            ModuleDep(
                                name=name,
                                provides_operators=provides,
                                override_path=in_tree_path,
                                is_internal=True,
                            )
                        )
                        continue
                raise ValueError(
                    f"Dependency '{name}' missing source.git_url or source.ref. "
                    f"Declare a complete source block, set "
                    f"{_override_env_name(name)}=<path>, or add a module descriptor "
                    f"at modules/{name}/metadata.json for in-tree modules."
                )
            if not _ref_is_immutable(ref):
                print(
                    f"WARNING: dependency '{name}' pinned to ref '{ref}', which "
                    "is not a 40-char commit SHA. Tags and branches are mutable; "
                    "consider pinning to an immutable SHA for reproducible builds.",
                    file=sys.stderr,
                )

        out.append(
            ModuleDep(
                name=name,
                git_url=git_url,
                ref=ref,
                provides_operators=provides,
                override_path=override_path,
            )
        )
    return out


def parse_module_sites(
    sites_path: Path,
    source_root: Optional[Path] = None,
    env: Optional[dict] = None,
) -> list[ModuleDep]:
    """Parse ``modules/module-sites.json`` into :class:`ModuleDep` records.

    External entries (url + ref present) become fetchable deps;
    ``provides_operators`` is read directly from the site entry and is
    authoritative. Project metadata serves only as a fallback via
    :func:`merge_deps`. In-tree entries (no url) resolve to ``is_internal=True``
    when ``source_root/modules/<name>/metadata.json`` exists; entries with
    neither a url nor an in-tree path are silently skipped.

    An entry with url but no ref (or ref but no url) raises ``ValueError`` —
    partial source specs hide typos and should fail loudly.

    Honors ``HOLOSCAN_CLI_LOCAL_<NAME>`` overrides with the same semantics as
    :func:`parse_module_dependencies`. Pass ``env`` to override the process
    environment for override lookups (defaults to ``os.environ``). A missing
    ``sites_path`` is treated as no module sites rather than an error.
    """
    try:
        with sites_path.open() as f:
            data = json.load(f)
    except FileNotFoundError:
        return []
    except json.JSONDecodeError as e:
        raise ValueError(f"Malformed JSON in {sites_path}: {e}") from e

    _env = env if env is not None else os.environ
    out: list[ModuleDep] = []
    for entry in data.get("modules") or []:
        name = entry.get("name")
        if not name:
            continue

        override = _env.get(_override_env_name(name))
        override_path: Optional[Path] = None
        if override:
            p = Path(override).expanduser().resolve()
            if not (p / "metadata.json").exists():
                raise FileNotFoundError(
                    f"{_override_env_name(name)}={override} does not contain a "
                    "metadata.json — point it at the root of a Holoscan Module "
                    "project."
                )
            override_path = p

        url = entry.get("url")
        ref = entry.get("ref")

        provides = list(entry.get("provides_operators") or [])

        if bool(url) != bool(ref):
            raise ValueError(
                f"module-sites entry '{name}' must declare both 'url' and 'ref' "
                "together, or neither for an in-tree/local-only module."
            )

        if url and ref:
            if not _ref_is_immutable(ref):
                print(
                    f"WARNING: module-sites entry '{name}' pinned to ref '{ref}', which "
                    "is not a 40-char commit SHA. Tags and branches are mutable; "
                    "consider pinning to an immutable SHA for reproducible builds.",
                    file=sys.stderr,
                )
            out.append(
                ModuleDep(
                    name=name,
                    git_url=url,
                    ref=ref,
                    provides_operators=provides,
                    override_path=override_path,
                )
            )
        elif override_path is not None:
            # No canonical url but a local override is active — treat as external.
            out.append(
                ModuleDep(name=name, provides_operators=provides, override_path=override_path)
            )
        elif source_root is not None:
            in_tree_path = source_root / "modules" / name
            if (in_tree_path / "metadata.json").exists():
                out.append(
                    ModuleDep(
                        name=name,
                        provides_operators=provides,
                        override_path=in_tree_path,
                        is_internal=True,
                    )
                )
            # else: not external and not in-tree — skip

    return out


def merge_deps(
    sites_deps: list[ModuleDep],
    project_deps: list[ModuleDep],
) -> list[ModuleDep]:
    """Merge module-sites deps with project-specific deps.

    Sites supply canonical git coordinates and own ``provides_operators``
    authoritatively; project deps provide ``override_path`` and serve as a
    fallback source for ``provides_operators`` when the site entry has none.
    For a module present in both lists the merged record takes the site's
    git_url/ref and is_internal classification, but the project dep's
    override_path. Modules only in ``project_deps`` are appended after all site
    entries (preserving sites order first).
    """
    project_by_name = {d.name: d for d in project_deps}
    seen: set[str] = set()
    result: list[ModuleDep] = []

    for sd in sites_deps:
        pd = project_by_name.get(sd.name)
        if pd is not None:
            # Sites owns the canonical git coords and is_internal classification.
            # Sites also owns provides_operators (authoritative module metadata);
            # project ops are used only as a fallback when sites has none.
            ops = sd.provides_operators if sd.provides_operators else pd.provides_operators
            result.append(
                ModuleDep(
                    name=sd.name,
                    git_url=sd.git_url,
                    ref=sd.ref,
                    provides_operators=ops,
                    override_path=pd.override_path,
                    is_internal=sd.is_internal,
                )
            )
        else:
            result.append(sd)
        seen.add(sd.name)

    for pd in project_deps:
        if pd.name not in seen:
            result.append(pd)

    return result

1	# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2	# SPDX-License-Identifier: Apache-2.0
3	#
4	# Licensed under the Apache License, Version 2.0 (the "License");
5	# you may not use this file except in compliance with the License.
6	# You may obtain a copy of the License at
7	#
8	# http://www.apache.org/licenses/LICENSE-2.0
9	#
10	# Unless required by applicable law or agreed to in writing, software
11	# distributed under the License is distributed on an "AS IS" BASIS,
12	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13	# See the License for the specific language governing permissions and
14	# limitations under the License.
15
16	"""Parser for external Holoscan Module dependencies declared in metadata.json.
17
18	Produces :class:`ModuleDep` records consumed by :mod:`holoscan_cli.utils.cmake_manifest`
19	(the CMake backend) and any future resolver backends.
20
21	Layered architecture:
22
23	* Package identity / dependencies: metadata.json:dependencies[] (module schema v2).
24	* Workspace materialization: :mod:`holoscan_cli.utils.cmake_manifest` emits
25	``FetchContent_Declare`` entries; CMake performs the actual fetch.
26	* Build resolution: CMake. Reads the manifest's ``FetchContent_Declare`` entries
27	plus ``HOLOHUB_EXT_OP_<op>_PROVIDER`` lookup table, and the consuming project's
28	root ``CMakeLists.txt`` post-step calls ``FetchContent_MakeAvailable`` for any
29	module whose operators ended up enabled (``OP_<x>=ON``).
30	"""
31
32	from __future__ import annotations	1✔
33
34	import json	1✔
35	import os	1✔
36	import re	1✔
37	import sys	1✔
38	from dataclasses import dataclass, field	1✔
39	from pathlib import Path	1✔
40	from typing import Optional	1✔
41
42	_SHA_RE = re.compile(r"^[0-9a-f]{40}$")	1✔
43
44
45	def _ref_is_immutable(ref: str) -> bool:	1✔
46	"""True if ``ref`` looks like a full commit SHA (40 hex chars)."""
47	return bool(_SHA_RE.match(ref or ""))	1✔
48
49
50	@dataclass	1✔
51	class ModuleDep:	1✔
52	"""A parsed module dependency from a consumer's metadata.json.
53
54	``is_internal=True`` means the module lives inside the active source tree
55	under ``modules/<name>/`` and requires no FetchContent fetch; its operators
56	are already part of that tree's normal CMake graph.
57	"""
58
59	name: str	1✔
60	git_url: Optional[str] = None	1✔
61	ref: Optional[str] = None	1✔
62	provides_operators: list[str] = field(default_factory=list)	1✔
63	override_path: Optional[Path] = None	1✔
64	is_internal: bool = False	1✔
65
66
67	def _override_env_name(module_name: str) -> str:	1✔
68	"""Translate a module name into the local-override env-var key.
69
70	``holoscan-example-utils`` -> ``HOLOSCAN_CLI_LOCAL_HOLOSCAN_EXAMPLE_UTILS``
71	"""
72	sanitized = re.sub(r"[^A-Za-z0-9]+", "_", module_name).strip("_").upper()	1✔
73	return f"HOLOSCAN_CLI_LOCAL_{sanitized}"	1✔
74
75
76	def _read_metadata(path: Path) -> dict:	1✔
77	with path.open() as f:	1✔
78	return json.load(f)	1✔
79
80
81	def _module_dependencies_raw(metadata: dict) -> list[dict]:	1✔
82	"""Pull the raw module dependency dicts out of either an app or module
83	metadata.json. Returns ``[]`` if no external deps are declared."""
84	if "module" in metadata:	1✔
85	return list(metadata["module"].get("dependencies", []) or [])	1✔
86	for key in ("application", "workflow", "benchmark"):	1✔
87	if key in metadata:	1✔
88	deps = metadata[key].get("dependencies") or {}	1✔
89	if isinstance(deps, dict) and "modules" in deps:	1✔
90	return list(deps["modules"] or [])	1✔
91	return []	1✔
92
93
94	def parse_module_dependencies(	1✔
95	metadata_path: Path,
96	source_root: Optional[Path] = None,
97	env: Optional[dict] = None,
98	) -> list[ModuleDep]:
99	"""Parse a metadata.json's external module dependency list into
100	:class:`ModuleDep` records.
101
102	Honors ``HOLOSCAN_CLI_LOCAL_<NAME>`` env-var overrides by populating
103	``override_path``. Pass ``env`` to override the process environment for
104	override lookups (defaults to ``os.environ``). When ``source_root`` is
105	provided, dependencies with no ``source`` block are checked against
106	``source_root/modules/<name>/``. If a metadata.json exists there, the
107	dependency is treated as an in-tree module instead of an error.
108
109	Does not fetch anything. A missing metadata.json is
110	treated as "no external deps" rather than an error — unifies the
111	file-doesn't-exist path with the file-vanished-between-exists-and-open
112	race window.
113	"""
114	try:	1✔
115	metadata = _read_metadata(metadata_path)	1✔
116	except FileNotFoundError:	1✔
117	return []	1✔
118	except json.JSONDecodeError as e:	1✔
119	raise ValueError(f"Malformed JSON in {metadata_path}: {e}") from e	1✔
120	_env = env if env is not None else os.environ	1✔
121	raw = _module_dependencies_raw(metadata)	1✔
122	out: list[ModuleDep] = []	1✔
123	for entry in raw:	1✔
124	name = entry.get("name")	1✔
125	if not name:	1✔
126	continue	1✔
127	source = entry.get("source") or {}	1✔
128	provides = list(entry.get("provides_operators") or [])	1✔
129
130	override = _env.get(_override_env_name(name))	1✔
131	override_path: Optional[Path] = None	1✔
132	if override:	1✔
133	p = Path(override).expanduser().resolve()	1✔
134	if not (p / "metadata.json").exists():	1✔
135	raise FileNotFoundError(	1✔
136	f"{_override_env_name(name)}={override} does not contain a "
137	"metadata.json — point it at the root of a Holoscan Module "
138	"project."
139	)
140	override_path = p	1✔
141
142	ref = source.get("ref")	1✔
143	git_url = source.get("git_url")	1✔
144	if override_path is None:	1✔
145	if not (git_url and ref):	1✔
146	if source_root is not None:	1✔
147	in_tree_path = source_root / "modules" / name	1✔
148	if (in_tree_path / "metadata.json").exists():	1✔
149	out.append(	1✔
150	ModuleDep(
151	name=name,
152	provides_operators=provides,
153	override_path=in_tree_path,
154	is_internal=True,
155	)
156	)
157	continue	1✔
158	raise ValueError(	1✔
159	f"Dependency '{name}' missing source.git_url or source.ref. "
160	f"Declare a complete source block, set "
161	f"{_override_env_name(name)}=<path>, or add a module descriptor "
162	f"at modules/{name}/metadata.json for in-tree modules."
163	)
164	if not _ref_is_immutable(ref):	1✔
165	print(	1✔
166	f"WARNING: dependency '{name}' pinned to ref '{ref}', which "
167	"is not a 40-char commit SHA. Tags and branches are mutable; "
168	"consider pinning to an immutable SHA for reproducible builds.",
169	file=sys.stderr,
170	)
171
172	out.append(	1✔
173	ModuleDep(
174	name=name,
175	git_url=git_url,
176	ref=ref,
177	provides_operators=provides,
178	override_path=override_path,
179	)
180	)
181	return out	1✔
182
183
184	def parse_module_sites(	1✔
185	sites_path: Path,
186	source_root: Optional[Path] = None,
187	env: Optional[dict] = None,
188	) -> list[ModuleDep]:
189	"""Parse ``modules/module-sites.json`` into :class:`ModuleDep` records.
190
191	External entries (url + ref present) become fetchable deps;
192	``provides_operators`` is read directly from the site entry and is
193	authoritative. Project metadata serves only as a fallback via
194	:func:`merge_deps`. In-tree entries (no url) resolve to ``is_internal=True``
195	when ``source_root/modules/<name>/metadata.json`` exists; entries with
196	neither a url nor an in-tree path are silently skipped.
197
198	An entry with url but no ref (or ref but no url) raises ``ValueError`` —
199	partial source specs hide typos and should fail loudly.
200
201	Honors ``HOLOSCAN_CLI_LOCAL_<NAME>`` overrides with the same semantics as
202	:func:`parse_module_dependencies`. Pass ``env`` to override the process
203	environment for override lookups (defaults to ``os.environ``). A missing
204	``sites_path`` is treated as no module sites rather than an error.
205	"""
206	try:	1✔
207	with sites_path.open() as f:	1✔
208	data = json.load(f)	1✔
209	except FileNotFoundError:	1✔
210	return []	1✔
NEW 211	except json.JSONDecodeError as e:	×
NEW 212	raise ValueError(f"Malformed JSON in {sites_path}: {e}") from e	×
213
214	_env = env if env is not None else os.environ	1✔
215	out: list[ModuleDep] = []	1✔
216	for entry in data.get("modules") or []:	1✔
217	name = entry.get("name")	1✔
218	if not name:	1✔
NEW 219	continue	×
220
221	override = _env.get(_override_env_name(name))	1✔
222	override_path: Optional[Path] = None	1✔
223	if override:	1✔
224	p = Path(override).expanduser().resolve()	1✔
225	if not (p / "metadata.json").exists():	1✔
NEW 226	raise FileNotFoundError(	×
227	f"{_override_env_name(name)}={override} does not contain a "
228	"metadata.json — point it at the root of a Holoscan Module "
229	"project."
230	)
231	override_path = p	1✔
232
233	url = entry.get("url")	1✔
234	ref = entry.get("ref")	1✔
235
236	provides = list(entry.get("provides_operators") or [])	1✔
237
238	if bool(url) != bool(ref):	1✔
239	raise ValueError(	1✔
240	f"module-sites entry '{name}' must declare both 'url' and 'ref' "
241	"together, or neither for an in-tree/local-only module."
242	)
243
244	if url and ref:	1✔
245	if not _ref_is_immutable(ref):	1✔
NEW 246	print(	×
247	f"WARNING: module-sites entry '{name}' pinned to ref '{ref}', which "
248	"is not a 40-char commit SHA. Tags and branches are mutable; "
249	"consider pinning to an immutable SHA for reproducible builds.",
250	file=sys.stderr,
251	)
252	out.append(	1✔
253	ModuleDep(
254	name=name,
255	git_url=url,
256	ref=ref,
257	provides_operators=provides,
258	override_path=override_path,
259	)
260	)
261	elif override_path is not None:	1✔
262	# No canonical url but a local override is active — treat as external.
263	out.append(	1✔
264	ModuleDep(name=name, provides_operators=provides, override_path=override_path)
265	)
266	elif source_root is not None:	1✔
267	in_tree_path = source_root / "modules" / name	1✔
268	if (in_tree_path / "metadata.json").exists():	1✔
269	out.append(	1✔
270	ModuleDep(
271	name=name,
272	provides_operators=provides,
273	override_path=in_tree_path,
274	is_internal=True,
275	)
276	)
277	# else: not external and not in-tree — skip
278
279	return out	1✔
280
281
282	def merge_deps(	1✔
283	sites_deps: list[ModuleDep],
284	project_deps: list[ModuleDep],
285	) -> list[ModuleDep]:
286	"""Merge module-sites deps with project-specific deps.
287
288	Sites supply canonical git coordinates and own ``provides_operators``
289	authoritatively; project deps provide ``override_path`` and serve as a
290	fallback source for ``provides_operators`` when the site entry has none.
291	For a module present in both lists the merged record takes the site's
292	git_url/ref and is_internal classification, but the project dep's
293	override_path. Modules only in ``project_deps`` are appended after all site
294	entries (preserving sites order first).
295	"""
296	project_by_name = {d.name: d for d in project_deps}	1✔
297	seen: set[str] = set()	1✔
298	result: list[ModuleDep] = []	1✔
299
300	for sd in sites_deps:	1✔
301	pd = project_by_name.get(sd.name)	1✔
302	if pd is not None:	1✔
303	# Sites owns the canonical git coords and is_internal classification.
304	# Sites also owns provides_operators (authoritative module metadata);
305	# project ops are used only as a fallback when sites has none.
306	ops = sd.provides_operators if sd.provides_operators else pd.provides_operators	1✔
307	result.append(	1✔
308	ModuleDep(
309	name=sd.name,
310	git_url=sd.git_url,
311	ref=sd.ref,
312	provides_operators=ops,
313	override_path=pd.override_path,
314	is_internal=sd.is_internal,
315	)
316	)
317	else:
318	result.append(sd)	1✔
319	seen.add(sd.name)	1✔
320
321	for pd in project_deps:	1✔
322	if pd.name not in seen:	1✔
323	result.append(pd)	1✔
324
325	return result	1✔

nvidia-holoscan / holoscan-cli / 26951957092

Source File Press 'n' to go to next uncovered line, 'b' for previous

Source File
Press 'n' to go to next uncovered line, 'b' for previous