diff options
Diffstat
| -rw-r--r-- | .github/workflows/ci.yml | 1 | +1 −0 |
| -rw-r--r-- | README.md | 16 | +16 −0 |
| -rwxr-xr-x | tasks/licenses-json.py | 249 | +249 −0 |
| -rwxr-xr-x | tests/licenses-json.py | 120 | +120 −0 |
4 files changed, 386 insertions, 0 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4783219..4da6f43 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: - name: Tests run: | + tests/licenses-json.py tests/oci-tasks.py tests/package.py diff --git a/README.md b/README.md index 057d394..6cd727f 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,22 @@ that cannot be installed as portable tools belong in `[bootstrap.packages]`. licensing metadata and canonical SPDX copyright headers. In Rust projects it also runs a pinned `cargo deny check`. +The separate `licenses-json` task uses cargo-about to generate a deterministic, +embeddable JSON bundle for a Rust binary. It accepts every license declared by +the dependency graph because policy enforcement remains the responsibility of +`cargo deny`. Repeat `--target` to produce one bundle for all supported targets: + +```console +mise run licenses-json -- \ + --manifest-path crates/server/Cargo.toml \ + --output crates/server/licenses.json \ + --target x86_64-unknown-linux-gnu \ + --target aarch64-unknown-linux-gnu +``` + +Use `--check` with the same arguments in CI to verify that a committed bundle is +up to date, or `--offline` when all dependency sources are already cached. + ### Sign-off policy `tasks/signoff.py` verifies that: diff --git a/tasks/licenses-json.py b/tasks/licenses-json.py new file mode 100755 --- /dev/null +++ b/tasks/licenses-json.py @@ -0,0 +1,249 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Generate embeddable Rust dependency license JSON" +#MISE tools={"pipx"="1.16.7","python"="3.14.7","cargo:cargo-about"="0.8.4"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import json +import re +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path +from typing import Any + +sys.dont_write_bytecode = True + +from libs.common import TaskError, capture, require_command, run, task_main + +TASK = "licenses-json" +# 0.8.4 intentionally matches the license-file deduplication behavior of the +# legacy in-tree generator. Keep the task tool pin and normalizer in sync. +SPDX_TOKEN = re.compile(r"[A-Za-z0-9][A-Za-z0-9.+:-]*|[()]") +SPDX_OPERATORS = {"AND", "OR", "WITH"} + + +def license_requirements(expression: str) -> list[str]: + """Return SPDX requirements in expression order, matching the legacy generator.""" + normalized = expression.replace("/", " OR ") + tokens = SPDX_TOKEN.findall(normalized) + requirements: list[str] = [] + + index = 0 + while index < len(tokens): + token = tokens[index] + if token in {"(", ")", "AND", "OR"}: + index += 1 + continue + if token == "WITH": + raise TaskError(f"{TASK}: invalid SPDX expression: {expression}") + + requirement = token + if index + 1 < len(tokens) and tokens[index + 1] == "WITH": + if index + 2 >= len(tokens) or tokens[index + 2] in SPDX_OPERATORS | { + "(", + ")", + }: + raise TaskError(f"{TASK}: invalid SPDX expression: {expression}") + requirement = f"{token} WITH {tokens[index + 2]}" + index += 2 + + if requirement not in requirements: + requirements.append(requirement) + index += 1 + + return requirements + + +def cargo_about_config(metadata: dict[str, Any]) -> str: + accepted_by_crate: dict[str, list[str]] = {} + for package in metadata.get("packages", []): + expression = package.get("license") + if not expression: + continue + + accepted = accepted_by_crate.setdefault(package["name"], []) + for requirement in license_requirements(expression): + if requirement not in accepted: + accepted.append(requirement) + + lines = [ + "accepted = []", + "private = { ignore = true }", + "ignore-build-dependencies = true", + "ignore-dev-dependencies = true", + "ignore-transitive-dependencies = false", + ] + for name, accepted in sorted(accepted_by_crate.items()): + lines.extend( + ("", f"[{json.dumps(name)}]", f"accepted = {json.dumps(accepted)}") + ) + + return "\n".join(lines) + "\n" + + +def normalized_output(report: dict[str, Any]) -> str: + licenses: list[dict[str, Any]] = [] + for source in report.get("licenses", []): + used_by = [ + { + "crate": { + "name": usage["crate"]["name"], + "version": usage["crate"]["version"], + "repository": usage["crate"].get("repository"), + } + } + for usage in source["used_by"] + ] + used_by.sort(key=lambda usage: len(usage["crate"]["name"])) + licenses.append( + { + "name": source["name"], + "id": source["id"], + "first_of_kind": False, + "text": source["text"], + "used_by": used_by, + } + ) + + licenses.sort(key=lambda license_: license_["id"]) + + overview_by_id: dict[str, dict[str, Any]] = {} + for license_ in licenses: + first = license_["id"] not in overview_by_id + license_["first_of_kind"] = first + overview = overview_by_id.setdefault( + license_["id"], + { + "count": 0, + "name": license_["name"], + "id": license_["id"], + }, + ) + overview["count"] += len(license_["used_by"]) + + overview = sorted(overview_by_id.values(), key=lambda item: item["name"]) + output = {"overview": overview, "licenses": licenses} + return json.dumps(output, ensure_ascii=False, indent=2) + "\n" + + +async def cargo_metadata(manifest_path: Path, offline: bool) -> dict[str, Any]: + arguments: list[str | Path] = [ + "cargo", + "metadata", + "--format-version", + "1", + "--locked", + "--manifest-path", + manifest_path, + ] + if offline: + arguments.append("--offline") + + output = await capture(arguments) + try: + return json.loads(output) + except json.JSONDecodeError as error: + raise TaskError( + f"{TASK}: cargo metadata returned invalid JSON: {error}" + ) from error + + +async def generate( + manifest_path: Path, + targets: Sequence[str], + offline: bool, +) -> str: + metadata = await cargo_metadata(manifest_path, offline) + + with tempfile.TemporaryDirectory(prefix=f"{TASK}-") as temporary: + work = Path(temporary) + config = work / "about.toml" + report = work / "report.json" + config.write_text(cargo_about_config(metadata), encoding="utf-8") + + arguments: list[str | Path] = [ + "cargo-about", + "generate", + "--config", + config, + "--manifest-path", + manifest_path, + "--format", + "json", + "--locked", + "--output-file", + report, + ] + if offline: + arguments.append("--offline") + for target in targets: + arguments.extend(("--target", target)) + + quiet_arguments = ["cargo-about", "-L", "off", *arguments[1:]] + result = await capture.result(quiet_arguments) + if result.exit_code != 0: + # Repeat with diagnostics enabled only on failure. cargo-about 0.8.4 + # otherwise reports harmless scanner errors for deprecated SPDX IDs. + await run(arguments) + try: + raw_report = json.loads(report.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise TaskError( + f"{TASK}: cargo-about returned invalid JSON: {error}" + ) from error + + return normalized_output(raw_report) + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser(prog="mise run licenses-json --") + command.add_argument( + "--manifest-path", + default=Path("Cargo.toml"), + type=Path, + ) + command.add_argument("--output", required=True, type=Path) + command.add_argument("--target", action="append", required=True) + command.add_argument("--offline", action="store_true") + command.add_argument("--check", action="store_true") + arguments = command.parse_args(args) + + require_command("cargo", TASK) + require_command("cargo-about", TASK) + if not arguments.manifest_path.is_file(): + raise TaskError(f"{TASK}: manifest not found: {arguments.manifest_path}") + + output = await generate( + arguments.manifest_path.resolve(), + arguments.target, + arguments.offline, + ) + if arguments.check: + if not arguments.output.is_file(): + raise TaskError(f"{TASK}: output not found: {arguments.output}") + if arguments.output.read_text(encoding="utf-8") != output: + raise TaskError( + f"{TASK}: {arguments.output} is out of date; regenerate it without --check" + ) + return + + arguments.output.parent.mkdir(parents=True, exist_ok=True) + if ( + not arguments.output.is_file() + or arguments.output.read_text(encoding="utf-8") != output + ): + arguments.output.write_text(output, encoding="utf-8") + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tests/licenses-json.py b/tests/licenses-json.py new file mode 100755 --- /dev/null +++ b/tests/licenses-json.py @@ -0,0 +1,120 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from types import ModuleType + + +def load_task() -> ModuleType: + path = Path(__file__).parents[1] / "tasks/licenses-json.py" + sys.path.insert(0, str(path.parent)) + spec = importlib.util.spec_from_file_location("licenses_json", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def main() -> None: + task = load_task() + + assert task.license_requirements("MIT/Apache-2.0") == ["MIT", "Apache-2.0"] + assert task.license_requirements( + "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" + ) == ["Apache-2.0 WITH LLVM-exception", "Apache-2.0", "MIT"] + + config = task.cargo_about_config( + { + "packages": [ + {"name": "dual", "license": "MIT OR Apache-2.0"}, + {"name": "dual", "license": "MIT/Apache-2.0"}, + {"name": "private", "license": None}, + ] + } + ) + assert "accepted = []\nprivate = { ignore = true }" in config + assert '["dual"]\naccepted = ["MIT", "Apache-2.0"]' in config + assert '["private"]' not in config + + output = json.loads( + task.normalized_output( + { + "overview": [], + "licenses": [ + { + "name": "MIT License", + "id": "MIT", + "first_of_kind": True, + "source_path": "/tmp/LICENSE", + "text": "MIT text", + "used_by": [ + { + "crate": { + "name": "long-name", + "version": "2.0.0", + "repository": None, + "manifest_path": "/tmp/Cargo.toml", + }, + "path": None, + }, + { + "crate": { + "name": "short", + "version": "1.0.0", + "repository": "https://example.invalid/short", + }, + "path": None, + }, + ], + }, + { + "name": "Apache License 2.0", + "id": "Apache-2.0", + "first_of_kind": False, + "text": "Apache text", + "used_by": [ + { + "crate": { + "name": "dependency", + "version": "3.0.0", + "repository": None, + } + } + ], + }, + ], + "crates": [{"package": {"manifest_path": "/tmp/Cargo.toml"}}], + } + ) + ) + + assert output["overview"] == [ + {"count": 1, "name": "Apache License 2.0", "id": "Apache-2.0"}, + {"count": 2, "name": "MIT License", "id": "MIT"}, + ] + assert [license_["id"] for license_ in output["licenses"]] == [ + "Apache-2.0", + "MIT", + ] + assert all(license_["first_of_kind"] for license_ in output["licenses"]) + assert [usage["crate"]["name"] for usage in output["licenses"][1]["used_by"]] == [ + "short", + "long-name", + ] + assert "source_path" not in output["licenses"][1] + assert "manifest_path" not in output["licenses"][1]["used_by"][0]["crate"] + assert "crates" not in output + + +if __name__ == "__main__": + main() |