From ac24d13cb672e8c22ccb08af102286486b7d091f Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Mon, 24 Aug 2026 07:03:14 +0100 Subject: Rewrite bash scripts to python --- .github/workflows/ci.yml | 15 +- README.md | 18 +- mise.lock | 80 ++++---- mise.toml | 2 +- tasks/_apk.py | 146 --------------- tasks/_apt.py | 87 --------- tasks/_lib.py | 153 ---------------- tasks/_repository.py | 111 ----------- tasks/_rpm.py | 54 ------ tasks/_storage.py | 155 ---------------- tasks/chart | 111 ----------- tasks/chart.py | 87 +++++++++ tasks/container | 173 ------------------ tasks/container.py | 105 +++++++++++ tasks/libs/__init__.py | 2 + tasks/libs/apk.py | 144 +++++++++++++++ tasks/libs/apt.py | 79 ++++++++ tasks/libs/common.py | 178 ++++++++++++++++++ tasks/libs/repository.py | 111 +++++++++++ tasks/libs/rpm.py | 54 ++++++ tasks/libs/storage.py | 156 ++++++++++++++++ tasks/licenses | 50 ----- tasks/licenses.py | 95 ++++++++++ tasks/package | 236 ------------------------ tasks/package.py | 224 +++++++++++++++++++++++ tasks/publish | 76 -------- tasks/publish.py | 66 +++++++ tasks/signoff | 155 ---------------- tasks/signoff.py | 208 +++++++++++++++++++++ tests/oci-tasks | 87 --------- tests/oci-tasks.py | 147 +++++++++++++++ tests/package | 177 ------------------ tests/package-integration | 42 ----- tests/package-integration.py | 93 ++++++++++ tests/package.py | 270 +++++++++++++++++++++++++++ tests/publish-integration | 185 ------------------- tests/publish-integration.py | 346 +++++++++++++++++++++++++++++++++++ 37 files changed, 2416 insertions(+), 2062 deletions(-) delete mode 100644 tasks/_apk.py delete mode 100644 tasks/_apt.py delete mode 100644 tasks/_lib.py delete mode 100644 tasks/_repository.py delete mode 100644 tasks/_rpm.py delete mode 100644 tasks/_storage.py delete mode 100755 tasks/chart create mode 100755 tasks/chart.py delete mode 100755 tasks/container create mode 100755 tasks/container.py create mode 100644 tasks/libs/__init__.py create mode 100644 tasks/libs/apk.py create mode 100644 tasks/libs/apt.py create mode 100644 tasks/libs/common.py create mode 100644 tasks/libs/repository.py create mode 100644 tasks/libs/rpm.py create mode 100644 tasks/libs/storage.py delete mode 100755 tasks/licenses create mode 100755 tasks/licenses.py delete mode 100755 tasks/package create mode 100755 tasks/package.py delete mode 100755 tasks/publish create mode 100755 tasks/publish.py delete mode 100755 tasks/signoff create mode 100755 tasks/signoff.py delete mode 100755 tests/oci-tasks create mode 100755 tests/oci-tasks.py delete mode 100755 tests/package delete mode 100755 tests/package-integration create mode 100755 tests/package-integration.py create mode 100755 tests/package.py delete mode 100755 tests/publish-integration create mode 100755 tests/publish-integration.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5835033..4783219 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,17 +30,14 @@ jobs: - name: Check Python run: | - ruff check tasks/_*.py tasks/package tasks/publish tests/fakes/boto3.py - ruff format --check tasks/_*.py tasks/package tasks/publish tests/fakes/boto3.py - - - name: Check shell - run: sh -n tasks/chart tasks/container tests/oci-tasks + ruff check tasks tests + ruff format --check tasks tests - name: Tests run: | - tests/oci-tasks + tests/oci-tasks.py - tests/package - tests/package-integration + tests/package.py + tests/package-integration.py - tests/publish-integration + tests/publish-integration.py diff --git a/README.md b/README.md index bc1517a..057d394 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ Consuming projects pin this repository by commit SHA. ## Packaging Projects build and stage their own binaries and keep their nFPM configuration. -The shared [`package`](tasks/package) task creates only the formats explicitly +The shared [`package`](tasks/package.py) task creates only the formats explicitly requested by a project: nFPM packages (`deb`, `rpm`, or `apk`) and portable archives (`tar.gz` or `zip`). APK configurations may use `${PACKAGE_KEY_VERSION}` in `apk.signature.key_name`; the task renders the four-digit generation before invoking nFPM. DEB and RPM payloads are built by nFPM and then signed through `debsigs` and `rpmsign`, allowing CI to use only an OpenPGP signing subkey while the certification key remains offline. The shared -[`publish`](tasks/publish) task adds +[`publish`](tasks/publish.py) task adds explicitly selected package formats to signed repositories in the organization package bucket. @@ -43,10 +43,10 @@ mise run package -- \ ## OCI artifacts -The shared [`container`](tasks/container) task builds one or more tagged OCI +The shared [`container`](tasks/container.py) task builds one or more tagged OCI images with Docker Buildx. Registry authentication is deliberately left to the calling workflow, so the same build can be pushed to GHCR, Cloudflare, or -another OCI registry. The [`chart`](tasks/chart) task strictly lints a Helm +another OCI registry. The [`chart`](tasks/chart.py) task strictly lints a Helm chart, packages an immutable version, and can push it to one or more OCI repositories. @@ -110,20 +110,22 @@ mise bootstrap ``` Shared tasks declare task-specific tools in their `#MISE tools` metadata, so -`mise run` installs the same pinned versions on demand. System libraries that -cannot be installed as portable tools belong in `[bootstrap.packages]`. +`mise run` installs the same pinned versions on demand. Python tasks declare +their dependencies inline with PEP 723 and run through pipx's standard pip +backend, without a project virtual environment or install step. System libraries +that cannot be installed as portable tools belong in `[bootstrap.packages]`. ## Guardrails ### Licensing policy -`tasks/licenses` runs a pinned REUSE version and verifies the repository's +`tasks/licenses.py` runs a pinned REUSE version and verifies the repository's licensing metadata and canonical SPDX copyright headers. In Rust projects it also runs a pinned `cargo deny check`. ### Sign-off policy -`tasks/signoff` verifies that: +`tasks/signoff.py` verifies that: - authors and co-authors with an email from `config/signoff-approved-emails` are trusted without a trailer; diff --git a/mise.lock b/mise.lock index 04ef64a..9305d16 100644 --- a/mise.lock +++ b/mise.lock @@ -1,5 +1,39 @@ # @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html +[[tools.pipx]] +version = "1.16.7" +backend = "aqua:pypa/pipx" + +[tools.pipx."platforms.linux-arm64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.linux-arm64-musl"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.linux-x64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.linux-x64-musl"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.macos-arm64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.macos-x64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + [[tools.ruff]] version = "0.16.3" backend = "aqua:astral-sh/ruff" @@ -45,49 +79,3 @@ checksum = "sha256:f10c709755b393fd9821506b21070bcca969b9966504edd1e490efd08e366 url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-x86_64-pc-windows-msvc.zip" url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140346" provenance = "github-attestations" - -[[tools.uv]] -version = "0.12.5" -backend = "aqua:astral-sh/uv" - -[tools.uv."platforms.linux-arm64"] -checksum = "sha256:8767a0e77f2cd45436401b1b42bf7e9ed5a4a91a74a5305d6fe93249d0f6dbc5" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850993" -provenance = "github-attestations" - -[tools.uv."platforms.linux-arm64-musl"] -checksum = "sha256:8767a0e77f2cd45436401b1b42bf7e9ed5a4a91a74a5305d6fe93249d0f6dbc5" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850993" -provenance = "github-attestations" - -[tools.uv."platforms.linux-x64"] -checksum = "sha256:a4742988791c9aeae68c78150d6cba762062ad2a47e53738c2779d2b596bfcdb" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851105" -provenance = "github-attestations" - -[tools.uv."platforms.linux-x64-musl"] -checksum = "sha256:a4742988791c9aeae68c78150d6cba762062ad2a47e53738c2779d2b596bfcdb" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851105" -provenance = "github-attestations" - -[tools.uv."platforms.macos-arm64"] -checksum = "sha256:5bb0e5fe008a773c3dbcb97ff79cd89e1241464fe9d2f986d52ad8f1b037bd62" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-apple-darwin.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850968" -provenance = "github-attestations" - -[tools.uv."platforms.macos-x64"] -checksum = "sha256:b3b2137477cf96c9686ebfb71524614cec780c673fd73e59bce099aef02e70e8" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-apple-darwin.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851078" -provenance = "github-attestations" - -[tools.uv."platforms.windows-x64"] -checksum = "sha256:4c4d49d8738847d9b71ba319e49a5688c93eac0fe6204b1df24e98528dddf39a" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-pc-windows-msvc.zip" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851084" -provenance = "github-attestations" diff --git a/mise.toml b/mise.toml index e4a7671..449df01 100644 --- a/mise.toml +++ b/mise.toml @@ -7,8 +7,8 @@ min_version = "2026.7.5" experimental = true [tools] +pipx = "1.16.7" ruff = "0.16.3" -uv = "0.12.5" [bootstrap.packages] # APT diff --git a/tasks/_apk.py b/tasks/_apk.py deleted file mode 100644 index 70d156c..0000000 --- a/tasks/_apk.py +++ /dev/null @@ -1,146 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -from __future__ import annotations - -import gzip -import hashlib -import io -import os -import platform -import shutil -import subprocess -import tarfile -import urllib.request -from pathlib import Path - -from _lib import TaskError, run - -TASK = "publish" -APK_TOOLS_VERSION = "2.14.10-r0" -APK_TOOLS_SHA256 = { - "x86_64": "c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61", - "aarch64": "3e22f80dd0272dc487e4ca84b2c6b660ca392cbad970764efe9ef9555b806ac8", -} - - -def architecture(package: Path) -> str: - metadata = run( - ["tar", "-xOzf", package, ".PKGINFO"], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ).stdout - for line in metadata.splitlines(): - if line.startswith("arch = "): - return line.removeprefix("arch = ") - raise TaskError(f"{TASK}: cannot read APK architecture: {package}") - - -def apk_tool(work: Path) -> Path: - configured = os.environ.get("APK_TOOL") - if configured: - tool = Path(configured) - if os.access(tool, os.X_OK): - return tool - raise TaskError(f"{TASK}: APK_TOOL is not executable: {tool}") - for name in ("apk.static", "apk"): - if command := shutil.which(name): - return Path(command) - - machine = platform.machine() - apk_arch = { - "x86_64": "x86_64", - "amd64": "x86_64", - "aarch64": "aarch64", - "arm64": "aarch64", - }.get(machine) - if apk_arch is None: - raise TaskError(f"{TASK}: apk-tools is unavailable for {machine}") - archive = work / "apk-tools-static.apk" - url = ( - "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/" - f"{apk_arch}/apk-tools-static-{APK_TOOLS_VERSION}.apk" - ) - urllib.request.urlretrieve(url, archive) - if hashlib.sha256(archive.read_bytes()).hexdigest() != APK_TOOLS_SHA256[apk_arch]: - raise TaskError(f"{TASK}: apk-tools checksum mismatch") - directory = work / "apk-tools" - directory.mkdir() - run( - ["tar", "-xzf", archive, "-C", directory, "sbin/apk.static"], - stderr=subprocess.DEVNULL, - ) - return directory / "sbin" / "apk.static" - - -def sign_index(context, index: Path) -> None: - name = f".SIGN.RSA256.{context.apk_signing.public_key_name}" - signature = index.parent / name - run( - [ - "openssl", - "dgst", - "-sha256", - "-sign", - context.apk_signing.private_key_file, - "-out", - signature, - index, - ], - env=context.apk_signing.environment, - ) - data = signature.read_bytes() - stream = io.BytesIO() - with tarfile.open(fileobj=stream, mode="w", format=tarfile.USTAR_FORMAT) as archive: - information = tarfile.TarInfo(name) - information.size = len(data) - information.mode = 0o644 - information.mtime = int(os.environ.get("SOURCE_DATE_EPOCH", "0")) - archive.addfile(information, io.BytesIO(data)) - size = 512 + ((len(data) + 511) // 512) * 512 - index.write_bytes( - gzip.compress(stream.getvalue()[:size], mtime=0) + index.read_bytes() - ) - signature.unlink() - - -def publish(context) -> None: - tool = apk_tool(context.work) - packages = context.packages["apk"] - architectures = sorted({architecture(package) for package in packages}) - keys = context.work / "apk-keys" - keys.mkdir() - for public_key in context.apk_public_keys: - shutil.copy2(public_key, keys / public_key.name) - - for apk_arch in architectures: - root = context.work / "apk" / apk_arch - root.mkdir(parents=True) - remote = context.storage.service_key("apk", context.channel, apk_arch) - context.storage.download_prefix(remote, root, "*.apk") - for package in packages: - if architecture(package) == apk_arch: - context.add_package(package, root) - for package in root.glob("*.apk"): - if run( - [tool, "verify", "--keys-dir", keys, package], check=False - ).returncode: - raise TaskError( - f"{TASK}: APK signature verification failed: {package.name}" - ) - index = root / "APKINDEX.tar.gz" - run( - [ - tool, - "--allow-untrusted", - "index", - "--description", - f"Dimidium Labs {context.service} {context.channel}", - "--output", - index, - *root.glob("*.apk"), - ] - ) - sign_index(context, index) - context.storage.upload_payloads(root, remote, "*.apk") - context.storage.upload(index, f"{remote}/APKINDEX.tar.gz") diff --git a/tasks/_apt.py b/tasks/_apt.py deleted file mode 100644 index d153d9b..0000000 --- a/tasks/_apt.py +++ /dev/null @@ -1,87 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -from __future__ import annotations - -import subprocess - -from _lib import TaskError, require_command, run - -TASK = "publish" - - -def publish(context) -> None: - require_command("apt-ftparchive", TASK) - require_command("dpkg-deb", TASK) - root = context.work / "apt" - pool = root / "pool" / context.channel - metadata = root / "dists" / context.channel - pool.mkdir(parents=True) - pool_prefix = context.storage.service_key("apt", "pool", context.channel) - metadata_prefix = context.storage.service_key("apt", "dists", context.channel) - context.storage.download_prefix(pool_prefix, pool, "*.deb") - for package in context.packages["deb"]: - context.add_package(package, pool) - - architectures = sorted( - { - run( - ["dpkg-deb", "-f", package, "Architecture"], - stdout=subprocess.PIPE, - ).stdout.strip() - for package in pool.glob("*.deb") - } - ) - if not architectures or "" in architectures: - raise TaskError(f"{TASK}: no DEB architectures found") - for architecture in architectures: - (metadata / "main" / f"binary-{architecture}").mkdir( - parents=True, exist_ok=True - ) - - cache = context.work / "apt-cache" - cache.mkdir() - config = context.work / "apt-ftparchive.conf" - architecture_list = " ".join(architectures) - config.write_text( - f'''Dir {{ ArchiveDir "{root}"; CacheDir "{cache}"; }}; -Default {{ Packages::Compress ". gzip"; Packages::Extensions ".deb"; }}; -TreeDefault {{ - Packages "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"; - BinCacheDB "packages-$(ARCH).db"; -}}; -Tree "dists/{context.channel}" {{ - Sections "main"; - Architectures "{architecture_list}"; - Directory "pool/{context.channel}"; -}}; -''' - ) - run(["apt-ftparchive", "generate", config]) - release = metadata / "Release" - with release.open("wb") as stream: - run( - [ - "apt-ftparchive", - "-o", - "APT::FTPArchive::Release::Origin=Dimidium Labs", - "-o", - f"APT::FTPArchive::Release::Label={context.service} {context.channel}", - "-o", - f"APT::FTPArchive::Release::Suite={context.channel}", - "-o", - f"APT::FTPArchive::Release::Codename={context.channel}", - "-o", - "APT::FTPArchive::Release::Components=main", - "-o", - f"APT::FTPArchive::Release::Architectures={architecture_list}", - "release", - f"{metadata}/", - ], - stdout=stream, - text=False, - ) - context.gpg.sign(metadata / "Release.gpg", "--armor", "--detach-sign", release) - context.gpg.sign(metadata / "InRelease", "--clearsign", release) - context.storage.upload_payloads(pool, pool_prefix, "*.deb") - context.storage.replace_prefix(metadata, metadata_prefix) diff --git a/tasks/_lib.py b/tasks/_lib.py deleted file mode 100644 index 051f71d..0000000 --- a/tasks/_lib.py +++ /dev/null @@ -1,153 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -from __future__ import annotations - -import os -import shutil -import subprocess -from pathlib import Path - - -class TaskError(RuntimeError): - pass - - -def require_command(name: str, task: str) -> str: - command = shutil.which(name) - if command is None: - raise TaskError(f"{task}: {name} is required") - return command - - -def required_env(name: str, task: str, purpose: str = "") -> str: - value = os.environ.get(name) - if value: - return value - suffix = f" {purpose}" if purpose else "" - raise TaskError(f"{task}: {name} is required{suffix}") - - -def run(args, **options) -> subprocess.CompletedProcess[str]: - check = options.pop("check", True) - options.setdefault("text", True) - if "input_text" in options: - options["input"] = options.pop("input_text") - return subprocess.run([str(arg) for arg in args], check=check, **options) - - -class GPGSigning: - def __init__(self, task: str, work: Path): - require_command("gpg", task) - private_key = required_env("GPG_PRIVATE_KEY", task) - self.passphrase = required_env("GPG_PASSPHRASE", task) - self.key_id = required_env("GPG_KEY_ID", task) - self.short_key_id = self.key_id[-16:] - self.home = work / "gnupg" - self.home.mkdir(mode=0o700) - self.private_key_file = work / "signing.asc" - self.private_key_file.write_text(private_key) - self.private_key_file.chmod(0o600) - self.environment = dict(os.environ) - self.environment["GNUPGHOME"] = str(self.home) - self.environment.pop("GPG_PRIVATE_KEY", None) - self.environment.pop("GPG_PASSPHRASE", None) - self.environment.pop("APK_PRIVATE_KEY", None) - run( - [ - "gpg", - "--batch", - "--yes", - "--pinentry-mode", - "loopback", - "--passphrase-fd", - "0", - "--import", - self.private_key_file, - ], - env=self.environment, - input_text=f"{self.passphrase}\n", - ) - - def package_environment(self) -> dict[str, str]: - environment = dict(self.environment) - environment["GPG_KEY_ID"] = self.short_key_id - return environment - - def prime_agent(self) -> None: - signature = self.private_key_file.with_suffix(".sig") - self.sign(signature, "--detach-sign", self.private_key_file) - signature.unlink() - - def export_public_key(self, output: Path) -> None: - with output.open("wb") as stream: - run( - ["gpg", "--batch", "--yes", "--armor", "--export", self.key_id], - env=self.environment, - stdout=stream, - text=False, - ) - - def verify_public_bundle(self, bundle: Path) -> None: - fingerprints = { - line.split(":")[9] - for line in run( - ["gpg", "--batch", "--with-colons", "--show-keys", bundle], - env=self.environment, - stdout=subprocess.PIPE, - ).stdout.splitlines() - if line.startswith("fpr:") - } - if self.key_id not in fingerprints: - raise TaskError( - f"{self.task}: packages.gpg does not contain signing key {self.key_id}" - ) - - def sign(self, output: Path, *arguments: str | Path) -> None: - run( - [ - "gpg", - f"--default-key={self.key_id}", - "--batch", - "--yes", - "--pinentry-mode", - "loopback", - "--passphrase-fd", - "0", - "-o", - output, - *arguments, - ], - env=self.environment, - input_text=f"{self.passphrase}\n", - ) - - -class APKSigning: - def __init__(self, task: str, work: Path, key_name: str = "packages"): - require_command("openssl", task) - private_key = required_env("APK_PRIVATE_KEY", task, "for APK signing") - self.key_name = key_name - self.public_key_name = f"{key_name}.rsa.pub" - self.private_key_file = work / f"{key_name}.rsa" - self.private_key_file.write_text(private_key) - self.private_key_file.chmod(0o600) - self.environment = dict(os.environ) - self.environment.pop("APK_PRIVATE_KEY", None) - self.environment.pop("GPG_PRIVATE_KEY", None) - self.environment.pop("GPG_PASSPHRASE", None) - self.environment["APK_SIGNING_KEY"] = str(self.private_key_file) - - def export_public_key(self, output: Path) -> None: - run( - [ - "openssl", - "rsa", - "-in", - self.private_key_file, - "-pubout", - "-out", - output, - ], - env=self.environment, - ) diff --git a/tasks/_repository.py b/tasks/_repository.py deleted file mode 100644 index 577b007..0000000 --- a/tasks/_repository.py +++ /dev/null @@ -1,111 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -from __future__ import annotations - -import re -import shutil -from pathlib import Path - -from _lib import APKSigning, GPGSigning, TaskError, required_env -from _storage import S3Storage - -TASK = "publish" -KEY_VERSION = re.compile(r"^[0-9]{4}$") -RSA_PUBLIC_KEY = re.compile(r"^keys/packages\.[0-9]{4}\.rsa\.pub$") - - -class Repository: - def __init__(self, service, channel, input_directory, formats, work): - self.service = service - self.channel = channel - self.formats = formats - self.work = work - self.public_url = required_env("S3_PUBLIC_URL", TASK).rstrip("/") - self.key_version = required_env("PACKAGE_KEY_VERSION", TASK) - if not KEY_VERSION.fullmatch(self.key_version): - raise TaskError(f"{TASK}: invalid PACKAGE_KEY_VERSION: {self.key_version}") - self.storage = S3Storage(TASK, service) - self.gpg = None - self.gpg_public_key = None - self.apk_signing = None - self.apk_public_keys = [] - self.packages = { - package_format: sorted(input_directory.glob(f"*.{package_format}")) - for package_format in formats - } - for package_format, packages in self.packages.items(): - if not packages: - raise TaskError( - f"{TASK}: no .{package_format} packages found in {input_directory}" - ) - - def check_public_key(self, source: Path, key: str) -> None: - existing = self.work / f"existing-{source.name}" - if not self.storage.download(key, existing): - raise TaskError(f"{TASK}: organization key {key} is not provisioned") - if source.read_bytes() != existing.read_bytes(): - raise TaskError(f"{TASK}: signing key does not match {key}") - - @staticmethod - def add_package(source: Path, directory: Path) -> None: - destination = directory / source.name - if destination.exists() and source.read_bytes() != destination.read_bytes(): - raise TaskError( - f"{TASK}: immutable package filename has different content: {source.name}" - ) - if not destination.exists(): - shutil.copy2(source, destination) - - def setup_openpgp(self) -> None: - self.gpg = GPGSigning(TASK, self.work) - current = self.work / "current-packages.gpg" - self.gpg.export_public_key(current) - self.check_public_key(current, f"keys/packages.{self.key_version}.gpg") - - bundle = self.work / "packages.gpg" - if not self.storage.download("packages.gpg", bundle): - raise TaskError(f"{TASK}: organization key packages.gpg is not provisioned") - self.gpg.verify_public_bundle(bundle) - self.gpg_public_key = bundle - - def setup_rsa(self) -> None: - key_name = f"packages.{self.key_version}" - self.apk_signing = APKSigning(TASK, self.work, key_name) - current = self.work / self.apk_signing.public_key_name - self.apk_signing.export_public_key(current) - self.check_public_key( - current, - f"keys/{self.apk_signing.public_key_name}", - ) - - key_directory = self.work / "rsa-public-keys" - key_directory.mkdir() - for key in sorted(self.storage.objects("keys/")): - if RSA_PUBLIC_KEY.fullmatch(key): - destination = key_directory / Path(key).name - if destination.exists(): - raise TaskError(f"{TASK}: duplicate RSA public key name: {key}") - self.storage.download(key, destination) - self.apk_public_keys.append(destination) - if current.name not in {key.name for key in self.apk_public_keys}: - raise TaskError( - f"{TASK}: current RSA public key is absent from key archive" - ) - - def setup_signing(self) -> None: - if {"deb", "rpm"} & set(self.formats): - self.setup_openpgp() - if "apk" in self.formats: - self.setup_rsa() - - def publish(self) -> None: - from _apk import publish as publish_apk - from _apt import publish as publish_apt - from _rpm import publish as publish_rpm - - publishers = {"deb": publish_apt, "rpm": publish_rpm, "apk": publish_apk} - with self.storage.lock(self.channel): - self.setup_signing() - for package_format in self.formats: - publishers[package_format](self) diff --git a/tasks/_rpm.py b/tasks/_rpm.py deleted file mode 100644 index c2f476f..0000000 --- a/tasks/_rpm.py +++ /dev/null @@ -1,54 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -from __future__ import annotations - -import shutil -import subprocess - -from _lib import TaskError, require_command, run - -TASK = "publish" - - -def publish(context) -> None: - require_command("createrepo_c", TASK) - require_command("rpmkeys", TASK) - root = context.work / "rpm" - root.mkdir() - remote = context.storage.service_key("rpm", context.channel) - context.storage.download_prefix(remote, root, "*.rpm") - for package in context.packages["rpm"]: - context.add_package(package, root) - - rpm_database = context.work / "rpmdb" - rpm_database.mkdir() - run(["rpmkeys", "--dbpath", rpm_database, "--import", context.gpg_public_key]) - for package in root.glob("*.rpm"): - result = run( - ["rpmkeys", "--dbpath", rpm_database, "--checksig", package], - stdout=subprocess.PIPE, - ).stdout - if "signatures OK" not in result: - raise TaskError( - f"{TASK}: RPM is not signed by a trusted key: {package.name}" - ) - - shutil.rmtree(root / "repodata", ignore_errors=True) - run(["createrepo_c", root]) - repomd = root / "repodata" / "repomd.xml" - context.gpg.sign(repomd.with_suffix(".xml.asc"), "--armor", "--detach-sign", repomd) - definition = root / f"{context.service}-{context.channel}.repo" - definition.write_text( - f"""[{context.service}-{context.channel}] -name={context.service} {context.channel} -gpgkey={context.public_url}/packages.gpg -baseurl={context.public_url}/{context.service}/rpm/{context.channel}/ -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -""" - ) - context.storage.upload_payloads(root, remote, "*.rpm") - context.storage.upload(definition, f"{remote}/{definition.name}") - context.storage.replace_prefix(root / "repodata", f"{remote}/repodata") diff --git a/tasks/_storage.py b/tasks/_storage.py deleted file mode 100644 index 1fc163e..0000000 --- a/tasks/_storage.py +++ /dev/null @@ -1,155 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -from __future__ import annotations - -import fnmatch -import hashlib -import json -import time -import uuid -from contextlib import contextmanager -from pathlib import Path - -import boto3 -from _lib import TaskError, required_env -from botocore.exceptions import ClientError - - -class S3Storage: - def __init__(self, task: str, service: str): - self.task = task - self.bucket = required_env("S3_BUCKET", task) - self.service_root = service - self.client = boto3.client( - "s3", - endpoint_url=required_env("S3_ENDPOINT", task), - aws_access_key_id=required_env("S3_ACCESS_KEY_ID", task), - aws_secret_access_key=required_env("S3_SECRET_ACCESS_KEY", task), - region_name="auto", - ) - - def service_key(self, *parts: str) -> str: - return "/".join((self.service_root, *parts)) - - def download(self, key: str, destination: Path) -> bool: - destination.parent.mkdir(parents=True, exist_ok=True) - try: - self.client.download_file(self.bucket, key, str(destination)) - except ClientError as error: - if error.response.get("Error", {}).get("Code") in { - "404", - "NoSuchKey", - "NotFound", - }: - return False - raise - return True - - def objects(self, prefix: str) -> set[str]: - pages = self.client.get_paginator("list_objects_v2").paginate( - Bucket=self.bucket, Prefix=prefix - ) - return {item["Key"] for page in pages for item in page.get("Contents", [])} - - def download_prefix(self, prefix: str, destination: Path, pattern: str) -> None: - destination.mkdir(parents=True, exist_ok=True) - prefix = prefix.rstrip("/") + "/" - for key in self.objects(prefix): - relative = key.removeprefix(prefix).lstrip("/") - if relative and "/" not in relative and fnmatch.fnmatch(relative, pattern): - self.client.download_file(self.bucket, key, str(destination / relative)) - - @staticmethod - def digest(path: Path) -> str: - with path.open("rb") as stream: - return hashlib.file_digest(stream, "sha256").hexdigest() - - @staticmethod - def conflict(error: ClientError) -> bool: - return error.response.get("Error", {}).get("Code") in { - "409", - "412", - "ConditionalRequestConflict", - "PreconditionFailed", - } - - def upload_immutable(self, source: Path, key: str) -> None: - digest = self.digest(source) - try: - with source.open("rb") as stream: - self.client.put_object( - Bucket=self.bucket, - Key=key, - Body=stream, - Metadata={"sha256": digest}, - IfNoneMatch="*", - ) - return - except ClientError as error: - if not self.conflict(error): - raise - - existing = self.client.head_object(Bucket=self.bucket, Key=key) - existing_digest = existing.get("Metadata", {}).get("sha256") - if not existing_digest: - body = self.client.get_object(Bucket=self.bucket, Key=key)["Body"] - existing_digest = hashlib.sha256(body.read()).hexdigest() - if existing_digest != digest: - raise TaskError( - f"{self.task}: immutable object has different content: {key}" - ) - - def upload_payloads(self, source: Path, prefix: str, pattern: str) -> None: - for path in sorted(source.glob(pattern)): - self.upload_immutable(path, f"{prefix.rstrip('/')}/{path.name}") - - def upload(self, source: Path, key: str) -> None: - self.client.upload_file(str(source), self.bucket, key) - - @contextmanager - def lock(self, name: str, lifetime: int = 3600): - key = self.service_key("_locks", name) - body = json.dumps( - {"expires": int(time.time()) + lifetime, "id": uuid.uuid4().hex} - ) - try: - result = self.client.put_object( - Bucket=self.bucket, Key=key, Body=body.encode(), IfNoneMatch="*" - ) - except ClientError as error: - if not self.conflict(error): - raise - current = self.client.get_object(Bucket=self.bucket, Key=key) - state = json.loads(current["Body"].read()) - if state["expires"] > time.time(): - raise TaskError(f"{self.task}: publication already in progress: {name}") - result = self.client.put_object( - Bucket=self.bucket, - Key=key, - Body=body.encode(), - IfMatch=current["ETag"], - ) - try: - yield - finally: - self.client.delete_object( - Bucket=self.bucket, Key=key, IfMatch=result["ETag"] - ) - - def replace_prefix(self, source: Path, prefix: str) -> None: - prefix = prefix.rstrip("/") + "/" - wanted: set[str] = set() - for path in sorted(item for item in source.rglob("*") if item.is_file()): - key = prefix + path.relative_to(source).as_posix() - wanted.add(key) - self.client.upload_file(str(path), self.bucket, key) - stale = sorted(set(self.objects(prefix)) - wanted) - for offset in range(0, len(stale), 1000): - self.client.delete_objects( - Bucket=self.bucket, - Delete={ - "Objects": [{"Key": key} for key in stale[offset : offset + 1000]], - "Quiet": True, - }, - ) diff --git a/tasks/chart b/tasks/chart deleted file mode 100755 index f334209..0000000 --- a/tasks/chart +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Lint, package, and optionally publish a Helm chart" -#MISE tools={"helm"="4.1.1"} - -usage() { - cat <<'EOF' -Usage: mise run chart -- --chart DIR [options] - -Options: - --version VERSION Immutable chart version (required for packaging) - --app-version VERSION Application version written to Chart.yaml - --output DIR Package directory (default: dist/charts) - --push OCI_URL Push package to an OCI registry; repeatable - --lint-only Strictly lint without packaging -EOF -} - -chart= -version= -app_version= -output=dist/charts -push_urls= -lint_only=false - -while [ "$#" -gt 0 ]; do - case "$1" in - --chart) - [ "$#" -ge 2 ] || { echo "chart: --chart requires a value" >&2; exit 2; } - chart=$2 - shift 2 - ;; - --version) - [ "$#" -ge 2 ] || { echo "chart: --version requires a value" >&2; exit 2; } - version=$2 - shift 2 - ;; - --app-version) - [ "$#" -ge 2 ] || { echo "chart: --app-version requires a value" >&2; exit 2; } - app_version=$2 - shift 2 - ;; - --output) - [ "$#" -ge 2 ] || { echo "chart: --output requires a value" >&2; exit 2; } - output=$2 - shift 2 - ;; - --push) - [ "$#" -ge 2 ] || { echo "chart: --push requires a value" >&2; exit 2; } - case $2 in - oci://*) ;; - *) echo "chart: registry must use oci://: $2" >&2; exit 2 ;; - esac - push_urls="${push_urls}${push_urls:+ -}$2" - shift 2 - ;; - --lint-only) - lint_only=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "chart: unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -command -v helm >/dev/null 2>&1 || { echo "chart: helm is not installed" >&2; exit 1; } -[ -n "$chart" ] || { echo "chart: --chart is required" >&2; exit 2; } -[ -d "$chart" ] || { echo "chart: directory not found: $chart" >&2; exit 1; } -[ -f "$chart/Chart.yaml" ] || { echo "chart: Chart.yaml not found in $chart" >&2; exit 1; } - -name=$(sed -n 's/^name:[[:space:]]*\([A-Za-z0-9_.-][A-Za-z0-9_.-]*\)[[:space:]]*$/\1/p' "$chart/Chart.yaml" | head -n 1) -[ -n "$name" ] || { echo "chart: cannot read chart name from $chart/Chart.yaml" >&2; exit 1; } - -helm lint "$chart" --strict - -if [ "$lint_only" = true ]; then - if [ -n "$version" ] || [ -n "$app_version" ] || [ -n "$push_urls" ]; then - echo "chart: --lint-only cannot package or push a chart" >&2 - exit 2 - fi - exit 0 -fi - -[ -n "$version" ] || { echo "chart: --version is required unless --lint-only is used" >&2; exit 2; } -mkdir -p "$output" - -set -- helm package "$chart" --destination "$output" --version "$version" -if [ -n "$app_version" ]; then - set -- "$@" --app-version "$app_version" -fi -"$@" - -package=$output/$name-$version.tgz -[ -f "$package" ] || { echo "chart: Helm did not create expected package: $package" >&2; exit 1; } - -old_ifs=$IFS -IFS=' -' -for registry in $push_urls; do - helm push "$package" "${registry%/}" -done -IFS=$old_ifs diff --git a/tasks/chart.py b/tasks/chart.py new file mode 100755 index 0000000..f7333a8 --- /dev/null +++ b/tasks/chart.py @@ -0,0 +1,87 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Lint, package, and optionally publish a Helm chart" +#MISE tools={"pipx"="1.16.7","python"="3.14.7","helm"="4.1.1"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import re +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, require_command, run, task_main + +TASK = "chart" +CHART_NAME = re.compile(r"^name:\s*([A-Za-z0-9_.-][A-Za-z0-9_.-]*)\s*$") + + +def chart_name(chart: Path) -> str: + for line in (chart / "Chart.yaml").read_text().splitlines(): + if match := CHART_NAME.fullmatch(line): + return match.group(1) + raise TaskError(f"{TASK}: cannot read chart name from {chart}/Chart.yaml") + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser(prog="mise run chart --") + command.add_argument("--chart", required=True, type=Path) + command.add_argument("--version") + command.add_argument("--app-version") + command.add_argument("--output", default=Path("dist/charts"), type=Path) + command.add_argument("--push", action="append", default=[], metavar="OCI_URL") + command.add_argument("--lint-only", action="store_true") + arguments = command.parse_args(args) + + require_command("helm", TASK) + if not arguments.chart.is_dir(): + raise TaskError(f"{TASK}: directory not found: {arguments.chart}") + if not (arguments.chart / "Chart.yaml").is_file(): + raise TaskError(f"{TASK}: Chart.yaml not found in {arguments.chart}") + for registry in arguments.push: + if not registry.startswith("oci://"): + command.error(f"registry must use oci://: {registry}") + + name = chart_name(arguments.chart) + await run("helm", "lint", arguments.chart, "--strict") + + if arguments.lint_only: + if arguments.version or arguments.app_version or arguments.push: + command.error("--lint-only cannot package or push a chart") + return + if not arguments.version: + command.error("--version is required unless --lint-only is used") + + arguments.output.mkdir(parents=True, exist_ok=True) + package_arguments: list[str | Path] = [ + "helm", + "package", + arguments.chart, + "--destination", + arguments.output, + "--version", + arguments.version, + ] + if arguments.app_version: + package_arguments.extend(("--app-version", arguments.app_version)) + await run(package_arguments) + + package = arguments.output / f"{name}-{arguments.version}.tgz" + if not package.is_file(): + raise TaskError(f"{TASK}: Helm did not create expected package: {package}") + for registry in arguments.push: + await run("helm", "push", package, registry.rstrip("/")) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/container b/tasks/container deleted file mode 100755 index b41a313..0000000 --- a/tasks/container +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Build and optionally publish an OCI container image" - -usage() { - cat <<'EOF' -Usage: mise run container -- --tag IMAGE [options] - -Options: - --context DIR Build context (default: .) - --file FILE Dockerfile relative to the context (default: Dockerfile) - --platform PLATFORMS Target platforms (default: linux/amd64) - --target TARGET Dockerfile build target - --build-arg VALUE Build argument; repeatable - --label VALUE OCI image label; repeatable - --tag IMAGE Image tag; repeatable and required - --cache-scope SCOPE GitHub Actions cache scope - --provenance BOOL BuildKit provenance (default: true) - --sbom BOOL BuildKit SBOM (default: true) - --push Push the resulting image - --load Load a single-platform image locally -EOF -} - -context=. -file=Dockerfile -platform=linux/amd64 -target= -build_args= -labels= -tags= -cache_scope= -provenance=true -sbom=true -push=false -load=false - -while [ "$#" -gt 0 ]; do - case "$1" in - --context) - [ "$#" -ge 2 ] || { echo "container: --context requires a value" >&2; exit 2; } - context=$2 - shift 2 - ;; - --file) - [ "$#" -ge 2 ] || { echo "container: --file requires a value" >&2; exit 2; } - file=$2 - shift 2 - ;; - --platform) - [ "$#" -ge 2 ] || { echo "container: --platform requires a value" >&2; exit 2; } - platform=$2 - shift 2 - ;; - --target) - [ "$#" -ge 2 ] || { echo "container: --target requires a value" >&2; exit 2; } - target=$2 - shift 2 - ;; - --build-arg) - [ "$#" -ge 2 ] || { echo "container: --build-arg requires a value" >&2; exit 2; } - case $2 in *' -'*) echo "container: build arguments cannot contain newlines" >&2; exit 2 ;; esac - build_args="${build_args}${build_args:+ -}$2" - shift 2 - ;; - --label) - [ "$#" -ge 2 ] || { echo "container: --label requires a value" >&2; exit 2; } - case $2 in *' -'*) echo "container: labels cannot contain newlines" >&2; exit 2 ;; esac - labels="${labels}${labels:+ -}$2" - shift 2 - ;; - --tag) - [ "$#" -ge 2 ] || { echo "container: --tag requires a value" >&2; exit 2; } - case $2 in ''|*[[:space:]]*) echo "container: invalid image tag: $2" >&2; exit 2 ;; esac - tags="${tags}${tags:+ -}$2" - shift 2 - ;; - --cache-scope) - [ "$#" -ge 2 ] || { echo "container: --cache-scope requires a value" >&2; exit 2; } - cache_scope=$2 - shift 2 - ;; - --provenance) - [ "$#" -ge 2 ] || { echo "container: --provenance requires a value" >&2; exit 2; } - case $2 in true|false) provenance=$2 ;; *) echo "container: --provenance must be true or false" >&2; exit 2 ;; esac - shift 2 - ;; - --sbom) - [ "$#" -ge 2 ] || { echo "container: --sbom requires a value" >&2; exit 2; } - case $2 in true|false) sbom=$2 ;; *) echo "container: --sbom must be true or false" >&2; exit 2 ;; esac - shift 2 - ;; - --push) - push=true - shift - ;; - --load) - load=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "container: unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -command -v docker >/dev/null 2>&1 || { echo "container: docker is not installed" >&2; exit 1; } -[ -d "$context" ] || { echo "container: context directory not found: $context" >&2; exit 1; } -[ -n "$tags" ] || { echo "container: at least one --tag is required" >&2; exit 2; } - -case $file in - /*) dockerfile=$file ;; - *) dockerfile=$context/$file ;; -esac -[ -f "$dockerfile" ] || { echo "container: Dockerfile not found: $dockerfile" >&2; exit 1; } - -if [ "$push" = true ] && [ "$load" = true ]; then - echo "container: --push and --load are mutually exclusive" >&2 - exit 2 -fi -case $platform:$load in - *,*:true) echo "container: --load supports exactly one platform" >&2; exit 2 ;; -esac - -set -- docker buildx build \ - --file "$dockerfile" \ - --platform "$platform" \ - "--provenance=$provenance" \ - "--sbom=$sbom" - -if [ -n "$target" ]; then - set -- "$@" --target "$target" -fi - -old_ifs=$IFS -IFS=' -' -for value in $build_args; do - set -- "$@" --build-arg "$value" -done -for value in $labels; do - set -- "$@" --label "$value" -done -for value in $tags; do - set -- "$@" --tag "$value" -done -IFS=$old_ifs - -if [ -n "$cache_scope" ]; then - set -- "$@" \ - --cache-from "type=gha,scope=$cache_scope" \ - --cache-to "type=gha,mode=max,scope=$cache_scope" -fi -if [ "$push" = true ]; then - set -- "$@" --push -elif [ "$load" = true ]; then - set -- "$@" --load -fi -set -- "$@" "$context" - -exec "$@" diff --git a/tasks/container.py b/tasks/container.py new file mode 100755 index 0000000..c68482b --- /dev/null +++ b/tasks/container.py @@ -0,0 +1,105 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Build and optionally publish an OCI container image" +#MISE tools={"pipx"="1.16.7","python"="3.14.7"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, require_command, run, task_main + +TASK = "container" + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser(prog="mise run container --") + command.add_argument("--context", default=Path("."), type=Path) + command.add_argument("--file", default=Path("Dockerfile"), type=Path) + command.add_argument("--platform", default="linux/amd64") + command.add_argument("--target") + command.add_argument("--build-arg", action="append", default=[]) + command.add_argument("--label", action="append", default=[]) + command.add_argument("--tag", action="append", required=True) + command.add_argument("--cache-scope") + command.add_argument("--provenance", choices=("true", "false"), default="true") + command.add_argument("--sbom", choices=("true", "false"), default="true") + command.add_argument("--push", action="store_true") + command.add_argument("--load", action="store_true") + arguments = command.parse_args(args) + + require_command("docker", TASK) + if not arguments.context.is_dir(): + raise TaskError(f"{TASK}: context directory not found: {arguments.context}") + dockerfile = ( + arguments.file + if arguments.file.is_absolute() + else arguments.context / arguments.file + ) + if not dockerfile.is_file(): + raise TaskError(f"{TASK}: Dockerfile not found: {dockerfile}") + for value in arguments.build_arg: + if "\n" in value: + command.error("build arguments cannot contain newlines") + for value in arguments.label: + if "\n" in value: + command.error("labels cannot contain newlines") + for tag in arguments.tag: + if not tag or any(character.isspace() for character in tag): + command.error(f"invalid image tag: {tag}") + if arguments.push and arguments.load: + command.error("--push and --load are mutually exclusive") + if arguments.load and "," in arguments.platform: + command.error("--load supports exactly one platform") + + build: list[str | Path] = [ + "docker", + "buildx", + "build", + "--file", + dockerfile, + "--platform", + arguments.platform, + f"--provenance={arguments.provenance}", + f"--sbom={arguments.sbom}", + ] + if arguments.target: + build.extend(("--target", arguments.target)) + for value in arguments.build_arg: + build.extend(("--build-arg", value)) + for value in arguments.label: + build.extend(("--label", value)) + for tag in arguments.tag: + build.extend(("--tag", tag)) + if arguments.cache_scope: + build.extend( + ( + "--cache-from", + f"type=gha,scope={arguments.cache_scope}", + "--cache-to", + f"type=gha,mode=max,scope={arguments.cache_scope}", + ) + ) + if arguments.push: + build.append("--push") + elif arguments.load: + build.append("--load") + build.append(arguments.context) + + await run(*build) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/libs/__init__.py b/tasks/libs/__init__.py new file mode 100644 index 0000000..951b296 --- /dev/null +++ b/tasks/libs/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD diff --git a/tasks/libs/apk.py b/tasks/libs/apk.py new file mode 100644 index 0000000..ae2efc3 --- /dev/null +++ b/tasks/libs/apk.py @@ -0,0 +1,144 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import asyncio +import gzip +import hashlib +import io +import os +import platform +import shutil +import tarfile +import urllib.request +from pathlib import Path + +from shellous import sh + +from .common import TaskError, capture, run + +TASK = "publish" +APK_TOOLS_VERSION = "2.14.10-r0" +APK_TOOLS_SHA256 = { + "x86_64": "c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61", + "aarch64": "3e22f80dd0272dc487e4ca84b2c6b660ca392cbad970764efe9ef9555b806ac8", +} + + +async def architecture(package: Path) -> str: + metadata = await capture("tar", "-xOzf", package, ".PKGINFO").stderr(sh.DEVNULL) + for line in metadata.splitlines(): + if line.startswith("arch = "): + return line.removeprefix("arch = ") + raise TaskError(f"{TASK}: cannot read APK architecture: {package}") + + +async def apk_tool(work: Path) -> Path: + configured = os.environ.get("APK_TOOL") + if configured: + tool = Path(configured) + if os.access(tool, os.X_OK): + return tool + raise TaskError(f"{TASK}: APK_TOOL is not executable: {tool}") + for name in ("apk.static", "apk"): + if command := shutil.which(name): + return Path(command) + + machine = platform.machine() + apk_arch = { + "x86_64": "x86_64", + "amd64": "x86_64", + "aarch64": "aarch64", + "arm64": "aarch64", + }.get(machine) + if apk_arch is None: + raise TaskError(f"{TASK}: apk-tools is unavailable for {machine}") + archive = work / "apk-tools-static.apk" + url = ( + "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/" + f"{apk_arch}/apk-tools-static-{APK_TOOLS_VERSION}.apk" + ) + await asyncio.to_thread(urllib.request.urlretrieve, url, archive) + if hashlib.sha256(archive.read_bytes()).hexdigest() != APK_TOOLS_SHA256[apk_arch]: + raise TaskError(f"{TASK}: apk-tools checksum mismatch") + directory = work / "apk-tools" + directory.mkdir() + await run("tar", "-xzf", archive, "-C", directory, "sbin/apk.static").stderr( + sh.DEVNULL + ) + return directory / "sbin" / "apk.static" + + +async def sign_index(context, index: Path) -> None: + name = f".SIGN.RSA256.{context.apk_signing.public_key_name}" + signature = index.parent / name + await run.set(env=context.apk_signing.environment, inherit_env=False)( + "openssl", + "dgst", + "-sha256", + "-sign", + context.apk_signing.private_key_file, + "-out", + signature, + index, + ) + data = signature.read_bytes() + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w", format=tarfile.USTAR_FORMAT) as archive: + information = tarfile.TarInfo(name) + information.size = len(data) + information.mode = 0o644 + information.mtime = int(os.environ.get("SOURCE_DATE_EPOCH", "0")) + archive.addfile(information, io.BytesIO(data)) + size = 512 + ((len(data) + 511) // 512) * 512 + index.write_bytes( + gzip.compress(stream.getvalue()[:size], mtime=0) + index.read_bytes() + ) + signature.unlink() + + +async def publish(context) -> None: + tool = await apk_tool(context.work) + packages = context.packages["apk"] + package_architectures = dict( + zip( + packages, + await asyncio.gather(*(architecture(package) for package in packages)), + strict=True, + ) + ) + architectures = sorted(set(package_architectures.values())) + keys = context.work / "apk-keys" + keys.mkdir() + for public_key in context.apk_public_keys: + shutil.copy2(public_key, keys / public_key.name) + + for apk_arch in architectures: + root = context.work / "apk" / apk_arch + root.mkdir(parents=True) + remote = context.storage.service_key("apk", context.channel, apk_arch) + context.storage.download_prefix(remote, root, "*.apk") + for package in packages: + if package_architectures[package] == apk_arch: + context.add_package(package, root) + for package in root.glob("*.apk"): + result = await run.result(tool, "verify", "--keys-dir", keys, package) + if result.exit_code: + raise TaskError( + f"{TASK}: APK signature verification failed: {package.name}" + ) + index = root / "APKINDEX.tar.gz" + await run( + tool, + "--allow-untrusted", + "index", + "--description", + f"Dimidium Labs {context.service} {context.channel}", + "--output", + index, + sorted(root.glob("*.apk")), + ) + await sign_index(context, index) + context.storage.upload_payloads(root, remote, "*.apk") + context.storage.upload(index, f"{remote}/APKINDEX.tar.gz") diff --git a/tasks/libs/apt.py b/tasks/libs/apt.py new file mode 100644 index 0000000..8952900 --- /dev/null +++ b/tasks/libs/apt.py @@ -0,0 +1,79 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +from .common import TaskError, capture, require_command, run + +TASK = "publish" + + +async def publish(context) -> None: + require_command("apt-ftparchive", TASK) + require_command("dpkg-deb", TASK) + root = context.work / "apt" + pool = root / "pool" / context.channel + metadata = root / "dists" / context.channel + pool.mkdir(parents=True) + pool_prefix = context.storage.service_key("apt", "pool", context.channel) + metadata_prefix = context.storage.service_key("apt", "dists", context.channel) + context.storage.download_prefix(pool_prefix, pool, "*.deb") + for package in context.packages["deb"]: + context.add_package(package, pool) + + architectures = sorted( + { + (await capture("dpkg-deb", "-f", package, "Architecture")).strip() + for package in pool.glob("*.deb") + } + ) + if not architectures or "" in architectures: + raise TaskError(f"{TASK}: no DEB architectures found") + for architecture in architectures: + (metadata / "main" / f"binary-{architecture}").mkdir( + parents=True, exist_ok=True + ) + + cache = context.work / "apt-cache" + cache.mkdir() + config = context.work / "apt-ftparchive.conf" + architecture_list = " ".join(architectures) + config.write_text( + f'''Dir {{ ArchiveDir "{root}"; CacheDir "{cache}"; }}; +Default {{ Packages::Compress ". gzip"; Packages::Extensions ".deb"; }}; +TreeDefault {{ + Packages "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"; + BinCacheDB "packages-$(ARCH).db"; +}}; +Tree "dists/{context.channel}" {{ + Sections "main"; + Architectures "{architecture_list}"; + Directory "pool/{context.channel}"; +}}; +''' + ) + await run("apt-ftparchive", "generate", config) + release = metadata / "Release" + await run( + "apt-ftparchive", + "-o", + "APT::FTPArchive::Release::Origin=Dimidium Labs", + "-o", + f"APT::FTPArchive::Release::Label={context.service} {context.channel}", + "-o", + f"APT::FTPArchive::Release::Suite={context.channel}", + "-o", + f"APT::FTPArchive::Release::Codename={context.channel}", + "-o", + "APT::FTPArchive::Release::Components=main", + "-o", + f"APT::FTPArchive::Release::Architectures={architecture_list}", + "release", + f"{metadata}/", + ).stdout(release) + await context.gpg.sign( + metadata / "Release.gpg", "--armor", "--detach-sign", release + ) + await context.gpg.sign(metadata / "InRelease", "--clearsign", release) + context.storage.upload_payloads(pool, pool_prefix, "*.deb") + context.storage.replace_prefix(metadata, metadata_prefix) diff --git a/tasks/libs/common.py b/tasks/libs/common.py new file mode 100644 index 0000000..e1373e6 --- /dev/null +++ b/tasks/libs/common.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import asyncio +import os +import shutil +import sys +from collections.abc import Awaitable, Callable, Sequence +from pathlib import Path + +from shellous import ResultError, sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) + + +class TaskError(RuntimeError): + pass + + +def require_command(name: str, task: str) -> str: + command = shutil.which(name) + if command is None: + raise TaskError(f"{task}: {name} is required") + return command + + +def required_env(name: str, task: str, purpose: str = "") -> str: + value = os.environ.get(name) + if value: + return value + suffix = f" {purpose}" if purpose else "" + raise TaskError(f"{task}: {name} is required{suffix}") + + +def task_main( + task: str, + main: Callable[[Sequence[str]], Awaitable[None]], + args: Sequence[str], +) -> None: + try: + asyncio.run(main(args)) + except TaskError as error: + print(error, file=sys.stderr) + raise SystemExit(1) from None + except ResultError as error: + exit_code = error.result.exit_code + print(f"{task}: command failed with exit code {exit_code}", file=sys.stderr) + raise SystemExit(exit_code) from None + + +class GPGSigning: + def __init__(self, task: str, work: Path): + self.task = task + require_command("gpg", task) + private_key = required_env("GPG_PRIVATE_KEY", task) + self.passphrase = required_env("GPG_PASSPHRASE", task) + self.key_id = required_env("GPG_KEY_ID", task) + self.short_key_id = self.key_id[-16:] + self.home = work / "gnupg" + self.home.mkdir(mode=0o700) + self.private_key_file = work / "signing.asc" + self.private_key_file.write_text(private_key) + self.private_key_file.chmod(0o600) + self.environment = dict(os.environ) + self.environment["GNUPGHOME"] = str(self.home) + self.environment.pop("GPG_PRIVATE_KEY", None) + self.environment.pop("GPG_PASSPHRASE", None) + self.environment.pop("APK_PRIVATE_KEY", None) + + @classmethod + async def create(cls, task: str, work: Path) -> GPGSigning: + signing = cls(task, work) + command = run.set(env=signing.environment, inherit_env=False) + await ( + f"{signing.passphrase}\n" + | command( + "gpg", + "--batch", + "--yes", + "--pinentry-mode", + "loopback", + "--passphrase-fd", + "0", + "--import", + signing.private_key_file, + ) + ) + return signing + + def package_environment(self) -> dict[str, str]: + environment = dict(self.environment) + environment["GPG_KEY_ID"] = self.short_key_id + return environment + + async def prime_agent(self) -> None: + signature = self.private_key_file.with_suffix(".sig") + await self.sign(signature, "--detach-sign", self.private_key_file) + signature.unlink() + + async def export_public_key(self, output: Path) -> None: + command = run.set(env=self.environment, inherit_env=False) + await command( + "gpg", + "--batch", + "--yes", + "--armor", + "--export", + self.key_id, + ).stdout(output) + + async def verify_public_bundle(self, bundle: Path) -> None: + command = capture.set(env=self.environment, inherit_env=False) + output = await command( + "gpg", + "--batch", + "--with-colons", + "--show-keys", + bundle, + ) + fingerprints = { + line.split(":")[9] + for line in output.splitlines() + if line.startswith("fpr:") + } + if self.key_id not in fingerprints: + raise TaskError( + f"{self.task}: packages.gpg does not contain signing key {self.key_id}" + ) + + async def sign(self, output: Path, *arguments: str | Path) -> None: + command = run.set(env=self.environment, inherit_env=False) + await ( + f"{self.passphrase}\n" + | command( + "gpg", + f"--default-key={self.key_id}", + "--batch", + "--yes", + "--pinentry-mode", + "loopback", + "--passphrase-fd", + "0", + "-o", + output, + arguments, + ) + ) + + +class APKSigning: + def __init__(self, task: str, work: Path, key_name: str = "packages"): + require_command("openssl", task) + private_key = required_env("APK_PRIVATE_KEY", task, "for APK signing") + self.key_name = key_name + self.public_key_name = f"{key_name}.rsa.pub" + self.private_key_file = work / f"{key_name}.rsa" + self.private_key_file.write_text(private_key) + self.private_key_file.chmod(0o600) + self.environment = dict(os.environ) + self.environment.pop("APK_PRIVATE_KEY", None) + self.environment.pop("GPG_PRIVATE_KEY", None) + self.environment.pop("GPG_PASSPHRASE", None) + self.environment["APK_SIGNING_KEY"] = str(self.private_key_file) + + async def export_public_key(self, output: Path) -> None: + command = run.set(env=self.environment, inherit_env=False) + await command( + "openssl", + "rsa", + "-in", + self.private_key_file, + "-pubout", + "-out", + output, + ) diff --git a/tasks/libs/repository.py b/tasks/libs/repository.py new file mode 100644 index 0000000..772e1ab --- /dev/null +++ b/tasks/libs/repository.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import re +import shutil +from pathlib import Path + +from .common import APKSigning, GPGSigning, TaskError, required_env +from .storage import S3Storage + +TASK = "publish" +KEY_VERSION = re.compile(r"^[0-9]{4}$") +RSA_PUBLIC_KEY = re.compile(r"^keys/packages\.[0-9]{4}\.rsa\.pub$") + + +class Repository: + def __init__(self, service, channel, input_directory, formats, work): + self.service = service + self.channel = channel + self.formats = formats + self.work = work + self.public_url = required_env("S3_PUBLIC_URL", TASK).rstrip("/") + self.key_version = required_env("PACKAGE_KEY_VERSION", TASK) + if not KEY_VERSION.fullmatch(self.key_version): + raise TaskError(f"{TASK}: invalid PACKAGE_KEY_VERSION: {self.key_version}") + self.storage = S3Storage(TASK, service) + self.gpg = None + self.gpg_public_key = None + self.apk_signing = None + self.apk_public_keys = [] + self.packages = { + package_format: sorted(input_directory.glob(f"*.{package_format}")) + for package_format in formats + } + for package_format, packages in self.packages.items(): + if not packages: + raise TaskError( + f"{TASK}: no .{package_format} packages found in {input_directory}" + ) + + def check_public_key(self, source: Path, key: str) -> None: + existing = self.work / f"existing-{source.name}" + if not self.storage.download(key, existing): + raise TaskError(f"{TASK}: organization key {key} is not provisioned") + if source.read_bytes() != existing.read_bytes(): + raise TaskError(f"{TASK}: signing key does not match {key}") + + @staticmethod + def add_package(source: Path, directory: Path) -> None: + destination = directory / source.name + if destination.exists() and source.read_bytes() != destination.read_bytes(): + raise TaskError( + f"{TASK}: immutable package filename has different content: {source.name}" + ) + if not destination.exists(): + shutil.copy2(source, destination) + + async def setup_openpgp(self) -> None: + self.gpg = await GPGSigning.create(TASK, self.work) + current = self.work / "current-packages.gpg" + await self.gpg.export_public_key(current) + self.check_public_key(current, f"keys/packages.{self.key_version}.gpg") + + bundle = self.work / "packages.gpg" + if not self.storage.download("packages.gpg", bundle): + raise TaskError(f"{TASK}: organization key packages.gpg is not provisioned") + await self.gpg.verify_public_bundle(bundle) + self.gpg_public_key = bundle + + async def setup_rsa(self) -> None: + key_name = f"packages.{self.key_version}" + self.apk_signing = APKSigning(TASK, self.work, key_name) + current = self.work / self.apk_signing.public_key_name + await self.apk_signing.export_public_key(current) + self.check_public_key( + current, + f"keys/{self.apk_signing.public_key_name}", + ) + + key_directory = self.work / "rsa-public-keys" + key_directory.mkdir() + for key in sorted(self.storage.objects("keys/")): + if RSA_PUBLIC_KEY.fullmatch(key): + destination = key_directory / Path(key).name + if destination.exists(): + raise TaskError(f"{TASK}: duplicate RSA public key name: {key}") + self.storage.download(key, destination) + self.apk_public_keys.append(destination) + if current.name not in {key.name for key in self.apk_public_keys}: + raise TaskError( + f"{TASK}: current RSA public key is absent from key archive" + ) + + async def setup_signing(self) -> None: + if {"deb", "rpm"} & set(self.formats): + await self.setup_openpgp() + if "apk" in self.formats: + await self.setup_rsa() + + async def publish(self) -> None: + from .apk import publish as publish_apk + from .apt import publish as publish_apt + from .rpm import publish as publish_rpm + + publishers = {"deb": publish_apt, "rpm": publish_rpm, "apk": publish_apk} + with self.storage.lock(self.channel): + await self.setup_signing() + for package_format in self.formats: + await publishers[package_format](self) diff --git a/tasks/libs/rpm.py b/tasks/libs/rpm.py new file mode 100644 index 0000000..390446e --- /dev/null +++ b/tasks/libs/rpm.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import shutil + +from .common import TaskError, capture, require_command, run + +TASK = "publish" + + +async def publish(context) -> None: + require_command("createrepo_c", TASK) + require_command("rpmkeys", TASK) + root = context.work / "rpm" + root.mkdir() + remote = context.storage.service_key("rpm", context.channel) + context.storage.download_prefix(remote, root, "*.rpm") + for package in context.packages["rpm"]: + context.add_package(package, root) + + rpm_database = context.work / "rpmdb" + rpm_database.mkdir() + await run("rpmkeys", "--dbpath", rpm_database, "--import", context.gpg_public_key) + for package in root.glob("*.rpm"): + result = await capture( + "rpmkeys", "--dbpath", rpm_database, "--checksig", package + ) + if "signatures OK" not in result: + raise TaskError( + f"{TASK}: RPM is not signed by a trusted key: {package.name}" + ) + + shutil.rmtree(root / "repodata", ignore_errors=True) + await run("createrepo_c", root) + repomd = root / "repodata" / "repomd.xml" + await context.gpg.sign( + repomd.with_suffix(".xml.asc"), "--armor", "--detach-sign", repomd + ) + definition = root / f"{context.service}-{context.channel}.repo" + definition.write_text( + f"""[{context.service}-{context.channel}] +name={context.service} {context.channel} +gpgkey={context.public_url}/packages.gpg +baseurl={context.public_url}/{context.service}/rpm/{context.channel}/ +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +""" + ) + context.storage.upload_payloads(root, remote, "*.rpm") + context.storage.upload(definition, f"{remote}/{definition.name}") + context.storage.replace_prefix(root / "repodata", f"{remote}/repodata") diff --git a/tasks/libs/storage.py b/tasks/libs/storage.py new file mode 100644 index 0000000..7bd89d2 --- /dev/null +++ b/tasks/libs/storage.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import fnmatch +import hashlib +import json +import time +import uuid +from contextlib import contextmanager +from pathlib import Path + +import boto3 +from botocore.exceptions import ClientError + +from .common import TaskError, required_env + + +class S3Storage: + def __init__(self, task: str, service: str): + self.task = task + self.bucket = required_env("S3_BUCKET", task) + self.service_root = service + self.client = boto3.client( + "s3", + endpoint_url=required_env("S3_ENDPOINT", task), + aws_access_key_id=required_env("S3_ACCESS_KEY_ID", task), + aws_secret_access_key=required_env("S3_SECRET_ACCESS_KEY", task), + region_name="auto", + ) + + def service_key(self, *parts: str) -> str: + return "/".join((self.service_root, *parts)) + + def download(self, key: str, destination: Path) -> bool: + destination.parent.mkdir(parents=True, exist_ok=True) + try: + self.client.download_file(self.bucket, key, str(destination)) + except ClientError as error: + if error.response.get("Error", {}).get("Code") in { + "404", + "NoSuchKey", + "NotFound", + }: + return False + raise + return True + + def objects(self, prefix: str) -> set[str]: + pages = self.client.get_paginator("list_objects_v2").paginate( + Bucket=self.bucket, Prefix=prefix + ) + return {item["Key"] for page in pages for item in page.get("Contents", [])} + + def download_prefix(self, prefix: str, destination: Path, pattern: str) -> None: + destination.mkdir(parents=True, exist_ok=True) + prefix = prefix.rstrip("/") + "/" + for key in self.objects(prefix): + relative = key.removeprefix(prefix).lstrip("/") + if relative and "/" not in relative and fnmatch.fnmatch(relative, pattern): + self.client.download_file(self.bucket, key, str(destination / relative)) + + @staticmethod + def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + @staticmethod + def conflict(error: ClientError) -> bool: + return error.response.get("Error", {}).get("Code") in { + "409", + "412", + "ConditionalRequestConflict", + "PreconditionFailed", + } + + def upload_immutable(self, source: Path, key: str) -> None: + digest = self.digest(source) + try: + with source.open("rb") as stream: + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=stream, + Metadata={"sha256": digest}, + IfNoneMatch="*", + ) + return + except ClientError as error: + if not self.conflict(error): + raise + + existing = self.client.head_object(Bucket=self.bucket, Key=key) + existing_digest = existing.get("Metadata", {}).get("sha256") + if not existing_digest: + body = self.client.get_object(Bucket=self.bucket, Key=key)["Body"] + existing_digest = hashlib.sha256(body.read()).hexdigest() + if existing_digest != digest: + raise TaskError( + f"{self.task}: immutable object has different content: {key}" + ) + + def upload_payloads(self, source: Path, prefix: str, pattern: str) -> None: + for path in sorted(source.glob(pattern)): + self.upload_immutable(path, f"{prefix.rstrip('/')}/{path.name}") + + def upload(self, source: Path, key: str) -> None: + self.client.upload_file(str(source), self.bucket, key) + + @contextmanager + def lock(self, name: str, lifetime: int = 3600): + key = self.service_key("_locks", name) + body = json.dumps( + {"expires": int(time.time()) + lifetime, "id": uuid.uuid4().hex} + ) + try: + result = self.client.put_object( + Bucket=self.bucket, Key=key, Body=body.encode(), IfNoneMatch="*" + ) + except ClientError as error: + if not self.conflict(error): + raise + current = self.client.get_object(Bucket=self.bucket, Key=key) + state = json.loads(current["Body"].read()) + if state["expires"] > time.time(): + raise TaskError(f"{self.task}: publication already in progress: {name}") + result = self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=body.encode(), + IfMatch=current["ETag"], + ) + try: + yield + finally: + self.client.delete_object( + Bucket=self.bucket, Key=key, IfMatch=result["ETag"] + ) + + def replace_prefix(self, source: Path, prefix: str) -> None: + prefix = prefix.rstrip("/") + "/" + wanted: set[str] = set() + for path in sorted(item for item in source.rglob("*") if item.is_file()): + key = prefix + path.relative_to(source).as_posix() + wanted.add(key) + self.client.upload_file(str(path), self.bucket, key) + stale = sorted(set(self.objects(prefix)) - wanted) + for offset in range(0, len(stale), 1000): + self.client.delete_objects( + Bucket=self.bucket, + Delete={ + "Objects": [{"Key": key} for key in stale[offset : offset + 1000]], + "Quiet": True, + }, + ) diff --git a/tasks/licenses b/tasks/licenses deleted file mode 100755 index 5956052..0000000 --- a/tasks/licenses +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Verify repository licensing metadata" -#MISE tools={"pipx:reuse"="6.2.0","aqua:EmbarkStudios/cargo-deny"="0.19.0"} - -check_copyright_headers() { - invalid_headers=$( - git grep -n -I -E \ - '^(())?$/) { - reason = "expected 2026 Nikolay Govorov" - } - - if (reason != "") { - print reason ": " $0 - } - }' - ) - - if [ -n "$invalid_headers" ]; then - printf '%s\n%s\n' 'Invalid copyright headers:' "$invalid_headers" >&2 - return 1 - fi -} - -check_copyright_headers -reuse lint - -if [ -f Cargo.toml ]; then - cargo-deny check -fi diff --git a/tasks/licenses.py b/tasks/licenses.py new file mode 100755 index 0000000..18504a2 --- /dev/null +++ b/tasks/licenses.py @@ -0,0 +1,95 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Verify repository licensing metadata" +#MISE tools={"pipx"="1.16.7","python"="3.14.7","pipx:reuse"="6.2.0","aqua:EmbarkStudios/cargo-deny"="0.19.0"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import re +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, capture, run, task_main + +TASK = "licenses" +HEADER = ( + r"^(())?$" +) + + +async def check_copyright_headers() -> None: + result = await capture.result( + "git", + "grep", + "-n", + "-I", + "-E", + HEADER, + "--", + ".", + ":(exclude)*.md", + ":(exclude)LICENSE", + ":(exclude)LICENSES/**", + ":(exclude)COPYING*", + ) + if result.exit_code not in {0, 1}: + raise TaskError(f"{TASK}: git grep failed with exit code {result.exit_code}") + + invalid: list[str] = [] + for line in result.output.splitlines(): + match = re.search(r":([0-9]+):", line) + if match is None or int(match.group(1)) > 10: + continue + text = line[match.end() :] + reason = "" + if LEGACY_COPYRIGHT.match(text): + reason = "legacy copyright header" + elif (position := text.find("SPDX-FileCopyrightText:")) >= 0: + suffix = text[position + len("SPDX-FileCopyrightText:") :] + if ( + not suffix + or not suffix.startswith(" ") + or (len(suffix) > 1 and suffix[1].isspace()) + ): + reason = "expected exactly one space after colon" + if ( + not reason + and "SPDX-FileCopyrightText:" in text + and "Nikolay Govorov" in text + and not CANONICAL_COPYRIGHT.search(text) + ): + reason = "expected 2026 Nikolay Govorov" + if reason: + invalid.append(f"{reason}: {line}") + + if invalid: + print("Invalid copyright headers:", *invalid, sep="\n", file=sys.stderr) + raise TaskError(f"{TASK}: invalid copyright headers") + + +async def main(args: Sequence[str]) -> None: + argparse.ArgumentParser(prog="mise run licenses --").parse_args(args) + await check_copyright_headers() + await run("reuse", "lint") + if Path("Cargo.toml").is_file(): + await run("cargo-deny", "check") + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/package b/tasks/package deleted file mode 100755 index 6db5f86..0000000 --- a/tasks/package +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env -S python3 -B -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -# fmt: off -#MISE description="Build release archives and signed Linux packages" -#MISE tools={"python"="3.14.7","nfpm"="2.47.0"} -# fmt: on - -from __future__ import annotations - -import argparse -import os -import re -import shutil -import subprocess -import sys -import tempfile -from pathlib import Path - -from _lib import APKSigning, GPGSigning, TaskError, require_command, run - -TASK = "package" -SYSTEM_FORMATS = {"deb", "rpm", "apk"} -ARCHIVE_FORMATS = {"tar.gz", "zip"} -SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") -SAFE_KEY_VERSION = re.compile(r"^[0-9]{4}$") - - -def parser() -> argparse.ArgumentParser: - result = argparse.ArgumentParser( - prog="mise run package --", - usage=( - "%(prog)s --output DIR [--config FILE] " - "[--version VERSION --arch ARCH] [--apk-public-key FILE] " - "[--archive-root DIR --archive-name NAME] " - "deb|rpm|apk|tar.gz|zip..." - ), - ) - result.add_argument("--config", default="nfpm.yaml", type=Path) - result.add_argument("--version") - result.add_argument("--arch") - result.add_argument("--output", required=True, type=Path) - result.add_argument("--archive-root", type=Path) - result.add_argument("--archive-name") - result.add_argument("--apk-public-key") - result.add_argument( - "formats", nargs="+", choices=sorted(SYSTEM_FORMATS | ARCHIVE_FORMATS) - ) - return result - - -def validate(arguments: argparse.Namespace, command: argparse.ArgumentParser) -> None: - formats = set(arguments.formats) - if formats & SYSTEM_FORMATS: - if not arguments.version or not arguments.arch: - command.error("--version and --arch are required for deb, rpm, and apk") - if not arguments.config.is_file(): - raise TaskError(f"{TASK}: {arguments.config} not found") - if formats & ARCHIVE_FORMATS: - if arguments.archive_root is None or not arguments.archive_name: - command.error( - "--archive-root and --archive-name are required for tar.gz and zip" - ) - if not arguments.archive_root.is_dir(): - raise TaskError(f"{TASK}: {arguments.archive_root} not found") - if not SAFE_NAME.fullmatch(arguments.archive_name): - raise TaskError(f"{TASK}: invalid archive name: {arguments.archive_name}") - if arguments.apk_public_key and not SAFE_NAME.fullmatch(arguments.apk_public_key): - raise TaskError( - f"{TASK}: invalid APK public key name: {arguments.apk_public_key}" - ) - - -def create_archive( - archive_format: str, - output: Path, - root: Path, - name: str, -) -> None: - if archive_format == "tar.gz": - require_command("tar", TASK) - run(["tar", "-czf", output / f"{name}.tar.gz", "-C", root, "."]) - return - require_command("zip", TASK) - destination = output / f"{name}.zip" - destination.unlink(missing_ok=True) - run(["zip", "-qry", destination, "."], cwd=root) - - -def sign_package( - package_format: str, package: Path, signing: GPGSigning | None -) -> None: - if signing is None or package_format not in {"deb", "rpm"}: - return - if package_format == "deb": - require_command("debsigs", TASK) - run( - [ - "debsigs", - "--sign=origin", - f"--default-key={signing.key_id}", - package, - ], - env=signing.environment, - ) - else: - require_command("rpmsign", TASK) - run( - [ - "rpmsign", - "--define", - f"_gpg_name {signing.key_id}", - "--addsign", - package, - ], - env=signing.environment, - ) - - -def main() -> None: - command = parser() - arguments = command.parse_args() - validate(arguments, command) - - arguments.output.mkdir(parents=True, exist_ok=True) - output = arguments.output.resolve() - formats = set(arguments.formats) - - with tempfile.TemporaryDirectory(prefix="package-") as directory: - work = Path(directory) - environment = dict(os.environ) - for name in ( - "GPG_PRIVATE_KEY", - "APK_PRIVATE_KEY", - "SIGNING_PRIVATE_KEY", - "NFPM_PASSPHRASE", - "NFPM_DEB_PASSPHRASE", - "NFPM_RPM_PASSPHRASE", - ): - environment.pop(name, None) - config = arguments.config - if "apk" in formats and "${PACKAGE_KEY_VERSION}" in config.read_text(): - key_version = os.environ.get("PACKAGE_KEY_VERSION", "") - if not SAFE_KEY_VERSION.fullmatch(key_version): - raise TaskError( - f"{TASK}: invalid PACKAGE_KEY_VERSION: {key_version or ''}" - ) - config = work / "nfpm.yaml" - config.write_text( - arguments.config.read_text().replace( - "${PACKAGE_KEY_VERSION}", key_version - ) - ) - - gpg_signing: GPGSigning | None = None - if formats & {"deb", "rpm"} and os.environ.get("GPG_PRIVATE_KEY"): - gpg_signing = GPGSigning(TASK, work) - gpg_signing.prime_agent() - environment = gpg_signing.package_environment() - - apk_signing: APKSigning | None = None - if "apk" in formats and os.environ.get("APK_PRIVATE_KEY"): - apk_signing = APKSigning(TASK, work) - environment["APK_SIGNING_KEY"] = str(apk_signing.private_key_file) - elif os.environ.get("APK_SIGNING_KEY"): - environment["APK_SIGNING_KEY"] = os.environ["APK_SIGNING_KEY"] - - if "apk" in formats and arguments.apk_public_key: - public_key = output / arguments.apk_public_key - if apk_signing is not None: - apk_signing.export_public_key(public_key) - elif environment.get("APK_SIGNING_KEY"): - require_command("openssl", TASK) - run( - [ - "openssl", - "rsa", - "-in", - environment["APK_SIGNING_KEY"], - "-pubout", - "-out", - public_key, - ], - env=environment, - ) - - for package_format in arguments.formats: - if package_format in SYSTEM_FORMATS: - require_command("nfpm", TASK) - package_environment = dict(environment) - package_environment.update( - ARCH=arguments.arch, - VERSION=arguments.version, - ) - package_output = work / f"output-{package_format}" - package_output.mkdir() - run( - [ - "nfpm", - "package", - "--config", - config, - "--packager", - package_format, - "--target", - f"{package_output}/", - ], - env=package_environment, - ) - packages = list(package_output.iterdir()) - if len(packages) != 1 or not packages[0].is_file(): - raise TaskError( - f"{TASK}: nFPM produced an unexpected number of packages" - ) - sign_package(package_format, packages[0], gpg_signing) - shutil.move(packages[0], output / packages[0].name) - else: - create_archive( - package_format, - output, - arguments.archive_root, - arguments.archive_name, - ) - - -if __name__ == "__main__": - try: - main() - except TaskError as error: - print(error, file=sys.stderr) - raise SystemExit(1) from None - except subprocess.CalledProcessError as error: - print( - f"{TASK}: command failed with exit code {error.returncode}", file=sys.stderr - ) - raise SystemExit(error.returncode) from None diff --git a/tasks/package.py b/tasks/package.py new file mode 100755 index 0000000..9856b26 --- /dev/null +++ b/tasks/package.py @@ -0,0 +1,224 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Build release archives and signed Linux packages" +#MISE tools={"pipx"="1.16.7","python"="3.14.7","nfpm"="2.47.0"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import ( + APKSigning, + GPGSigning, + TaskError, + require_command, + run, + task_main, +) + +TASK = "package" +SYSTEM_FORMATS = {"deb", "rpm", "apk"} +ARCHIVE_FORMATS = {"tar.gz", "zip"} +SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +SAFE_KEY_VERSION = re.compile(r"^[0-9]{4}$") + + +def validate(arguments: argparse.Namespace, command: argparse.ArgumentParser) -> None: + formats = set(arguments.formats) + if formats & SYSTEM_FORMATS: + if not arguments.version or not arguments.arch: + command.error("--version and --arch are required for deb, rpm, and apk") + if not arguments.config.is_file(): + raise TaskError(f"{TASK}: {arguments.config} not found") + if formats & ARCHIVE_FORMATS: + if arguments.archive_root is None or not arguments.archive_name: + command.error( + "--archive-root and --archive-name are required for tar.gz and zip" + ) + if not arguments.archive_root.is_dir(): + raise TaskError(f"{TASK}: {arguments.archive_root} not found") + if not SAFE_NAME.fullmatch(arguments.archive_name): + raise TaskError(f"{TASK}: invalid archive name: {arguments.archive_name}") + if arguments.apk_public_key and not SAFE_NAME.fullmatch(arguments.apk_public_key): + raise TaskError( + f"{TASK}: invalid APK public key name: {arguments.apk_public_key}" + ) + + +async def create_archive( + archive_format: str, + output: Path, + root: Path, + name: str, +) -> None: + if archive_format == "tar.gz": + require_command("tar", TASK) + await run("tar", "-czf", output / f"{name}.tar.gz", "-C", root, ".") + return + require_command("zip", TASK) + destination = output / f"{name}.zip" + destination.unlink(missing_ok=True) + await run.set(cwd=root)("zip", "-qry", destination, ".") + + +async def sign_package( + package_format: str, package: Path, signing: GPGSigning | None +) -> None: + if signing is None or package_format not in {"deb", "rpm"}: + return + command = run.set(env=signing.environment, inherit_env=False) + if package_format == "deb": + require_command("debsigs", TASK) + await command( + "debsigs", + "--sign=origin", + f"--default-key={signing.key_id}", + package, + ) + else: + require_command("rpmsign", TASK) + await command( + "rpmsign", + "--define", + f"_gpg_name {signing.key_id}", + "--addsign", + package, + ) + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser( + prog="mise run package --", + usage=( + "%(prog)s --output DIR [--config FILE] " + "[--version VERSION --arch ARCH] [--apk-public-key FILE] " + "[--archive-root DIR --archive-name NAME] " + "deb|rpm|apk|tar.gz|zip..." + ), + ) + command.add_argument("--config", default="nfpm.yaml", type=Path) + command.add_argument("--version") + command.add_argument("--arch") + command.add_argument("--output", required=True, type=Path) + command.add_argument("--archive-root", type=Path) + command.add_argument("--archive-name") + command.add_argument("--apk-public-key") + command.add_argument( + "formats", nargs="+", choices=sorted(SYSTEM_FORMATS | ARCHIVE_FORMATS) + ) + arguments = command.parse_args(args) + validate(arguments, command) + + arguments.output.mkdir(parents=True, exist_ok=True) + output = arguments.output.resolve() + formats = set(arguments.formats) + + with tempfile.TemporaryDirectory(prefix="package-") as directory: + work = Path(directory) + environment = dict(os.environ) + for name in ( + "GPG_PRIVATE_KEY", + "APK_PRIVATE_KEY", + "SIGNING_PRIVATE_KEY", + "NFPM_PASSPHRASE", + "NFPM_DEB_PASSPHRASE", + "NFPM_RPM_PASSPHRASE", + ): + environment.pop(name, None) + config = arguments.config + if "apk" in formats and "${PACKAGE_KEY_VERSION}" in config.read_text(): + key_version = os.environ.get("PACKAGE_KEY_VERSION", "") + if not SAFE_KEY_VERSION.fullmatch(key_version): + raise TaskError( + f"{TASK}: invalid PACKAGE_KEY_VERSION: {key_version or ''}" + ) + config = work / "nfpm.yaml" + config.write_text( + arguments.config.read_text().replace( + "${PACKAGE_KEY_VERSION}", key_version + ) + ) + + gpg_signing: GPGSigning | None = None + if formats & {"deb", "rpm"} and os.environ.get("GPG_PRIVATE_KEY"): + gpg_signing = await GPGSigning.create(TASK, work) + await gpg_signing.prime_agent() + environment = gpg_signing.package_environment() + + apk_signing: APKSigning | None = None + if "apk" in formats and os.environ.get("APK_PRIVATE_KEY"): + apk_signing = APKSigning(TASK, work) + environment["APK_SIGNING_KEY"] = str(apk_signing.private_key_file) + elif os.environ.get("APK_SIGNING_KEY"): + environment["APK_SIGNING_KEY"] = os.environ["APK_SIGNING_KEY"] + + if "apk" in formats and arguments.apk_public_key: + public_key = output / arguments.apk_public_key + if apk_signing is not None: + await apk_signing.export_public_key(public_key) + elif environment.get("APK_SIGNING_KEY"): + require_command("openssl", TASK) + await run.set(env=environment, inherit_env=False)( + "openssl", + "rsa", + "-in", + environment["APK_SIGNING_KEY"], + "-pubout", + "-out", + public_key, + ) + + for package_format in arguments.formats: + if package_format in SYSTEM_FORMATS: + require_command("nfpm", TASK) + package_environment = dict(environment) + package_environment.update( + ARCH=arguments.arch, + VERSION=arguments.version, + ) + package_output = work / f"output-{package_format}" + package_output.mkdir() + await run.set(env=package_environment, inherit_env=False)( + "nfpm", + "package", + "--config", + config, + "--packager", + package_format, + "--target", + f"{package_output}/", + ) + packages = list(package_output.iterdir()) + if len(packages) != 1 or not packages[0].is_file(): + raise TaskError( + f"{TASK}: nFPM produced an unexpected number of packages" + ) + await sign_package(package_format, packages[0], gpg_signing) + shutil.move(packages[0], output / packages[0].name) + else: + await create_archive( + package_format, + output, + arguments.archive_root, + arguments.archive_name, + ) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/publish b/tasks/publish deleted file mode 100755 index 74cf2db..0000000 --- a/tasks/publish +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env -S uv run --script --python 3.14.7 -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -# fmt: off -#MISE description="Publish signed package repositories to shared S3 storage" -#MISE tools={"uv"="0.12.5"} -# fmt: on -# /// script -# requires-python = ">=3.14" -# dependencies = ["boto3==1.43.75"] -# /// - -from __future__ import annotations - -import argparse -import re -import subprocess -import sys -import tempfile -from pathlib import Path - -sys.dont_write_bytecode = True - -from _lib import TaskError -from _repository import Repository - -TASK = "publish" -FORMATS = {"deb", "rpm", "apk"} -SAFE_SLUG = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") - - -def parser() -> argparse.ArgumentParser: - result = argparse.ArgumentParser( - prog="mise run publish --", - usage=("%(prog)s --service NAME --channel CHANNEL --input DIR deb|rpm|apk..."), - ) - result.add_argument("--service", required=True) - result.add_argument("--channel", required=True) - result.add_argument("--input", required=True, type=Path) - result.add_argument("formats", nargs="+", choices=sorted(FORMATS)) - return result - - -def main() -> None: - command = parser() - arguments = command.parse_args() - for label, value in ( - ("service name", arguments.service), - ("channel", arguments.channel), - ): - if not SAFE_SLUG.fullmatch(value) or "--" in value: - raise TaskError(f"{TASK}: invalid {label}: {value}") - if not arguments.input.is_dir(): - raise TaskError(f"{TASK}: {arguments.input} not found") - with tempfile.TemporaryDirectory(prefix="publish-") as directory: - Repository( - arguments.service, - arguments.channel, - arguments.input.resolve(), - arguments.formats, - Path(directory), - ).publish() - - -if __name__ == "__main__": - try: - main() - except TaskError as error: - print(error, file=sys.stderr) - raise SystemExit(1) from None - except subprocess.CalledProcessError as error: - print( - f"{TASK}: command failed with exit code {error.returncode}", - file=sys.stderr, - ) - raise SystemExit(error.returncode) from None diff --git a/tasks/publish.py b/tasks/publish.py new file mode 100755 index 0000000..75c5299 --- /dev/null +++ b/tasks/publish.py @@ -0,0 +1,66 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Publish signed package repositories to shared S3 storage" +#MISE tools={"pipx"="1.16.7","python"="3.14.7"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["boto3==1.43.75", "shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import os +import re +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, task_main + +if test_path := os.environ.get("PUBLISH_TEST_PYTHONPATH"): + sys.path.insert(0, test_path) + +from libs.repository import Repository + +TASK = "publish" +FORMATS = {"deb", "rpm", "apk"} +SAFE_SLUG = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser( + prog="mise run publish --", + usage="%(prog)s --service NAME --channel CHANNEL --input DIR deb|rpm|apk...", + ) + command.add_argument("--service", required=True) + command.add_argument("--channel", required=True) + command.add_argument("--input", required=True, type=Path) + command.add_argument("formats", nargs="+", choices=sorted(FORMATS)) + arguments = command.parse_args(args) + for label, value in ( + ("service name", arguments.service), + ("channel", arguments.channel), + ): + if not SAFE_SLUG.fullmatch(value) or "--" in value: + raise TaskError(f"{TASK}: invalid {label}: {value}") + if not arguments.input.is_dir(): + raise TaskError(f"{TASK}: {arguments.input} not found") + with tempfile.TemporaryDirectory(prefix="publish-") as directory: + await Repository( + arguments.service, + arguments.channel, + arguments.input.resolve(), + arguments.formats, + Path(directory), + ).publish() + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/signoff b/tasks/signoff deleted file mode 100755 index d9252b8..0000000 --- a/tasks/signoff +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Verify contributor identities and CLA acceptance trailers" - -root=$(git rev-parse --show-toplevel) -cd "$root" - -task_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -approved_emails_file="$task_dir/../config/signoff-approved-emails" -unsupported_commits_file="$task_dir/../config/cla-unsupported-commits" - -if [ ! -f CLA.md ]; then - echo "CLA.md is missing" >&2 - exit 1 -fi - -head_cla_version_count=$(grep -c '^Version ' CLA.md || :) -if [ "$head_cla_version_count" -ne 1 ]; then - echo "CLA.md must declare exactly one version" >&2 - exit 1 -fi - -head_cla_version=$(sed -n 's/^Version //p' CLA.md) -if [ -z "$head_cla_version" ]; then - echo "CLA.md declares an empty version" >&2 - exit 1 -fi - -is_approved_email() { - grep -v '^[[:space:]]*#' "$approved_emails_file" | - grep -Fqx -- "$1" -} - -is_unsupported_commit() { - grep -v '^[[:space:]]*#' "$unsupported_commits_file" | - grep -Fqx -- "$1" -} - -check_identity() { - identity=$1 - role=$2 - email=$(printf '%s\n' "$identity" | sed -n 's/^.*<\([^<>]*\)>$/\1/p') - - if [ -z "$email" ]; then - echo "Commit $short_sha has an invalid $role identity: $identity" - bad=1 - return - fi - - if is_approved_email "$email"; then - return - fi - - requires_cla=1 - if ! printf '%s\n' "$signoffs" | grep -Fqx -- "$identity"; then - echo "Commit $short_sha $role $identity is missing a matching Signed-off-by" - bad=1 - fi -} - -if [ ! -f "$approved_emails_file" ]; then - echo "Approved email configuration is missing: $approved_emails_file" >&2 - exit 1 -fi - -if [ ! -f "$unsupported_commits_file" ]; then - echo "Unsupported commit configuration is missing: $unsupported_commits_file" >&2 - exit 1 -fi - -bad=0 -for sha in $(git log --no-merges --format=%H); do - short_sha=$(printf '%.8s' "$sha") - signoffs=$(git show -s --format='%(trailers:key=Signed-off-by,valueonly)' "$sha") - requires_cla=0 - - author=$(git show -s --format='%an <%ae>' "$sha") - check_identity "$author" "author" - - coauthors=$(git show -s --format='%(trailers:key=Co-authored-by,valueonly)' "$sha") - if [ -n "$coauthors" ]; then - old_ifs=$IFS - IFS=' -' - for coauthor in $coauthors; do - check_identity "$coauthor" "co-author" - done - IFS=$old_ifs - fi - - if [ "$requires_cla" -eq 1 ] && ! is_unsupported_commit "$sha"; then - expected_cla_version_count=$( - git show "$sha:CLA.md" 2>/dev/null | grep -c '^Version ' || : - ) - if [ "$expected_cla_version_count" -ne 1 ]; then - echo "Commit $short_sha does not contain a CLA.md with exactly one version" - bad=1 - continue - fi - - expected_cla_version=$( - git show "$sha:CLA.md" | sed -n 's/^Version //p' - ) - if [ -z "$expected_cla_version" ]; then - echo "Commit $short_sha contains an empty CLA version" - bad=1 - continue - fi - - commit_cla_version=$( - git show -s --format='%(trailers:key=CLA-Version,valueonly)' "$sha" - ) - if [ "$commit_cla_version" != "$expected_cla_version" ]; then - if [ -z "$commit_cla_version" ]; then - echo "Commit $short_sha is missing CLA-Version: $expected_cla_version" - else - echo "Commit $short_sha has invalid CLA-Version: $commit_cla_version" - echo "Expected CLA-Version: $expected_cla_version" - fi - bad=1 - fi - fi -done - -if [ "$bad" -ne 0 ]; then - echo "Every non-approved author and co-author must accept the CLA in their commit" - echo "Required trailers:" - echo " CLA-Version: " - echo " Signed-off-by: Name " - echo "See CLA.md" - exit 1 -fi - -if [ ! -f .mailmap ]; then - echo "Contributor registry .mailmap is missing" >&2 - exit 1 -fi - -missing=0 -for email in $(git log --no-merges --format='%ae%n%ce' | sort -u); do - if is_approved_email "$email"; then - continue - fi - - if ! grep -v '^[[:space:]]*#' .mailmap | grep -qF -- "<${email}>"; then - echo "Email <${email}> is not in .mailmap" - missing=1 - fi -done - -if [ "$missing" -ne 0 ]; then - echo "All authors and committers must be listed in .mailmap" - exit 1 -fi diff --git a/tasks/signoff.py b/tasks/signoff.py new file mode 100755 index 0000000..a3d7e6a --- /dev/null +++ b/tasks/signoff.py @@ -0,0 +1,208 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Verify contributor identities and CLA acceptance trailers" +#MISE tools={"pipx"="1.16.7","python"="3.14.7"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import os +import re +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, capture, task_main +from shellous import sh + +TASK = "signoff" + + +def configured_values(path: Path) -> set[str]: + return { + line for line in path.read_text().splitlines() if not re.match(r"^\s*#", line) + } + + +def cla_version(document: str) -> tuple[int, str]: + versions = [ + line.removeprefix("Version ") + for line in document.splitlines() + if line.startswith("Version ") + ] + return len(versions), versions[0] if len(versions) == 1 else "" + + +def check_identity( + identity: str, + role: str, + short_sha: str, + approved_emails: set[str], + signoffs: set[str], +) -> tuple[bool, bool]: + match = re.fullmatch(r".*<([^<>]*)>", identity) + if match is None or not match.group(1): + print(f"Commit {short_sha} has an invalid {role} identity: {identity}") + return True, False + if match.group(1) in approved_emails: + return False, False + if identity not in signoffs: + print( + f"Commit {short_sha} {role} {identity} is missing a matching Signed-off-by" + ) + return True, True + return False, True + + +async def git(*arguments: str) -> str: + return await capture("git", arguments) + + +async def main(args: Sequence[str]) -> None: + argparse.ArgumentParser(prog="mise run signoff --").parse_args(args) + root = Path((await git("rev-parse", "--show-toplevel")).strip()) + os.chdir(root) + + task_directory = Path(__file__).resolve().parent + approved_emails_file = task_directory.parent / "config/signoff-approved-emails" + unsupported_commits_file = task_directory.parent / "config/cla-unsupported-commits" + + cla_file = Path("CLA.md") + if not cla_file.is_file(): + raise TaskError("CLA.md is missing") + version_count, head_version = cla_version(cla_file.read_text()) + if version_count != 1: + raise TaskError("CLA.md must declare exactly one version") + if not head_version: + raise TaskError("CLA.md declares an empty version") + if not approved_emails_file.is_file(): + raise TaskError( + f"Approved email configuration is missing: {approved_emails_file}" + ) + if not unsupported_commits_file.is_file(): + raise TaskError( + f"Unsupported commit configuration is missing: {unsupported_commits_file}" + ) + + approved_emails = configured_values(approved_emails_file) + unsupported_commits = configured_values(unsupported_commits_file) + bad = False + + commits = (await git("log", "--no-merges", "--format=%H")).splitlines() + for sha in commits: + short_sha = sha[:8] + signoffs = set( + ( + await git( + "show", + "-s", + "--format=%(trailers:key=Signed-off-by,valueonly)", + sha, + ) + ).splitlines() + ) + requires_cla = False + + author = (await git("show", "-s", "--format=%an <%ae>", sha)).rstrip("\n") + invalid, required = check_identity( + author, "author", short_sha, approved_emails, signoffs + ) + bad |= invalid + requires_cla |= required + + coauthor_output = ( + await git( + "show", + "-s", + "--format=%(trailers:key=Co-authored-by,valueonly)", + sha, + ) + ).rstrip("\n") + for coauthor in coauthor_output.splitlines() if coauthor_output else (): + invalid, required = check_identity( + coauthor, "co-author", short_sha, approved_emails, signoffs + ) + bad |= invalid + requires_cla |= required + + if requires_cla and sha not in unsupported_commits: + result = await capture.result("git", "show", f"{sha}:CLA.md").stderr( + sh.DEVNULL + ) + document = result.output if result.exit_code == 0 else "" + expected_count, expected_version = cla_version(document) + if expected_count != 1: + print( + f"Commit {short_sha} does not contain a CLA.md with exactly one version" + ) + bad = True + continue + if not expected_version: + print(f"Commit {short_sha} contains an empty CLA version") + bad = True + continue + + commit_version = ( + await git( + "show", + "-s", + "--format=%(trailers:key=CLA-Version,valueonly)", + sha, + ) + ).rstrip("\n") + if commit_version != expected_version: + if not commit_version: + print( + f"Commit {short_sha} is missing CLA-Version: {expected_version}" + ) + else: + print( + f"Commit {short_sha} has invalid CLA-Version: {commit_version}" + ) + print(f"Expected CLA-Version: {expected_version}") + bad = True + + if bad: + print( + "Every non-approved author and co-author must accept the CLA in their commit" + ) + print("Required trailers:") + print(" CLA-Version: ") + print(" Signed-off-by: Name ") + print("See CLA.md") + raise SystemExit(1) + + mailmap = Path(".mailmap") + if not mailmap.is_file(): + raise TaskError("Contributor registry .mailmap is missing") + mailmap_lines = [ + line + for line in mailmap.read_text().splitlines() + if not re.match(r"^\s*#", line) + ] + emails = sorted( + set((await git("log", "--no-merges", "--format=%ae%n%ce")).splitlines()) + ) + missing = False + for email in emails: + if email in approved_emails: + continue + if not any(f"<{email}>" in line for line in mailmap_lines): + print(f"Email <{email}> is not in .mailmap") + missing = True + if missing: + print("All authors and committers must be listed in .mailmap") + raise SystemExit(1) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tests/oci-tasks b/tests/oci-tasks deleted file mode 100755 index 1bc54f2..0000000 --- a/tests/oci-tasks +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT -mkdir -p "$work/bin" "$work/project/chart/templates" -: > "$work/project/Dockerfile" -cat > "$work/project/chart/Chart.yaml" <<'EOF' -apiVersion: v2 -name: fixture -version: 0.0.0 -EOF -cat > "$work/project/chart/values.yaml" <<'EOF' -image: fixture -EOF -cat > "$work/project/chart/templates/configmap.yaml" <<'EOF' -apiVersion: v1 -kind: ConfigMap -metadata: - name: fixture -EOF - -cat > "$work/bin/docker" <<'EOF' -#!/bin/sh -printf '%s\n' "$@" > "$OCI_TEST_DOCKER_LOG" -EOF -cat > "$work/bin/helm" <<'EOF' -#!/bin/sh -printf '%s\n' "$@" >> "$OCI_TEST_HELM_LOG" -if [ "$1" = package ]; then - destination= - version= - shift - while [ "$#" -gt 0 ]; do - case "$1" in - --destination) destination=$2; shift 2 ;; - --version) version=$2; shift 2 ;; - *) shift ;; - esac - done - : > "$destination/fixture-$version.tgz" -fi -EOF -chmod +x "$work/bin/docker" "$work/bin/helm" -export PATH="$work/bin:$PATH" -export OCI_TEST_DOCKER_LOG="$work/docker.log" -export OCI_TEST_HELM_LOG="$work/helm.log" - -"$root/tasks/container" \ - --context "$work/project" \ - --platform linux/amd64 \ - --target site \ - --build-arg APP=site \ - --build-arg 'TITLE=hello world' \ - --label 'org.example.title=Example site' \ - --tag ghcr.io/example/site:sha-abc \ - --tag ghcr.io/example/site:latest \ - --cache-scope site \ - --provenance false \ - --sbom false \ - --push - -grep -qx -- 'buildx' "$work/docker.log" -grep -qx -- '--target' "$work/docker.log" -grep -qx -- 'site' "$work/docker.log" -grep -qx -- 'TITLE=hello world' "$work/docker.log" -grep -qx -- 'org.example.title=Example site' "$work/docker.log" -grep -qx -- 'ghcr.io/example/site:sha-abc' "$work/docker.log" -grep -qx -- 'ghcr.io/example/site:latest' "$work/docker.log" -grep -qx -- 'type=gha,mode=max,scope=site' "$work/docker.log" -grep -qx -- '--push' "$work/docker.log" - -"$root/tasks/chart" \ - --chart "$work/project/chart" \ - --version 1.2.3 \ - --app-version sha-abc \ - --output "$work/output" \ - --push oci://ghcr.io/example/charts - -grep -qx -- 'lint' "$work/helm.log" -grep -qx -- 'package' "$work/helm.log" -grep -qx -- 'push' "$work/helm.log" -grep -qx -- 'oci://ghcr.io/example/charts' "$work/helm.log" - -echo 'oci tasks: ok' diff --git a/tests/oci-tasks.py b/tests/oci-tasks.py new file mode 100755 index 0000000..b236394 --- /dev/null +++ b/tests/oci-tasks.py @@ -0,0 +1,147 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from shellous import sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) + + +def executable(path: Path, source: str) -> None: + path.write_text("#!/usr/bin/env python3\n" + source) + path.chmod(0o755) + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + binary = work / "bin" + project = work / "project" + chart = project / "chart" + (chart / "templates").mkdir(parents=True) + (project / "Dockerfile").touch() + (chart / "Chart.yaml").write_text( + "apiVersion: v2\nname: fixture\nversion: 0.0.0\n" + ) + (chart / "values.yaml").write_text("image: fixture\n") + (chart / "templates/configmap.yaml").write_text( + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: fixture\n" + ) + binary.mkdir() + executable( + binary / "docker", + """import os +import sys +from pathlib import Path +Path(os.environ["OCI_TEST_DOCKER_LOG"]).write_text("\\n".join(sys.argv[1:]) + "\\n") +""", + ) + executable( + binary / "helm", + """import os +import sys +from pathlib import Path +arguments = sys.argv[1:] +log = Path(os.environ["OCI_TEST_HELM_LOG"]) +with log.open("a") as stream: + stream.write("\\n".join(arguments) + "\\n") +if arguments and arguments[0] == "package": + destination = Path(arguments[arguments.index("--destination") + 1]) + version = arguments[arguments.index("--version") + 1] + (destination / f"fixture-{version}.tgz").touch() +""", + ) + + docker_log = work / "docker.log" + helm_log = work / "helm.log" + environment = dict(os.environ) + environment.update( + PATH=f"{binary}:{environment['PATH']}", + OCI_TEST_DOCKER_LOG=str(docker_log), + OCI_TEST_HELM_LOG=str(helm_log), + ) + command = run.set(env=environment, inherit_env=False) + + await command( + root / "tasks/container.py", + "--context", + project, + "--platform", + "linux/amd64", + "--target", + "site", + "--build-arg", + "APP=site", + "--build-arg", + "TITLE=hello world", + "--label", + "org.example.title=Example site", + "--tag", + "ghcr.io/example/site:sha-abc", + "--tag", + "ghcr.io/example/site:latest", + "--cache-scope", + "site", + "--provenance", + "false", + "--sbom", + "false", + "--push", + ) + docker_arguments = docker_log.read_text().splitlines() + for expected in ( + "buildx", + "--target", + "site", + "TITLE=hello world", + "org.example.title=Example site", + "ghcr.io/example/site:sha-abc", + "ghcr.io/example/site:latest", + "type=gha,mode=max,scope=site", + "--push", + ): + assert expected in docker_arguments + + await command( + root / "tasks/chart.py", + "--chart", + chart, + "--version", + "1.2.3", + "--app-version", + "sha-abc", + "--output", + work / "output", + "--push", + "oci://ghcr.io/example/charts", + ) + helm_arguments = helm_log.read_text().splitlines() + for expected in ( + "lint", + "package", + "push", + "oci://ghcr.io/example/charts", + ): + assert expected in helm_arguments + + print("oci tasks: ok") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/tests/package b/tests/package deleted file mode 100755 index 6aa3fc0..0000000 --- a/tests/package +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -set -eu - -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM - -mkdir -p "$work/bin" "$work/stage/sub" "$work/out" -printf 'payload\n' > "$work/stage/tool" -printf 'nested\n' > "$work/stage/sub/file" -printf 'outside\n' > "$work/outside" -ln -s "$work/outside" "$work/stage/outside-link" - -cat > "$work/bin/nfpm" <<'EOF' -#!/bin/sh -set -eu -[ -z "${GPG_PRIVATE_KEY:-}" ] && \ -[ -z "${APK_PRIVATE_KEY:-}" ] && \ -[ -z "${SIGNING_PRIVATE_KEY:-}" ] && \ -[ -z "${NFPM_PASSPHRASE:-}" ] || { - echo 'raw private key leaked to nFPM' >&2 - exit 1 -} -printf 'ARCH=%s\nVERSION=%s\nGPG_KEY_ID=%s\n' "$ARCH" "$VERSION" "${GPG_KEY_ID:-}" >> "$PACKAGE_TEST_LOG" -while [ "$#" -gt 0 ]; do - case "$1" in - --config) config=$2; shift 2 ;; - --packager) packager=$2; shift 2 ;; - --target) target=$2; shift 2 ;; - *) shift ;; - esac -done -printf '%s\n' "$packager" >> "$PACKAGE_TEST_LOG" -grep '^key_name:' "$config" >> "$PACKAGE_TEST_LOG" || true -: > "$target/test.$packager" -EOF - -cat > "$work/bin/gpg" <<'EOF' -#!/bin/sh -set -eu -printf 'gpg\n' >> "$PACKAGE_TEST_LOG" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) output=$2; shift 2 ;; - *) shift ;; - esac -done -[ -z "${output:-}" ] || printf signature > "$output" -EOF - -for command in debsigs rpmsign; do - cat > "$work/bin/$command" <<'EOF' -#!/bin/sh -set -eu -printf '%s\n' "${0##*/}" >> "$PACKAGE_TEST_LOG" -EOF - chmod +x "$work/bin/$command" -done - -cat > "$work/bin/openssl" <<'EOF' -#!/bin/sh -set -eu -while [ "$#" -gt 0 ]; do - case "$1" in - -out) output=$2; shift 2 ;; - *) shift ;; - esac -done -printf 'public key\n' > "$output" -EOF - -chmod +x "$work/bin/nfpm" "$work/bin/gpg" "$work/bin/openssl" -printf 'name: test\n' > "$work/nfpm.yaml" -export PACKAGE_TEST_LOG="$work/package.log" -PATH="$work/bin:$PATH" -export PATH - -"$root/tasks/package" \ - --output "$work/out" \ - --archive-root "$work/stage" \ - --archive-name test-linux-amd64 \ - tar.gz zip - -test -f "$work/out/test-linux-amd64.tar.gz" -test -f "$work/out/test-linux-amd64.zip" -tar -tzf "$work/out/test-linux-amd64.tar.gz" | grep -q './tool' -unzip -l "$work/out/test-linux-amd64.zip" | grep -q 'sub/file' -test "$(unzip -p "$work/out/test-linux-amd64.zip" outside-link)" = "$work/outside" - -"$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version '1.2.3~nightly.42' \ - --arch arm64 \ - --output "$work/out" \ - deb rpm - -test -f "$work/out/test.deb" -test -f "$work/out/test.rpm" -grep -q '^ARCH=arm64$' "$work/package.log" -grep -q '^VERSION=1.2.3~nightly.42$' "$work/package.log" - -GPG_PRIVATE_KEY=private \ -GPG_PASSPHRASE=passphrase \ -GPG_KEY_ID=0123456789ABCDEF0123456789ABCDEF01234567 \ -APK_PRIVATE_KEY=apk-private \ - "$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - --apk-public-key test.rsa.pub \ - deb rpm apk - -test -f "$work/out/test.apk" -test -f "$work/out/test.rsa.pub" -grep -q '^gpg$' "$work/package.log" -grep -q '^debsigs$' "$work/package.log" -grep -q '^rpmsign$' "$work/package.log" -grep -q '^GPG_KEY_ID=89ABCDEF01234567$' "$work/package.log" - -# Repository publication exports the organization APK public key, so package -# builds do not need to stage a per-service public key artifact. -APK_PRIVATE_KEY=apk-private \ - "$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - apk - -printf 'key_name: packages.${PACKAGE_KEY_VERSION}\n' > "$work/versioned.yaml" -if APK_PRIVATE_KEY=apk-private "$root/tasks/package" \ - --config "$work/versioned.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - apk 2>/dev/null; then - echo 'package test: accepted versioned APK config without key version' >&2 - exit 1 -fi -APK_PRIVATE_KEY=apk-private \ -PACKAGE_KEY_VERSION=0001 \ - "$root/tasks/package" \ - --config "$work/versioned.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - apk -grep -q '^key_name: packages.0001$' "$work/package.log" - -if "$root/tasks/package" --output "$work/out" deb 2>/dev/null; then - echo 'package test: accepted system format without version and architecture' >&2 - exit 1 -fi - -if "$root/tasks/package" \ - --output "$work/out" \ - --archive-root "$work/stage" \ - --archive-name '../escape' \ - zip 2>/dev/null; then - echo 'package test: accepted unsafe archive name' >&2 - exit 1 -fi - -if APK_SIGNING_KEY="$work/apk.rsa" "$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - --apk-public-key '../escape' \ - apk 2>/dev/null; then - echo 'package test: accepted unsafe APK public key name' >&2 - exit 1 -fi diff --git a/tests/package-integration b/tests/package-integration deleted file mode 100755 index 362e0bf..0000000 --- a/tests/package-integration +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -set -eu - -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM - -mkdir -p "$work/stage" -printf 'package contract\n' > "$work/stage/package-contract" -cat > "$work/nfpm.yaml" <<'EOF' -name: package-contract -arch: ${ARCH} -version: ${VERSION} -platform: linux -maintainer: Dimidium Labs -description: Shared package task integration fixture -license: 0BSD -EOF - -mise --cd "$root" run package -- \ - --config "$work/nfpm.yaml" \ - --version '1.2.3~nightly.42' \ - --arch amd64 \ - --output "$work/out" \ - --archive-root "$work/stage" \ - --archive-name package-contract-linux-amd64 \ - deb rpm apk tar.gz zip - -test "$(find "$work/out" -maxdepth 1 -name '*.deb' | wc -l)" -eq 1 -test "$(find "$work/out" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 1 -test "$(find "$work/out" -maxdepth 1 -name '*.apk' | wc -l)" -eq 1 -test -f "$work/out/package-contract-linux-amd64.tar.gz" -test -f "$work/out/package-contract-linux-amd64.zip" - -test "$(dpkg-deb -f "$work/out"/*.deb Package)" = package-contract -test "$(dpkg-deb -f "$work/out"/*.deb Version)" = '1.2.3~nightly.42' -test "$(dpkg-deb -f "$work/out"/*.deb Architecture)" = amd64 -tar -tzf "$work/out/package-contract-linux-amd64.tar.gz" | grep -q './package-contract' -unzip -l "$work/out/package-contract-linux-amd64.zip" | grep -q 'package-contract' diff --git a/tests/package-integration.py b/tests/package-integration.py new file mode 100755 index 0000000..25212b5 --- /dev/null +++ b/tests/package-integration.py @@ -0,0 +1,93 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from shellous import sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + stage = work / "stage" + stage.mkdir() + (stage / "package-contract").write_text("package contract\n") + config = work / "nfpm.yaml" + config.write_text( + """name: package-contract +arch: ${ARCH} +version: ${VERSION} +platform: linux +maintainer: Dimidium Labs +description: Shared package task integration fixture +license: 0BSD +""" + ) + output = work / "out" + await run( + "mise", + "--cd", + root, + "run", + "package", + "--", + "--config", + config, + "--version", + "1.2.3~nightly.42", + "--arch", + "amd64", + "--output", + output, + "--archive-root", + stage, + "--archive-name", + "package-contract-linux-amd64", + "deb", + "rpm", + "apk", + "tar.gz", + "zip", + ) + + assert len(list(output.glob("*.deb"))) == 1 + assert len(list(output.glob("*.rpm"))) == 1 + assert len(list(output.glob("*.apk"))) == 1 + tarball = output / "package-contract-linux-amd64.tar.gz" + zipfile = output / "package-contract-linux-amd64.zip" + assert tarball.is_file() + assert zipfile.is_file() + + package = next(output.glob("*.deb")) + assert (await capture("dpkg-deb", "-f", package, "Package")).strip() == ( + "package-contract" + ) + assert (await capture("dpkg-deb", "-f", package, "Version")).strip() == ( + "1.2.3~nightly.42" + ) + assert (await capture("dpkg-deb", "-f", package, "Architecture")).strip() == ( + "amd64" + ) + assert "./package-contract" in await capture("tar", "-tzf", tarball) + assert "package-contract" in await capture("unzip", "-l", zipfile) + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/tests/package.py b/tests/package.py new file mode 100755 index 0000000..5a2c756 --- /dev/null +++ b/tests/package.py @@ -0,0 +1,270 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from shellous import sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) + + +def executable(path: Path, source: str) -> None: + path.write_text("#!/usr/bin/env python3\n" + source) + path.chmod(0o755) + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + binary = work / "bin" + stage = work / "stage" + output = work / "out" + binary.mkdir() + (stage / "sub").mkdir(parents=True) + output.mkdir() + (stage / "tool").write_text("payload\n") + (stage / "sub/file").write_text("nested\n") + outside = work / "outside" + outside.write_text("outside\n") + (stage / "outside-link").symlink_to(outside) + + executable( + binary / "nfpm", + """import os +import sys +from pathlib import Path +secrets = ( + "GPG_PRIVATE_KEY", "APK_PRIVATE_KEY", "SIGNING_PRIVATE_KEY", + "NFPM_PASSPHRASE", +) +if any(os.environ.get(name) for name in secrets): + print("raw private key leaked to nFPM", file=sys.stderr) + raise SystemExit(1) +arguments = sys.argv[1:] +config = Path(arguments[arguments.index("--config") + 1]) +packager = arguments[arguments.index("--packager") + 1] +target = Path(arguments[arguments.index("--target") + 1]) +with Path(os.environ["PACKAGE_TEST_LOG"]).open("a") as stream: + stream.write(f"ARCH={os.environ['ARCH']}\\n") + stream.write(f"VERSION={os.environ['VERSION']}\\n") + stream.write(f"GPG_KEY_ID={os.environ.get('GPG_KEY_ID', '')}\\n") + stream.write(f"{packager}\\n") + for line in config.read_text().splitlines(): + if line.startswith("key_name:"): + stream.write(f"{line}\\n") +(target / f"test.{packager}").touch() +""", + ) + executable( + binary / "gpg", + """import os +import sys +from pathlib import Path +with Path(os.environ["PACKAGE_TEST_LOG"]).open("a") as stream: + stream.write("gpg\\n") +arguments = sys.argv[1:] +if "-o" in arguments: + Path(arguments[arguments.index("-o") + 1]).write_text("signature") +""", + ) + for name in ("debsigs", "rpmsign"): + executable( + binary / name, + """import os +import sys +from pathlib import Path +with Path(os.environ["PACKAGE_TEST_LOG"]).open("a") as stream: + stream.write(f"{Path(sys.argv[0]).name}\\n") +""", + ) + executable( + binary / "openssl", + """import sys +from pathlib import Path +arguments = sys.argv[1:] +Path(arguments[arguments.index("-out") + 1]).write_text("public key\\n") +""", + ) + + config = work / "nfpm.yaml" + config.write_text("name: test\n") + log = work / "package.log" + environment = dict(os.environ) + environment.update( + PATH=f"{binary}:{environment['PATH']}", + PACKAGE_TEST_LOG=str(log), + ) + command = run.set(env=environment, inherit_env=False) + package = root / "tasks/package.py" + + await command( + package, + "--output", + output, + "--archive-root", + stage, + "--archive-name", + "test-linux-amd64", + "tar.gz", + "zip", + ) + tarball = output / "test-linux-amd64.tar.gz" + zipfile = output / "test-linux-amd64.zip" + assert tarball.is_file() + assert zipfile.is_file() + assert "./tool" in await capture("tar", "-tzf", tarball) + assert "sub/file" in await capture("unzip", "-l", zipfile) + assert (await capture("unzip", "-p", zipfile, "outside-link")) == str(outside) + + await command( + package, + "--config", + config, + "--version", + "1.2.3~nightly.42", + "--arch", + "arm64", + "--output", + output, + "deb", + "rpm", + ) + assert (output / "test.deb").is_file() + assert (output / "test.rpm").is_file() + assert "ARCH=arm64" in log.read_text().splitlines() + assert "VERSION=1.2.3~nightly.42" in log.read_text().splitlines() + + signing_environment = dict(environment) + signing_environment.update( + GPG_PRIVATE_KEY="private", + GPG_PASSPHRASE="passphrase", + GPG_KEY_ID="0123456789ABCDEF0123456789ABCDEF01234567", + APK_PRIVATE_KEY="apk-private", + ) + await run.set(env=signing_environment, inherit_env=False)( + package, + "--config", + config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "--apk-public-key", + "test.rsa.pub", + "deb", + "rpm", + "apk", + ) + assert (output / "test.apk").is_file() + assert (output / "test.rsa.pub").is_file() + log_lines = log.read_text().splitlines() + for expected in ( + "gpg", + "debsigs", + "rpmsign", + "GPG_KEY_ID=89ABCDEF01234567", + ): + assert expected in log_lines + + apk_environment = dict(environment, APK_PRIVATE_KEY="apk-private") + await run.set(env=apk_environment, inherit_env=False)( + package, + "--config", + config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "apk", + ) + + versioned_config = work / "versioned.yaml" + versioned_config.write_text("key_name: packages.${PACKAGE_KEY_VERSION}\n") + result = await run.result.set(env=apk_environment, inherit_env=False)( + package, + "--config", + versioned_config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "apk", + ).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + versioned_environment = dict(apk_environment, PACKAGE_KEY_VERSION="0001") + await run.set(env=versioned_environment, inherit_env=False)( + package, + "--config", + versioned_config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "apk", + ) + assert "key_name: packages.0001" in log.read_text().splitlines() + + invalid_commands = ( + (package, "--output", output, "deb"), + ( + package, + "--output", + output, + "--archive-root", + stage, + "--archive-name", + "../escape", + "zip", + ), + ) + for arguments in invalid_commands: + result = await command.result(*arguments).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + external_key_environment = dict( + environment, APK_SIGNING_KEY=str(work / "apk.rsa") + ) + result = await run.result.set(env=external_key_environment, inherit_env=False)( + package, + "--config", + config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "--apk-public-key", + "../escape", + "apk", + ).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/tests/publish-integration b/tests/publish-integration deleted file mode 100755 index 5081943..0000000 --- a/tests/publish-integration +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -set -eu - -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM -remote="$work/remote/integration" -mkdir -p "$work/bin" "$work/input" "$remote" "$work/fixture" - -printf 'publish integration fixture\n' > "$work/fixture/publish-contract" -cat > "$work/fixture/nfpm.yaml" < -description: Shared publish task integration fixture -license: 0BSD -contents: - - src: $work/fixture/publish-contract - dst: /usr/local/bin/publish-contract -deb: - signature: - method: debsign - key_id: \${GPG_KEY_ID} - key_file: \${SIGNING_PRIVATE_KEY} -rpm: - signature: - key_id: \${GPG_KEY_ID} - key_file: \${SIGNING_PRIVATE_KEY} -apk: - signature: - key_file: \${APK_SIGNING_KEY} - key_name: packages.\${PACKAGE_KEY_VERSION} -EOF - -export GNUPGHOME="$work/source-gnupg" -mkdir -m 700 "$GNUPGHOME" -gpg --batch --pinentry-mode loopback --passphrase integration-pass \ - --quick-generate-key 'Publish Integration ' rsa2048 sign 1d >/dev/null -fingerprint=$(gpg --batch --with-colons --list-secret-keys | - awk -F: '$1 == "fpr" { print $10; exit }') -gpg --batch --pinentry-mode loopback --passphrase integration-pass \ - --armor --export-secret-keys "$fingerprint" > "$work/private.gpg" -openssl genrsa -out "$work/private.rsa" 2048 >/dev/null 2>&1 -# Organization keys are provisioned independently under a versioned archive. -key_version=0001 -mkdir -p "$remote/keys" -gpg --batch --armor --export "$fingerprint" > "$remote/packages.gpg" -cp "$remote/packages.gpg" "$remote/keys/packages.$key_version.gpg" -openssl rsa -in "$work/private.rsa" -pubout \ - -out "$remote/keys/packages.$key_version.rsa.pub" >/dev/null 2>&1 - -gpg_private_key=$(cat "$work/private.gpg") -apk_private_key=$(cat "$work/private.rsa") - -package_version() { - version=$1 - GPG_PRIVATE_KEY="$gpg_private_key" \ - GPG_PASSPHRASE=integration-pass \ - GPG_KEY_ID="$fingerprint" \ - APK_PRIVATE_KEY="$apk_private_key" \ - PACKAGE_KEY_VERSION="$key_version" \ - mise --cd "$root" run package -- \ - --config "$work/fixture/nfpm.yaml" \ - --version "$version" \ - --arch amd64 \ - --output "$work/input" \ - deb rpm apk -} - -publish() { - PUBLISH_REMOTE="$work/remote" \ - PUBLISH_TEST_LOG="$work/s3.log" \ - PYTHONPATH="$root/tests/fakes" \ - PATH="$work/bin:$PATH" \ - S3_BUCKET=integration \ - S3_ENDPOINT=https://example.invalid \ - S3_PUBLIC_URL=https://pkg.dimidiumlabs.io \ - S3_ACCESS_KEY_ID=integration \ - S3_SECRET_ACCESS_KEY=integration \ - GPG_PRIVATE_KEY="$gpg_private_key" \ - GPG_PASSPHRASE=integration-pass \ - GPG_KEY_ID="$fingerprint" \ - APK_PRIVATE_KEY="$apk_private_key" \ - PACKAGE_KEY_VERSION="$key_version" \ - "$root/tasks/publish" \ - --service publish-contract \ - --channel nightly \ - --input "$work/input" \ - deb rpm apk -} - -package_version '1.2.3~nightly.42' -publish - -apt_root="$remote/publish-contract/apt" -rpm_root="$remote/publish-contract/rpm/nightly" -apk_root="$remote/publish-contract/apk/nightly/x86_64" - -test -f "$remote/packages.gpg" -test -f "$remote/keys/packages.$key_version.gpg" -test -f "$remote/keys/packages.$key_version.rsa.pub" -test "$(find "$apt_root/pool/nightly" -name '*.deb' | wc -l)" -eq 1 -test "$(find "$rpm_root" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 1 -test "$(find "$apk_root" -maxdepth 1 -name '*.apk' | wc -l)" -eq 1 -grep -q '^Package: publish-contract$' \ - "$apt_root/dists/nightly/main/binary-amd64/Packages" -grep -q 'baseurl=https://pkg.dimidiumlabs.io/publish-contract/rpm/nightly/' \ - "$rpm_root/publish-contract-nightly.repo" - -keyring="$work/packages.gpg" -gpg --batch --dearmor -o "$keyring" "$remote/packages.gpg" -gpgv --keyring "$keyring" "$apt_root/dists/nightly/InRelease" >/dev/null -gpgv --keyring "$keyring" "$rpm_root/repodata/repomd.xml.asc" \ - "$rpm_root/repodata/repomd.xml" >/dev/null - -apk_tool=$(find "$HOME/.cache/mise" "$HOME/.local/share/mise" -type f -name apk.static 2>/dev/null | head -n 1 || true) -if [ -z "$apk_tool" ]; then - archive="$work/apk-tools-static.apk" - curl -fsSL \ - https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/apk-tools-static-2.14.10-r0.apk \ - -o "$archive" - printf '%s %s\n' c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61 "$archive" | - sha256sum -c - >/dev/null - mkdir -p "$work/apk-tools" - tar -xzf "$archive" -C "$work/apk-tools" sbin/apk.static 2>/dev/null - apk_tool="$work/apk-tools/sbin/apk.static" -fi -mkdir "$work/apk-keys" -cp "$remote/keys/packages.$key_version.rsa.pub" "$work/apk-keys/" -"$apk_tool" verify --keys-dir "$work/apk-keys" "$apk_root/APKINDEX.tar.gz" >/dev/null - -# A later publication keeps every package payload, regenerates metadata, and -# removes stale metadata only inside the corresponding repository prefixes. -printf stale > "$apt_root/dists/nightly/stale" -printf stale > "$rpm_root/repodata/stale" -rm -rf "$work/input" -mkdir "$work/input" -package_version '1.2.3~nightly.43' -publish -test "$(find "$apt_root/pool/nightly" -name '*.deb' | wc -l)" -eq 2 -test "$(find "$rpm_root" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 2 -test "$(find "$apk_root" -maxdepth 1 -name '*.apk' | wc -l)" -eq 2 -test ! -e "$apt_root/dists/nightly/stale" -test ! -e "$rpm_root/repodata/stale" -test "$(grep -c '^Package: publish-contract$' \ - "$apt_root/dists/nightly/main/binary-amd64/Packages")" -eq 2 -"$apk_tool" verify --keys-dir "$work/apk-keys" "$apk_root/APKINDEX.tar.gz" >/dev/null - -# Existing package names and organization keys are immutable. -deb=$(find "$work/input" -maxdepth 1 -name '*.deb') -printf '\nchanged\n' >> "$deb" -if publish 2>/dev/null; then - echo 'publish integration: replaced an immutable package payload' >&2 - exit 1 -fi -cp "$apt_root/pool/nightly/${deb##*/}" "$deb" -lock="$remote/publish-contract/_locks/nightly" -mkdir -p "${lock%/*}" -printf '{"expires":9999999999}\n' > "$lock" -if publish 2>/dev/null; then - echo 'publish integration: ignored an active publication lock' >&2 - exit 1 -fi -rm "$lock" -printf 'different key\n' > "$remote/keys/packages.$key_version.gpg" -if publish 2>/dev/null; then - echo 'publish integration: ignored a mismatched versioned key' >&2 - exit 1 -fi - -# Metadata cleanup must never escape generated repository prefixes. -if awk '$1 == "s3" && $2 == "delete" && $3 !~ /^publish-contract\/(apt\/dists|rpm\/nightly\/repodata|_locks\/nightly)/' "$work/s3.log" | grep . >/dev/null; then - echo 'publish integration: deleted outside a metadata prefix' >&2 - exit 1 -fi -if PYTHONPATH="$root/tests/fakes" "$root/tasks/publish" \ - --service '../escape' --channel nightly --input "$work/input" deb 2>/dev/null; then - echo 'publish integration: accepted an unsafe service name' >&2 - exit 1 -fi diff --git a/tests/publish-integration.py b/tests/publish-integration.py new file mode 100755 index 0000000..db2ee9c --- /dev/null +++ b/tests/publish-integration.py @@ -0,0 +1,346 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import hashlib +import os +import re +import shutil +import sys +import tempfile +import urllib.request +from collections.abc import Sequence +from pathlib import Path + +from shellous import Result, sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) +APK_TOOLS_URL = ( + "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/" + "apk-tools-static-2.14.10-r0.apk" +) +APK_TOOLS_SHA256 = "c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61" + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + remote = work / "remote/integration" + binary = work / "bin" + package_input = work / "input" + fixture = work / "fixture" + for path in (remote, binary, package_input, fixture): + path.mkdir(parents=True) + + payload = fixture / "publish-contract" + payload.write_text("publish integration fixture\n") + config = fixture / "nfpm.yaml" + config.write_text( + f"""name: publish-contract +arch: ${{ARCH}} +version: ${{VERSION}} +platform: linux +maintainer: Dimidium Labs +description: Shared publish task integration fixture +license: 0BSD +contents: + - src: {payload} + dst: /usr/local/bin/publish-contract +deb: + signature: + method: debsign + key_id: ${{GPG_KEY_ID}} + key_file: ${{SIGNING_PRIVATE_KEY}} +rpm: + signature: + key_id: ${{GPG_KEY_ID}} + key_file: ${{SIGNING_PRIVATE_KEY}} +apk: + signature: + key_file: ${{APK_SIGNING_KEY}} + key_name: packages.${{PACKAGE_KEY_VERSION}} +""" + ) + + source_gnupg = work / "source-gnupg" + source_gnupg.mkdir(mode=0o700) + source_environment = dict(os.environ, GNUPGHOME=str(source_gnupg)) + gpg = run.set(env=source_environment, inherit_env=False) + gpg_capture = capture.set(env=source_environment, inherit_env=False) + await gpg( + "gpg", + "--batch", + "--pinentry-mode", + "loopback", + "--passphrase", + "integration-pass", + "--quick-generate-key", + "Publish Integration ", + "rsa2048", + "sign", + "1d", + ).stdout(sh.DEVNULL) + key_listing = await gpg_capture( + "gpg", "--batch", "--with-colons", "--list-secret-keys" + ) + fingerprint = next( + fields[9] + for line in key_listing.splitlines() + if (fields := line.split(":"))[0] == "fpr" + ) + private_gpg = work / "private.gpg" + await gpg( + "gpg", + "--batch", + "--pinentry-mode", + "loopback", + "--passphrase", + "integration-pass", + "--armor", + "--export-secret-keys", + fingerprint, + ).stdout(private_gpg) + private_rsa = work / "private.rsa" + await ( + run("openssl", "genrsa", "-out", private_rsa, "2048") + .stdout(sh.DEVNULL) + .stderr(sh.DEVNULL) + ) + + key_version = "0001" + keys = remote / "keys" + keys.mkdir() + public_gpg = remote / "packages.gpg" + await gpg("gpg", "--batch", "--armor", "--export", fingerprint).stdout( + public_gpg + ) + shutil.copy2(public_gpg, keys / f"packages.{key_version}.gpg") + await ( + run( + "openssl", + "rsa", + "-in", + private_rsa, + "-pubout", + "-out", + keys / f"packages.{key_version}.rsa.pub", + ) + .stdout(sh.DEVNULL) + .stderr(sh.DEVNULL) + ) + gpg_private_key = private_gpg.read_text() + apk_private_key = private_rsa.read_text() + + async def package_version(version: str) -> None: + environment = dict(os.environ) + environment.update( + GPG_PRIVATE_KEY=gpg_private_key, + GPG_PASSPHRASE="integration-pass", + GPG_KEY_ID=fingerprint, + APK_PRIVATE_KEY=apk_private_key, + PACKAGE_KEY_VERSION=key_version, + ) + await run.set(env=environment, inherit_env=False)( + "mise", + "--cd", + root, + "run", + "package", + "--", + "--config", + config, + "--version", + version, + "--arch", + "amd64", + "--output", + package_input, + "deb", + "rpm", + "apk", + ) + + publish_environment = dict(os.environ) + publish_environment.update( + PUBLISH_REMOTE=str(work / "remote"), + PUBLISH_TEST_LOG=str(work / "s3.log"), + PUBLISH_TEST_PYTHONPATH=str(root / "tests/fakes"), + PATH=f"{binary}:{os.environ['PATH']}", + S3_BUCKET="integration", + S3_ENDPOINT="https://example.invalid", + S3_PUBLIC_URL="https://pkg.dimidiumlabs.io", + S3_ACCESS_KEY_ID="integration", + S3_SECRET_ACCESS_KEY="integration", + GPG_PRIVATE_KEY=gpg_private_key, + GPG_PASSPHRASE="integration-pass", + GPG_KEY_ID=fingerprint, + APK_PRIVATE_KEY=apk_private_key, + PACKAGE_KEY_VERSION=key_version, + ) + publish_command = run.result.set(env=publish_environment, inherit_env=False) + + async def publish(*, quiet: bool = False) -> Result: + command = publish_command( + root / "tasks/publish.py", + "--service", + "publish-contract", + "--channel", + "nightly", + "--input", + package_input, + "deb", + "rpm", + "apk", + ) + if quiet: + command = command.stderr(sh.DEVNULL) + return await command + + await package_version("1.2.3~nightly.42") + assert (await publish()).exit_code == 0 + + apt_root = remote / "publish-contract/apt" + rpm_root = remote / "publish-contract/rpm/nightly" + apk_root = remote / "publish-contract/apk/nightly/x86_64" + assert public_gpg.is_file() + assert (keys / f"packages.{key_version}.gpg").is_file() + assert (keys / f"packages.{key_version}.rsa.pub").is_file() + assert len(list((apt_root / "pool/nightly").glob("*.deb"))) == 1 + assert len(list(rpm_root.glob("*.rpm"))) == 1 + assert len(list(apk_root.glob("*.apk"))) == 1 + packages_file = apt_root / "dists/nightly/main/binary-amd64/Packages" + assert "Package: publish-contract" in packages_file.read_text().splitlines() + repository_file = rpm_root / "publish-contract-nightly.repo" + assert ( + "baseurl=https://pkg.dimidiumlabs.io/publish-contract/rpm/nightly/" + in repository_file.read_text().splitlines() + ) + + keyring = work / "packages.gpg" + await run("gpg", "--batch", "--dearmor", "-o", keyring, public_gpg) + await run( + "gpgv", + "--keyring", + keyring, + apt_root / "dists/nightly/InRelease", + ).stdout(sh.DEVNULL) + await run( + "gpgv", + "--keyring", + keyring, + rpm_root / "repodata/repomd.xml.asc", + rpm_root / "repodata/repomd.xml", + ).stdout(sh.DEVNULL) + + apk_tool = next( + ( + path + for base in ( + Path.home() / ".cache/mise", + Path.home() / ".local/share/mise", + ) + if base.is_dir() + for path in base.rglob("apk.static") + if path.is_file() + ), + None, + ) + if apk_tool is None: + archive = work / "apk-tools-static.apk" + await asyncio.to_thread(urllib.request.urlretrieve, APK_TOOLS_URL, archive) + assert hashlib.sha256(archive.read_bytes()).hexdigest() == APK_TOOLS_SHA256 + apk_directory = work / "apk-tools" + apk_directory.mkdir() + await run( + "tar", + "-xzf", + archive, + "-C", + apk_directory, + "sbin/apk.static", + ).stderr(sh.DEVNULL) + apk_tool = apk_directory / "sbin/apk.static" + apk_keys = work / "apk-keys" + apk_keys.mkdir() + shutil.copy2(keys / f"packages.{key_version}.rsa.pub", apk_keys) + await run( + apk_tool, + "verify", + "--keys-dir", + apk_keys, + apk_root / "APKINDEX.tar.gz", + ).stdout(sh.DEVNULL) + + (apt_root / "dists/nightly/stale").write_text("stale") + (rpm_root / "repodata/stale").write_text("stale") + shutil.rmtree(package_input) + package_input.mkdir() + await package_version("1.2.3~nightly.43") + assert (await publish()).exit_code == 0 + assert len(list((apt_root / "pool/nightly").glob("*.deb"))) == 2 + assert len(list(rpm_root.glob("*.rpm"))) == 2 + assert len(list(apk_root.glob("*.apk"))) == 2 + assert not (apt_root / "dists/nightly/stale").exists() + assert not (rpm_root / "repodata/stale").exists() + assert ( + packages_file.read_text().splitlines().count("Package: publish-contract") + == 2 + ) + await run( + apk_tool, + "verify", + "--keys-dir", + apk_keys, + apk_root / "APKINDEX.tar.gz", + ).stdout(sh.DEVNULL) + + deb = next(package_input.glob("*.deb")) + with deb.open("a") as stream: + stream.write("\nchanged\n") + assert (await publish(quiet=True)).exit_code != 0 + shutil.copy2(apt_root / "pool/nightly" / deb.name, deb) + lock = remote / "publish-contract/_locks/nightly" + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text('{"expires":9999999999}\n') + assert (await publish(quiet=True)).exit_code != 0 + lock.unlink() + (keys / f"packages.{key_version}.gpg").write_text("different key\n") + assert (await publish(quiet=True)).exit_code != 0 + + allowed_delete = re.compile( + r"^publish-contract/(apt/dists|rpm/nightly/repodata|_locks/nightly)" + ) + for line in (work / "s3.log").read_text().splitlines(): + fields = line.split(maxsplit=2) + if len(fields) == 3 and fields[:2] == ["s3", "delete"]: + assert allowed_delete.match(fields[2]) + + invalid_environment = dict( + os.environ, PUBLISH_TEST_PYTHONPATH=str(root / "tests/fakes") + ) + result = await run.result.set(env=invalid_environment, inherit_env=False)( + root / "tasks/publish.py", + "--service", + "../escape", + "--channel", + "nightly", + "--input", + package_input, + "deb", + ).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) -- Gilti