From 4a4589fdb968686a7c91b380ff369aba23bd8662 Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Thu, 20 Aug 2026 05:15:43 +0100 Subject: Add shared script for publish linux repositories --- .github/workflows/ci.yml | 15 +- .gitignore | 5 + README.md | 42 ++++- REUSE.toml | 1 + mise.lock | 93 ++++++++++ mise.toml | 19 ++ tasks/_apk.py | 146 +++++++++++++++ tasks/_apt.py | 87 +++++++++ tasks/_lib.py | 153 ++++++++++++++++ tasks/_repository.py | 111 ++++++++++++ tasks/_rpm.py | 54 ++++++ tasks/_storage.py | 155 ++++++++++++++++ tasks/package | 364 ++++++++++++++++++++++++-------------- tasks/publish | 76 ++++++++ tests/fakes/boto3.py | 127 +++++++++++++ tests/package | 57 +++++- tests/publish-integration | 185 +++++++++++++++++++ 17 files changed, 1553 insertions(+), 137 deletions(-) create mode 100644 .gitignore create mode 100644 mise.lock create mode 100644 tasks/_apk.py create mode 100644 tasks/_apt.py create mode 100644 tasks/_lib.py create mode 100644 tasks/_repository.py create mode 100644 tasks/_rpm.py create mode 100644 tasks/_storage.py create mode 100755 tasks/publish create mode 100644 tests/fakes/boto3.py create mode 100755 tests/publish-integration diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44a9e4c..fc0c63e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,8 +13,8 @@ permissions: contents: read jobs: - package: - name: Package task + release-tasks: + name: Package and publish tasks runs-on: ubuntu-latest steps: - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -25,7 +25,18 @@ jobs: experimental: true install: false + - name: Install dependencies + run: mise bootstrap --locked --yes --update + + - name: Check Python + run: | + ruff check tasks/_*.py tasks/package tasks/publish tests/fakes/boto3.py + ruff format --check tasks/_*.py tasks/package tasks/publish tests/fakes/boto3.py + - name: Check package contract run: | tests/package tests/package-integration + + - name: Check publish contract + run: tests/publish-integration diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a56ba84 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +__pycache__/ +*.py[cod] diff --git a/README.md b/README.md index 4635e04..1bffdef 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,14 @@ Consuming projects pin this repository by commit SHA. Projects build and stage their own binaries and keep their nFPM configuration. The shared [`package`](tasks/package) task creates only the formats explicitly requested by a project: nFPM packages (`deb`, `rpm`, or `apk`) and portable -archives (`tar.gz` or `zip`). Repository publication is intentionally outside -this task. +archives (`tar.gz` or `zip`). APK configurations may use +`${PACKAGE_KEY_VERSION}` in `apk.signature.key_name`; the task renders the +four-digit generation before invoking nFPM. DEB and RPM payloads are built by +nFPM and then signed through `debsigs` and `rpmsign`, allowing CI to use only an +OpenPGP signing subkey while the certification key remains offline. The shared +[`publish`](tasks/publish) task adds +explicitly selected package formats to signed repositories in the organization +package bucket. ```console mise run package -- \ @@ -33,6 +39,38 @@ mise run package -- \ tar.gz zip ``` +## Package repositories + +Projects publish beneath a service-owned prefix at +`https://pkg.dimidiumlabs.io//`. Channels are explicit, previously +published package payloads are retained, and an S3 lock serializes metadata +updates for each service/channel. + +```console +mise run publish -- \ + --service SERVICE --channel CHANNEL --input DIR \ + deb rpm apk +``` + +The selected formats map to these layouts: + +- APT: `/apt/{dists,pool}//` +- RPM: `/rpm//` +- APK: `/apk///` + +APT and RPM metadata refer to the aggregate organization OpenPGP bundle at +`/packages.gpg`. Immutable generation keys live at +`/keys/packages..gpg` and +`/keys/packages..rsa.pub`. APK packages and indexes embed +the versioned RSA key name. Public keys are provisioned independently; each +publication checks its signing keys against the selected generation and never +creates or replaces key objects. + +Bucket configuration comes from `S3_BUCKET`, `S3_ENDPOINT`, `S3_PUBLIC_URL`, +`S3_ACCESS_KEY_ID`, and `S3_SECRET_ACCESS_KEY`. `PACKAGE_KEY_VERSION` selects +the four-digit key generation. OpenPGP signing uses `GPG_PRIVATE_KEY`, +`GPG_PASSPHRASE`, and `GPG_KEY_ID`; APK index signing uses `APK_PRIVATE_KEY`. + ## Tool provisioning Each project declares its toolchain and standalone CLI dependencies in diff --git a/REUSE.toml b/REUSE.toml index bff4b49..e4b6255 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -20,6 +20,7 @@ SPDX-License-Identifier = "CC-BY-3.0" [[annotations]] path = [ ".mailmap", + "mise.lock", ] SPDX-FileCopyrightText = "2026 Nikolay Govorov" SPDX-License-Identifier = "0BSD" diff --git a/mise.lock b/mise.lock new file mode 100644 index 0000000..04ef64a --- /dev/null +++ b/mise.lock @@ -0,0 +1,93 @@ +# @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html + +[[tools.ruff]] +version = "0.16.3" +backend = "aqua:astral-sh/ruff" + +[tools.ruff."platforms.linux-arm64"] +checksum = "sha256:8319ba22f655e6efc086103486d7165bf0de73f71ff1c5f25ba580153ad05feb" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-aarch64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140238" +provenance = "github-attestations" + +[tools.ruff."platforms.linux-arm64-musl"] +checksum = "sha256:8319ba22f655e6efc086103486d7165bf0de73f71ff1c5f25ba580153ad05feb" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-aarch64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140238" +provenance = "github-attestations" + +[tools.ruff."platforms.linux-x64"] +checksum = "sha256:d67c9b5949981698c48915abf65e0b3406ba9184ad73521cdf20a926bc889c73" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140356" +provenance = "github-attestations" + +[tools.ruff."platforms.linux-x64-musl"] +checksum = "sha256:d67c9b5949981698c48915abf65e0b3406ba9184ad73521cdf20a926bc889c73" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140356" +provenance = "github-attestations" + +[tools.ruff."platforms.macos-arm64"] +checksum = "sha256:136a4db6512d9b16dda56ac8604696ed65c3b1a914a142de029e7f8d5006f1d9" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-aarch64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140218" +provenance = "github-attestations" + +[tools.ruff."platforms.macos-x64"] +checksum = "sha256:05c2a6705e7c0c056d6d93ff538978583f0c47b4c28d334ab9d58d2e8daf4c24" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-x86_64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140338" +provenance = "github-attestations" + +[tools.ruff."platforms.windows-x64"] +checksum = "sha256:f10c709755b393fd9821506b21070bcca969b9966504edd1e490efd08e3662ba" +url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-x86_64-pc-windows-msvc.zip" +url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140346" +provenance = "github-attestations" + +[[tools.uv]] +version = "0.12.5" +backend = "aqua:astral-sh/uv" + +[tools.uv."platforms.linux-arm64"] +checksum = "sha256:8767a0e77f2cd45436401b1b42bf7e9ed5a4a91a74a5305d6fe93249d0f6dbc5" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850993" +provenance = "github-attestations" + +[tools.uv."platforms.linux-arm64-musl"] +checksum = "sha256:8767a0e77f2cd45436401b1b42bf7e9ed5a4a91a74a5305d6fe93249d0f6dbc5" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850993" +provenance = "github-attestations" + +[tools.uv."platforms.linux-x64"] +checksum = "sha256:a4742988791c9aeae68c78150d6cba762062ad2a47e53738c2779d2b596bfcdb" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851105" +provenance = "github-attestations" + +[tools.uv."platforms.linux-x64-musl"] +checksum = "sha256:a4742988791c9aeae68c78150d6cba762062ad2a47e53738c2779d2b596bfcdb" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-unknown-linux-musl.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851105" +provenance = "github-attestations" + +[tools.uv."platforms.macos-arm64"] +checksum = "sha256:5bb0e5fe008a773c3dbcb97ff79cd89e1241464fe9d2f986d52ad8f1b037bd62" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850968" +provenance = "github-attestations" + +[tools.uv."platforms.macos-x64"] +checksum = "sha256:b3b2137477cf96c9686ebfb71524614cec780c673fd73e59bce099aef02e70e8" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-apple-darwin.tar.gz" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851078" +provenance = "github-attestations" + +[tools.uv."platforms.windows-x64"] +checksum = "sha256:4c4d49d8738847d9b71ba319e49a5688c93eac0fe6204b1df24e98528dddf39a" +url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-pc-windows-msvc.zip" +url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851084" +provenance = "github-attestations" diff --git a/mise.toml b/mise.toml index d9dc88f..e4a7671 100644 --- a/mise.toml +++ b/mise.toml @@ -6,6 +6,25 @@ min_version = "2026.7.5" [settings] experimental = true +[tools] +ruff = "0.16.3" +uv = "0.12.5" + +[bootstrap.packages] +# APT +"apt:apt-utils" = "latest" +"apt:createrepo-c" = "latest" +"apt:debsigs" = "latest" +"apt:gnupg" = "latest" +"apt:openssl" = "latest" +"apt:rpm" = "latest" + +# DNF +"dnf:createrepo_c" = "latest" +"dnf:gnupg2" = "latest" +"dnf:openssl" = "latest" +"dnf:rpm" = "latest" + [task_config] dir = "{{cwd}}" includes = ["tasks"] diff --git a/tasks/_apk.py b/tasks/_apk.py new file mode 100644 index 0000000..70d156c --- /dev/null +++ b/tasks/_apk.py @@ -0,0 +1,146 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import gzip +import hashlib +import io +import os +import platform +import shutil +import subprocess +import tarfile +import urllib.request +from pathlib import Path + +from _lib import TaskError, run + +TASK = "publish" +APK_TOOLS_VERSION = "2.14.10-r0" +APK_TOOLS_SHA256 = { + "x86_64": "c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61", + "aarch64": "3e22f80dd0272dc487e4ca84b2c6b660ca392cbad970764efe9ef9555b806ac8", +} + + +def architecture(package: Path) -> str: + metadata = run( + ["tar", "-xOzf", package, ".PKGINFO"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ).stdout + for line in metadata.splitlines(): + if line.startswith("arch = "): + return line.removeprefix("arch = ") + raise TaskError(f"{TASK}: cannot read APK architecture: {package}") + + +def apk_tool(work: Path) -> Path: + configured = os.environ.get("APK_TOOL") + if configured: + tool = Path(configured) + if os.access(tool, os.X_OK): + return tool + raise TaskError(f"{TASK}: APK_TOOL is not executable: {tool}") + for name in ("apk.static", "apk"): + if command := shutil.which(name): + return Path(command) + + machine = platform.machine() + apk_arch = { + "x86_64": "x86_64", + "amd64": "x86_64", + "aarch64": "aarch64", + "arm64": "aarch64", + }.get(machine) + if apk_arch is None: + raise TaskError(f"{TASK}: apk-tools is unavailable for {machine}") + archive = work / "apk-tools-static.apk" + url = ( + "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/" + f"{apk_arch}/apk-tools-static-{APK_TOOLS_VERSION}.apk" + ) + urllib.request.urlretrieve(url, archive) + if hashlib.sha256(archive.read_bytes()).hexdigest() != APK_TOOLS_SHA256[apk_arch]: + raise TaskError(f"{TASK}: apk-tools checksum mismatch") + directory = work / "apk-tools" + directory.mkdir() + run( + ["tar", "-xzf", archive, "-C", directory, "sbin/apk.static"], + stderr=subprocess.DEVNULL, + ) + return directory / "sbin" / "apk.static" + + +def sign_index(context, index: Path) -> None: + name = f".SIGN.RSA256.{context.apk_signing.public_key_name}" + signature = index.parent / name + run( + [ + "openssl", + "dgst", + "-sha256", + "-sign", + context.apk_signing.private_key_file, + "-out", + signature, + index, + ], + env=context.apk_signing.environment, + ) + data = signature.read_bytes() + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode="w", format=tarfile.USTAR_FORMAT) as archive: + information = tarfile.TarInfo(name) + information.size = len(data) + information.mode = 0o644 + information.mtime = int(os.environ.get("SOURCE_DATE_EPOCH", "0")) + archive.addfile(information, io.BytesIO(data)) + size = 512 + ((len(data) + 511) // 512) * 512 + index.write_bytes( + gzip.compress(stream.getvalue()[:size], mtime=0) + index.read_bytes() + ) + signature.unlink() + + +def publish(context) -> None: + tool = apk_tool(context.work) + packages = context.packages["apk"] + architectures = sorted({architecture(package) for package in packages}) + keys = context.work / "apk-keys" + keys.mkdir() + for public_key in context.apk_public_keys: + shutil.copy2(public_key, keys / public_key.name) + + for apk_arch in architectures: + root = context.work / "apk" / apk_arch + root.mkdir(parents=True) + remote = context.storage.service_key("apk", context.channel, apk_arch) + context.storage.download_prefix(remote, root, "*.apk") + for package in packages: + if architecture(package) == apk_arch: + context.add_package(package, root) + for package in root.glob("*.apk"): + if run( + [tool, "verify", "--keys-dir", keys, package], check=False + ).returncode: + raise TaskError( + f"{TASK}: APK signature verification failed: {package.name}" + ) + index = root / "APKINDEX.tar.gz" + run( + [ + tool, + "--allow-untrusted", + "index", + "--description", + f"Dimidium Labs {context.service} {context.channel}", + "--output", + index, + *root.glob("*.apk"), + ] + ) + sign_index(context, index) + context.storage.upload_payloads(root, remote, "*.apk") + context.storage.upload(index, f"{remote}/APKINDEX.tar.gz") diff --git a/tasks/_apt.py b/tasks/_apt.py new file mode 100644 index 0000000..d153d9b --- /dev/null +++ b/tasks/_apt.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import subprocess + +from _lib import TaskError, require_command, run + +TASK = "publish" + + +def publish(context) -> None: + require_command("apt-ftparchive", TASK) + require_command("dpkg-deb", TASK) + root = context.work / "apt" + pool = root / "pool" / context.channel + metadata = root / "dists" / context.channel + pool.mkdir(parents=True) + pool_prefix = context.storage.service_key("apt", "pool", context.channel) + metadata_prefix = context.storage.service_key("apt", "dists", context.channel) + context.storage.download_prefix(pool_prefix, pool, "*.deb") + for package in context.packages["deb"]: + context.add_package(package, pool) + + architectures = sorted( + { + run( + ["dpkg-deb", "-f", package, "Architecture"], + stdout=subprocess.PIPE, + ).stdout.strip() + for package in pool.glob("*.deb") + } + ) + if not architectures or "" in architectures: + raise TaskError(f"{TASK}: no DEB architectures found") + for architecture in architectures: + (metadata / "main" / f"binary-{architecture}").mkdir( + parents=True, exist_ok=True + ) + + cache = context.work / "apt-cache" + cache.mkdir() + config = context.work / "apt-ftparchive.conf" + architecture_list = " ".join(architectures) + config.write_text( + f'''Dir {{ ArchiveDir "{root}"; CacheDir "{cache}"; }}; +Default {{ Packages::Compress ". gzip"; Packages::Extensions ".deb"; }}; +TreeDefault {{ + Packages "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"; + BinCacheDB "packages-$(ARCH).db"; +}}; +Tree "dists/{context.channel}" {{ + Sections "main"; + Architectures "{architecture_list}"; + Directory "pool/{context.channel}"; +}}; +''' + ) + run(["apt-ftparchive", "generate", config]) + release = metadata / "Release" + with release.open("wb") as stream: + run( + [ + "apt-ftparchive", + "-o", + "APT::FTPArchive::Release::Origin=Dimidium Labs", + "-o", + f"APT::FTPArchive::Release::Label={context.service} {context.channel}", + "-o", + f"APT::FTPArchive::Release::Suite={context.channel}", + "-o", + f"APT::FTPArchive::Release::Codename={context.channel}", + "-o", + "APT::FTPArchive::Release::Components=main", + "-o", + f"APT::FTPArchive::Release::Architectures={architecture_list}", + "release", + f"{metadata}/", + ], + stdout=stream, + text=False, + ) + context.gpg.sign(metadata / "Release.gpg", "--armor", "--detach-sign", release) + context.gpg.sign(metadata / "InRelease", "--clearsign", release) + context.storage.upload_payloads(pool, pool_prefix, "*.deb") + context.storage.replace_prefix(metadata, metadata_prefix) diff --git a/tasks/_lib.py b/tasks/_lib.py new file mode 100644 index 0000000..051f71d --- /dev/null +++ b/tasks/_lib.py @@ -0,0 +1,153 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + + +class TaskError(RuntimeError): + pass + + +def require_command(name: str, task: str) -> str: + command = shutil.which(name) + if command is None: + raise TaskError(f"{task}: {name} is required") + return command + + +def required_env(name: str, task: str, purpose: str = "") -> str: + value = os.environ.get(name) + if value: + return value + suffix = f" {purpose}" if purpose else "" + raise TaskError(f"{task}: {name} is required{suffix}") + + +def run(args, **options) -> subprocess.CompletedProcess[str]: + check = options.pop("check", True) + options.setdefault("text", True) + if "input_text" in options: + options["input"] = options.pop("input_text") + return subprocess.run([str(arg) for arg in args], check=check, **options) + + +class GPGSigning: + def __init__(self, task: str, work: Path): + require_command("gpg", task) + private_key = required_env("GPG_PRIVATE_KEY", task) + self.passphrase = required_env("GPG_PASSPHRASE", task) + self.key_id = required_env("GPG_KEY_ID", task) + self.short_key_id = self.key_id[-16:] + self.home = work / "gnupg" + self.home.mkdir(mode=0o700) + self.private_key_file = work / "signing.asc" + self.private_key_file.write_text(private_key) + self.private_key_file.chmod(0o600) + self.environment = dict(os.environ) + self.environment["GNUPGHOME"] = str(self.home) + self.environment.pop("GPG_PRIVATE_KEY", None) + self.environment.pop("GPG_PASSPHRASE", None) + self.environment.pop("APK_PRIVATE_KEY", None) + run( + [ + "gpg", + "--batch", + "--yes", + "--pinentry-mode", + "loopback", + "--passphrase-fd", + "0", + "--import", + self.private_key_file, + ], + env=self.environment, + input_text=f"{self.passphrase}\n", + ) + + def package_environment(self) -> dict[str, str]: + environment = dict(self.environment) + environment["GPG_KEY_ID"] = self.short_key_id + return environment + + def prime_agent(self) -> None: + signature = self.private_key_file.with_suffix(".sig") + self.sign(signature, "--detach-sign", self.private_key_file) + signature.unlink() + + def export_public_key(self, output: Path) -> None: + with output.open("wb") as stream: + run( + ["gpg", "--batch", "--yes", "--armor", "--export", self.key_id], + env=self.environment, + stdout=stream, + text=False, + ) + + def verify_public_bundle(self, bundle: Path) -> None: + fingerprints = { + line.split(":")[9] + for line in run( + ["gpg", "--batch", "--with-colons", "--show-keys", bundle], + env=self.environment, + stdout=subprocess.PIPE, + ).stdout.splitlines() + if line.startswith("fpr:") + } + if self.key_id not in fingerprints: + raise TaskError( + f"{self.task}: packages.gpg does not contain signing key {self.key_id}" + ) + + def sign(self, output: Path, *arguments: str | Path) -> None: + run( + [ + "gpg", + f"--default-key={self.key_id}", + "--batch", + "--yes", + "--pinentry-mode", + "loopback", + "--passphrase-fd", + "0", + "-o", + output, + *arguments, + ], + env=self.environment, + input_text=f"{self.passphrase}\n", + ) + + +class APKSigning: + def __init__(self, task: str, work: Path, key_name: str = "packages"): + require_command("openssl", task) + private_key = required_env("APK_PRIVATE_KEY", task, "for APK signing") + self.key_name = key_name + self.public_key_name = f"{key_name}.rsa.pub" + self.private_key_file = work / f"{key_name}.rsa" + self.private_key_file.write_text(private_key) + self.private_key_file.chmod(0o600) + self.environment = dict(os.environ) + self.environment.pop("APK_PRIVATE_KEY", None) + self.environment.pop("GPG_PRIVATE_KEY", None) + self.environment.pop("GPG_PASSPHRASE", None) + self.environment["APK_SIGNING_KEY"] = str(self.private_key_file) + + def export_public_key(self, output: Path) -> None: + run( + [ + "openssl", + "rsa", + "-in", + self.private_key_file, + "-pubout", + "-out", + output, + ], + env=self.environment, + ) diff --git a/tasks/_repository.py b/tasks/_repository.py new file mode 100644 index 0000000..577b007 --- /dev/null +++ b/tasks/_repository.py @@ -0,0 +1,111 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import re +import shutil +from pathlib import Path + +from _lib import APKSigning, GPGSigning, TaskError, required_env +from _storage import S3Storage + +TASK = "publish" +KEY_VERSION = re.compile(r"^[0-9]{4}$") +RSA_PUBLIC_KEY = re.compile(r"^keys/packages\.[0-9]{4}\.rsa\.pub$") + + +class Repository: + def __init__(self, service, channel, input_directory, formats, work): + self.service = service + self.channel = channel + self.formats = formats + self.work = work + self.public_url = required_env("S3_PUBLIC_URL", TASK).rstrip("/") + self.key_version = required_env("PACKAGE_KEY_VERSION", TASK) + if not KEY_VERSION.fullmatch(self.key_version): + raise TaskError(f"{TASK}: invalid PACKAGE_KEY_VERSION: {self.key_version}") + self.storage = S3Storage(TASK, service) + self.gpg = None + self.gpg_public_key = None + self.apk_signing = None + self.apk_public_keys = [] + self.packages = { + package_format: sorted(input_directory.glob(f"*.{package_format}")) + for package_format in formats + } + for package_format, packages in self.packages.items(): + if not packages: + raise TaskError( + f"{TASK}: no .{package_format} packages found in {input_directory}" + ) + + def check_public_key(self, source: Path, key: str) -> None: + existing = self.work / f"existing-{source.name}" + if not self.storage.download(key, existing): + raise TaskError(f"{TASK}: organization key {key} is not provisioned") + if source.read_bytes() != existing.read_bytes(): + raise TaskError(f"{TASK}: signing key does not match {key}") + + @staticmethod + def add_package(source: Path, directory: Path) -> None: + destination = directory / source.name + if destination.exists() and source.read_bytes() != destination.read_bytes(): + raise TaskError( + f"{TASK}: immutable package filename has different content: {source.name}" + ) + if not destination.exists(): + shutil.copy2(source, destination) + + def setup_openpgp(self) -> None: + self.gpg = GPGSigning(TASK, self.work) + current = self.work / "current-packages.gpg" + self.gpg.export_public_key(current) + self.check_public_key(current, f"keys/packages.{self.key_version}.gpg") + + bundle = self.work / "packages.gpg" + if not self.storage.download("packages.gpg", bundle): + raise TaskError(f"{TASK}: organization key packages.gpg is not provisioned") + self.gpg.verify_public_bundle(bundle) + self.gpg_public_key = bundle + + def setup_rsa(self) -> None: + key_name = f"packages.{self.key_version}" + self.apk_signing = APKSigning(TASK, self.work, key_name) + current = self.work / self.apk_signing.public_key_name + self.apk_signing.export_public_key(current) + self.check_public_key( + current, + f"keys/{self.apk_signing.public_key_name}", + ) + + key_directory = self.work / "rsa-public-keys" + key_directory.mkdir() + for key in sorted(self.storage.objects("keys/")): + if RSA_PUBLIC_KEY.fullmatch(key): + destination = key_directory / Path(key).name + if destination.exists(): + raise TaskError(f"{TASK}: duplicate RSA public key name: {key}") + self.storage.download(key, destination) + self.apk_public_keys.append(destination) + if current.name not in {key.name for key in self.apk_public_keys}: + raise TaskError( + f"{TASK}: current RSA public key is absent from key archive" + ) + + def setup_signing(self) -> None: + if {"deb", "rpm"} & set(self.formats): + self.setup_openpgp() + if "apk" in self.formats: + self.setup_rsa() + + def publish(self) -> None: + from _apk import publish as publish_apk + from _apt import publish as publish_apt + from _rpm import publish as publish_rpm + + publishers = {"deb": publish_apt, "rpm": publish_rpm, "apk": publish_apk} + with self.storage.lock(self.channel): + self.setup_signing() + for package_format in self.formats: + publishers[package_format](self) diff --git a/tasks/_rpm.py b/tasks/_rpm.py new file mode 100644 index 0000000..c2f476f --- /dev/null +++ b/tasks/_rpm.py @@ -0,0 +1,54 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import shutil +import subprocess + +from _lib import TaskError, require_command, run + +TASK = "publish" + + +def publish(context) -> None: + require_command("createrepo_c", TASK) + require_command("rpmkeys", TASK) + root = context.work / "rpm" + root.mkdir() + remote = context.storage.service_key("rpm", context.channel) + context.storage.download_prefix(remote, root, "*.rpm") + for package in context.packages["rpm"]: + context.add_package(package, root) + + rpm_database = context.work / "rpmdb" + rpm_database.mkdir() + run(["rpmkeys", "--dbpath", rpm_database, "--import", context.gpg_public_key]) + for package in root.glob("*.rpm"): + result = run( + ["rpmkeys", "--dbpath", rpm_database, "--checksig", package], + stdout=subprocess.PIPE, + ).stdout + if "signatures OK" not in result: + raise TaskError( + f"{TASK}: RPM is not signed by a trusted key: {package.name}" + ) + + shutil.rmtree(root / "repodata", ignore_errors=True) + run(["createrepo_c", root]) + repomd = root / "repodata" / "repomd.xml" + context.gpg.sign(repomd.with_suffix(".xml.asc"), "--armor", "--detach-sign", repomd) + definition = root / f"{context.service}-{context.channel}.repo" + definition.write_text( + f"""[{context.service}-{context.channel}] +name={context.service} {context.channel} +gpgkey={context.public_url}/packages.gpg +baseurl={context.public_url}/{context.service}/rpm/{context.channel}/ +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +""" + ) + context.storage.upload_payloads(root, remote, "*.rpm") + context.storage.upload(definition, f"{remote}/{definition.name}") + context.storage.replace_prefix(root / "repodata", f"{remote}/repodata") diff --git a/tasks/_storage.py b/tasks/_storage.py new file mode 100644 index 0000000..1fc163e --- /dev/null +++ b/tasks/_storage.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import fnmatch +import hashlib +import json +import time +import uuid +from contextlib import contextmanager +from pathlib import Path + +import boto3 +from _lib import TaskError, required_env +from botocore.exceptions import ClientError + + +class S3Storage: + def __init__(self, task: str, service: str): + self.task = task + self.bucket = required_env("S3_BUCKET", task) + self.service_root = service + self.client = boto3.client( + "s3", + endpoint_url=required_env("S3_ENDPOINT", task), + aws_access_key_id=required_env("S3_ACCESS_KEY_ID", task), + aws_secret_access_key=required_env("S3_SECRET_ACCESS_KEY", task), + region_name="auto", + ) + + def service_key(self, *parts: str) -> str: + return "/".join((self.service_root, *parts)) + + def download(self, key: str, destination: Path) -> bool: + destination.parent.mkdir(parents=True, exist_ok=True) + try: + self.client.download_file(self.bucket, key, str(destination)) + except ClientError as error: + if error.response.get("Error", {}).get("Code") in { + "404", + "NoSuchKey", + "NotFound", + }: + return False + raise + return True + + def objects(self, prefix: str) -> set[str]: + pages = self.client.get_paginator("list_objects_v2").paginate( + Bucket=self.bucket, Prefix=prefix + ) + return {item["Key"] for page in pages for item in page.get("Contents", [])} + + def download_prefix(self, prefix: str, destination: Path, pattern: str) -> None: + destination.mkdir(parents=True, exist_ok=True) + prefix = prefix.rstrip("/") + "/" + for key in self.objects(prefix): + relative = key.removeprefix(prefix).lstrip("/") + if relative and "/" not in relative and fnmatch.fnmatch(relative, pattern): + self.client.download_file(self.bucket, key, str(destination / relative)) + + @staticmethod + def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + @staticmethod + def conflict(error: ClientError) -> bool: + return error.response.get("Error", {}).get("Code") in { + "409", + "412", + "ConditionalRequestConflict", + "PreconditionFailed", + } + + def upload_immutable(self, source: Path, key: str) -> None: + digest = self.digest(source) + try: + with source.open("rb") as stream: + self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=stream, + Metadata={"sha256": digest}, + IfNoneMatch="*", + ) + return + except ClientError as error: + if not self.conflict(error): + raise + + existing = self.client.head_object(Bucket=self.bucket, Key=key) + existing_digest = existing.get("Metadata", {}).get("sha256") + if not existing_digest: + body = self.client.get_object(Bucket=self.bucket, Key=key)["Body"] + existing_digest = hashlib.sha256(body.read()).hexdigest() + if existing_digest != digest: + raise TaskError( + f"{self.task}: immutable object has different content: {key}" + ) + + def upload_payloads(self, source: Path, prefix: str, pattern: str) -> None: + for path in sorted(source.glob(pattern)): + self.upload_immutable(path, f"{prefix.rstrip('/')}/{path.name}") + + def upload(self, source: Path, key: str) -> None: + self.client.upload_file(str(source), self.bucket, key) + + @contextmanager + def lock(self, name: str, lifetime: int = 3600): + key = self.service_key("_locks", name) + body = json.dumps( + {"expires": int(time.time()) + lifetime, "id": uuid.uuid4().hex} + ) + try: + result = self.client.put_object( + Bucket=self.bucket, Key=key, Body=body.encode(), IfNoneMatch="*" + ) + except ClientError as error: + if not self.conflict(error): + raise + current = self.client.get_object(Bucket=self.bucket, Key=key) + state = json.loads(current["Body"].read()) + if state["expires"] > time.time(): + raise TaskError(f"{self.task}: publication already in progress: {name}") + result = self.client.put_object( + Bucket=self.bucket, + Key=key, + Body=body.encode(), + IfMatch=current["ETag"], + ) + try: + yield + finally: + self.client.delete_object( + Bucket=self.bucket, Key=key, IfMatch=result["ETag"] + ) + + def replace_prefix(self, source: Path, prefix: str) -> None: + prefix = prefix.rstrip("/") + "/" + wanted: set[str] = set() + for path in sorted(item for item in source.rglob("*") if item.is_file()): + key = prefix + path.relative_to(source).as_posix() + wanted.add(key) + self.client.upload_file(str(path), self.bucket, key) + stale = sorted(set(self.objects(prefix)) - wanted) + for offset in range(0, len(stale), 1000): + self.client.delete_objects( + Bucket=self.bucket, + Delete={ + "Objects": [{"Key": key} for key in stale[offset : offset + 1000]], + "Quiet": True, + }, + ) diff --git a/tasks/package b/tasks/package index da44865..6db5f86 100755 --- a/tasks/package +++ b/tasks/package @@ -1,134 +1,236 @@ -#!/bin/sh -eu +#!/usr/bin/env -S python3 -B # SPDX-FileCopyrightText: 2026 Nikolay Govorov # SPDX-License-Identifier: 0BSD +# fmt: off #MISE description="Build release archives and signed Linux packages" -#MISE tools={"nfpm"="2.47.0"} - -usage() { - echo "usage: mise run package -- --output DIR [--config FILE] [--version VERSION --arch ARCH] [--apk-public-key FILE] [--archive-root DIR --archive-name NAME] deb|rpm|apk|tar.gz|zip..." >&2 - exit 2 -} - -setup_gpg_signing() { - : "${GPG_KEY_ID:?package: GPG_KEY_ID is required with GPG_PRIVATE_KEY}" - : "${GPG_PASSPHRASE:?package: GPG_PASSPHRASE is required with GPG_PRIVATE_KEY}" - - export GNUPGHOME="$work/gnupg" - mkdir -m 700 "$GNUPGHOME" - - SIGNING_PRIVATE_KEY="$work/signing.asc" - printf '%s' "$GPG_PRIVATE_KEY" > "$SIGNING_PRIVATE_KEY" - - chmod 600 "$SIGNING_PRIVATE_KEY" - printf '%s\n' "$GPG_PASSPHRASE" | gpg --batch --yes \ - --pinentry-mode loopback --passphrase-fd 0 \ - --import "$SIGNING_PRIVATE_KEY" - - NFPM_PASSPHRASE=$GPG_PASSPHRASE - GPG_KEY_ID=$(printf '%s' "$GPG_KEY_ID" | sed 's/.*\(.\{16\}\)$/\1/') - export SIGNING_PRIVATE_KEY NFPM_PASSPHRASE GPG_KEY_ID - unset GPG_PRIVATE_KEY GPG_PASSPHRASE -} - -config=nfpm.yaml -version= -arch= -output= -archive_root= -archive_name= -apk_public_key= -formats= -system_formats= -archive_formats= - -while [ "$#" -gt 0 ]; do - case "$1" in - --config) [ "$#" -ge 2 ] || usage; config=$2; shift 2 ;; - --version) [ "$#" -ge 2 ] || usage; version=$2; shift 2 ;; - --arch) [ "$#" -ge 2 ] || usage; arch=$2; shift 2 ;; - --output) [ "$#" -ge 2 ] || usage; output=$2; shift 2 ;; - --archive-root) [ "$#" -ge 2 ] || usage; archive_root=$2; shift 2 ;; - --archive-name) [ "$#" -ge 2 ] || usage; archive_name=$2; shift 2 ;; - --apk-public-key) [ "$#" -ge 2 ] || usage; apk_public_key=$2; shift 2 ;; - deb|rpm|apk) - formats="$formats $1" - system_formats="$system_formats $1" - shift - ;; - tar.gz|zip) - formats="$formats $1" - archive_formats="$archive_formats $1" - shift - ;; - *) usage ;; - esac -done - -[ -n "$output" ] && [ -n "$formats" ] || usage - -if [ -n "$system_formats" ]; then - [ -n "$version" ] && [ -n "$arch" ] || usage - [ -f "$config" ] || { echo "package: $config not found" >&2; exit 1; } -fi - -if [ -n "$archive_formats" ]; then - [ -n "$archive_root" ] && [ -n "$archive_name" ] || usage - [ -d "$archive_root" ] || { echo "package: $archive_root not found" >&2; exit 1; } - case "$archive_name" in - *[!A-Za-z0-9._-]*|.|..) echo "package: invalid archive name: $archive_name" >&2; exit 1 ;; - esac -fi - -case "$apk_public_key" in - *[!A-Za-z0-9._-]*|.|..) echo "package: invalid APK public key name: $apk_public_key" >&2; exit 1 ;; -esac - -mkdir -p "$output" -output=$(cd "$output" && pwd) - -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM - -case " $system_formats " in - *" deb "*|*" rpm "*) - if [ -n "${GPG_PRIVATE_KEY:-}" ]; then - setup_gpg_signing - fi - ;; -esac - -case " $system_formats " in - *" apk "*) - if [ -n "${APK_PRIVATE_KEY:-}" ]; then - APK_SIGNING_KEY="$work/signing.rsa" - printf '%s' "$APK_PRIVATE_KEY" > "$APK_SIGNING_KEY" - chmod 600 "$APK_SIGNING_KEY" - export APK_SIGNING_KEY - unset APK_PRIVATE_KEY - fi - if [ -n "${APK_SIGNING_KEY:-}" ]; then - [ -n "$apk_public_key" ] || { - echo "package: --apk-public-key is required for signed APK packages" >&2 - exit 1 - } - openssl rsa -in "$APK_SIGNING_KEY" -pubout -out "$output/$apk_public_key" - fi - ;; -esac - -for format in $formats; do - case "$format" in - deb|rpm|apk) - ARCH=$arch VERSION=$version - export ARCH VERSION - nfpm package --config "$config" --packager "$format" --target "$output/" - ;; - tar.gz) - tar -czf "$output/$archive_name.tar.gz" -C "$archive_root" . - ;; - zip) - rm -f "$output/$archive_name.zip" - (cd "$archive_root" && zip -qry "$output/$archive_name.zip" .) - ;; - esac -done +#MISE tools={"python"="3.14.7","nfpm"="2.47.0"} +# fmt: on + +from __future__ import annotations + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +from _lib import APKSigning, GPGSigning, TaskError, require_command, run + +TASK = "package" +SYSTEM_FORMATS = {"deb", "rpm", "apk"} +ARCHIVE_FORMATS = {"tar.gz", "zip"} +SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +SAFE_KEY_VERSION = re.compile(r"^[0-9]{4}$") + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser( + prog="mise run package --", + usage=( + "%(prog)s --output DIR [--config FILE] " + "[--version VERSION --arch ARCH] [--apk-public-key FILE] " + "[--archive-root DIR --archive-name NAME] " + "deb|rpm|apk|tar.gz|zip..." + ), + ) + result.add_argument("--config", default="nfpm.yaml", type=Path) + result.add_argument("--version") + result.add_argument("--arch") + result.add_argument("--output", required=True, type=Path) + result.add_argument("--archive-root", type=Path) + result.add_argument("--archive-name") + result.add_argument("--apk-public-key") + result.add_argument( + "formats", nargs="+", choices=sorted(SYSTEM_FORMATS | ARCHIVE_FORMATS) + ) + return result + + +def validate(arguments: argparse.Namespace, command: argparse.ArgumentParser) -> None: + formats = set(arguments.formats) + if formats & SYSTEM_FORMATS: + if not arguments.version or not arguments.arch: + command.error("--version and --arch are required for deb, rpm, and apk") + if not arguments.config.is_file(): + raise TaskError(f"{TASK}: {arguments.config} not found") + if formats & ARCHIVE_FORMATS: + if arguments.archive_root is None or not arguments.archive_name: + command.error( + "--archive-root and --archive-name are required for tar.gz and zip" + ) + if not arguments.archive_root.is_dir(): + raise TaskError(f"{TASK}: {arguments.archive_root} not found") + if not SAFE_NAME.fullmatch(arguments.archive_name): + raise TaskError(f"{TASK}: invalid archive name: {arguments.archive_name}") + if arguments.apk_public_key and not SAFE_NAME.fullmatch(arguments.apk_public_key): + raise TaskError( + f"{TASK}: invalid APK public key name: {arguments.apk_public_key}" + ) + + +def create_archive( + archive_format: str, + output: Path, + root: Path, + name: str, +) -> None: + if archive_format == "tar.gz": + require_command("tar", TASK) + run(["tar", "-czf", output / f"{name}.tar.gz", "-C", root, "."]) + return + require_command("zip", TASK) + destination = output / f"{name}.zip" + destination.unlink(missing_ok=True) + run(["zip", "-qry", destination, "."], cwd=root) + + +def sign_package( + package_format: str, package: Path, signing: GPGSigning | None +) -> None: + if signing is None or package_format not in {"deb", "rpm"}: + return + if package_format == "deb": + require_command("debsigs", TASK) + run( + [ + "debsigs", + "--sign=origin", + f"--default-key={signing.key_id}", + package, + ], + env=signing.environment, + ) + else: + require_command("rpmsign", TASK) + run( + [ + "rpmsign", + "--define", + f"_gpg_name {signing.key_id}", + "--addsign", + package, + ], + env=signing.environment, + ) + + +def main() -> None: + command = parser() + arguments = command.parse_args() + validate(arguments, command) + + arguments.output.mkdir(parents=True, exist_ok=True) + output = arguments.output.resolve() + formats = set(arguments.formats) + + with tempfile.TemporaryDirectory(prefix="package-") as directory: + work = Path(directory) + environment = dict(os.environ) + for name in ( + "GPG_PRIVATE_KEY", + "APK_PRIVATE_KEY", + "SIGNING_PRIVATE_KEY", + "NFPM_PASSPHRASE", + "NFPM_DEB_PASSPHRASE", + "NFPM_RPM_PASSPHRASE", + ): + environment.pop(name, None) + config = arguments.config + if "apk" in formats and "${PACKAGE_KEY_VERSION}" in config.read_text(): + key_version = os.environ.get("PACKAGE_KEY_VERSION", "") + if not SAFE_KEY_VERSION.fullmatch(key_version): + raise TaskError( + f"{TASK}: invalid PACKAGE_KEY_VERSION: {key_version or ''}" + ) + config = work / "nfpm.yaml" + config.write_text( + arguments.config.read_text().replace( + "${PACKAGE_KEY_VERSION}", key_version + ) + ) + + gpg_signing: GPGSigning | None = None + if formats & {"deb", "rpm"} and os.environ.get("GPG_PRIVATE_KEY"): + gpg_signing = GPGSigning(TASK, work) + gpg_signing.prime_agent() + environment = gpg_signing.package_environment() + + apk_signing: APKSigning | None = None + if "apk" in formats and os.environ.get("APK_PRIVATE_KEY"): + apk_signing = APKSigning(TASK, work) + environment["APK_SIGNING_KEY"] = str(apk_signing.private_key_file) + elif os.environ.get("APK_SIGNING_KEY"): + environment["APK_SIGNING_KEY"] = os.environ["APK_SIGNING_KEY"] + + if "apk" in formats and arguments.apk_public_key: + public_key = output / arguments.apk_public_key + if apk_signing is not None: + apk_signing.export_public_key(public_key) + elif environment.get("APK_SIGNING_KEY"): + require_command("openssl", TASK) + run( + [ + "openssl", + "rsa", + "-in", + environment["APK_SIGNING_KEY"], + "-pubout", + "-out", + public_key, + ], + env=environment, + ) + + for package_format in arguments.formats: + if package_format in SYSTEM_FORMATS: + require_command("nfpm", TASK) + package_environment = dict(environment) + package_environment.update( + ARCH=arguments.arch, + VERSION=arguments.version, + ) + package_output = work / f"output-{package_format}" + package_output.mkdir() + run( + [ + "nfpm", + "package", + "--config", + config, + "--packager", + package_format, + "--target", + f"{package_output}/", + ], + env=package_environment, + ) + packages = list(package_output.iterdir()) + if len(packages) != 1 or not packages[0].is_file(): + raise TaskError( + f"{TASK}: nFPM produced an unexpected number of packages" + ) + sign_package(package_format, packages[0], gpg_signing) + shutil.move(packages[0], output / packages[0].name) + else: + create_archive( + package_format, + output, + arguments.archive_root, + arguments.archive_name, + ) + + +if __name__ == "__main__": + try: + main() + except TaskError as error: + print(error, file=sys.stderr) + raise SystemExit(1) from None + except subprocess.CalledProcessError as error: + print( + f"{TASK}: command failed with exit code {error.returncode}", file=sys.stderr + ) + raise SystemExit(error.returncode) from None diff --git a/tasks/publish b/tasks/publish new file mode 100755 index 0000000..74cf2db --- /dev/null +++ b/tasks/publish @@ -0,0 +1,76 @@ +#!/usr/bin/env -S uv run --script --python 3.14.7 +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Publish signed package repositories to shared S3 storage" +#MISE tools={"uv"="0.12.5"} +# fmt: on +# /// script +# requires-python = ">=3.14" +# dependencies = ["boto3==1.43.75"] +# /// + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.dont_write_bytecode = True + +from _lib import TaskError +from _repository import Repository + +TASK = "publish" +FORMATS = {"deb", "rpm", "apk"} +SAFE_SLUG = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser( + prog="mise run publish --", + usage=("%(prog)s --service NAME --channel CHANNEL --input DIR deb|rpm|apk..."), + ) + result.add_argument("--service", required=True) + result.add_argument("--channel", required=True) + result.add_argument("--input", required=True, type=Path) + result.add_argument("formats", nargs="+", choices=sorted(FORMATS)) + return result + + +def main() -> None: + command = parser() + arguments = command.parse_args() + for label, value in ( + ("service name", arguments.service), + ("channel", arguments.channel), + ): + if not SAFE_SLUG.fullmatch(value) or "--" in value: + raise TaskError(f"{TASK}: invalid {label}: {value}") + if not arguments.input.is_dir(): + raise TaskError(f"{TASK}: {arguments.input} not found") + with tempfile.TemporaryDirectory(prefix="publish-") as directory: + Repository( + arguments.service, + arguments.channel, + arguments.input.resolve(), + arguments.formats, + Path(directory), + ).publish() + + +if __name__ == "__main__": + try: + main() + except TaskError as error: + print(error, file=sys.stderr) + raise SystemExit(1) from None + except subprocess.CalledProcessError as error: + print( + f"{TASK}: command failed with exit code {error.returncode}", + file=sys.stderr, + ) + raise SystemExit(error.returncode) from None diff --git a/tests/fakes/boto3.py b/tests/fakes/boto3.py new file mode 100644 index 0000000..8ddbf86 --- /dev/null +++ b/tests/fakes/boto3.py @@ -0,0 +1,127 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +from __future__ import annotations + +import hashlib +import io +import os +import shutil +from pathlib import Path + +from botocore.exceptions import ClientError + + +def error(code: str, operation: str) -> ClientError: + return ClientError({"Error": {"Code": code, "Message": code}}, operation) + + +class Paginator: + def __init__(self, client): + self.client = client + + def paginate(self, *, Bucket: str, Prefix: str): + root = self.client.root / Bucket + contents = [] + if root.is_dir(): + for path in root.rglob("*"): + if path.is_file(): + key = path.relative_to(root).as_posix() + if key.startswith(Prefix): + contents.append({"Key": key, "Size": path.stat().st_size}) + yield {"Contents": contents} + + +class Client: + def __init__(self): + self.root = Path(os.environ["PUBLISH_REMOTE"]) + self.metadata: dict[tuple[str, str], dict[str, str]] = {} + self.log = os.environ.get("PUBLISH_TEST_LOG") + + def record(self, operation: str, key: str) -> None: + if self.log: + with Path(self.log).open("a") as stream: + stream.write(f"s3 {operation} {key}\n") + + def path(self, bucket: str, key: str) -> Path: + path = self.root / bucket / key + path.resolve().relative_to(self.root.resolve()) + return path + + def get_paginator(self, name: str): + assert name == "list_objects_v2" + return Paginator(self) + + def download_file(self, bucket: str, key: str, destination: str) -> None: + source = self.path(bucket, key) + if not source.is_file(): + raise error("NoSuchKey", "DownloadFile") + Path(destination).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + self.record("download", key) + + def upload_file(self, source: str, bucket: str, key: str) -> None: + destination = self.path(bucket, key) + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, destination) + self.record("upload", key) + + def put_object( + self, + *, + Bucket, + Key, + Body, + Metadata=None, + IfNoneMatch=None, + IfMatch=None, + ): + destination = self.path(Bucket, Key) + if IfNoneMatch == "*" and destination.exists(): + raise error("PreconditionFailed", "PutObject") + if IfMatch and self.etag(destination) != IfMatch: + raise error("PreconditionFailed", "PutObject") + destination.parent.mkdir(parents=True, exist_ok=True) + data = Body.read() if hasattr(Body, "read") else Body + destination.write_bytes(data) + self.metadata[(Bucket, Key)] = Metadata or {} + self.record("immutable", Key) + return {"ETag": self.etag(destination)} + + def head_object(self, *, Bucket, Key): + path = self.path(Bucket, Key) + if not path.exists(): + raise error("NoSuchKey", "HeadObject") + return {"Metadata": self.metadata.get((Bucket, Key), {})} + + @staticmethod + def etag(path: Path) -> str | None: + if not path.exists(): + return None + return f'"{hashlib.md5(path.read_bytes(), usedforsecurity=False).hexdigest()}"' + + def get_object(self, *, Bucket, Key): + path = self.path(Bucket, Key) + return {"Body": io.BytesIO(path.read_bytes()), "ETag": self.etag(path)} + + def delete_object(self, *, Bucket, Key, IfMatch): + path = self.path(Bucket, Key) + if self.etag(path) != IfMatch: + raise error("PreconditionFailed", "DeleteObject") + path.unlink() + self.record("delete", Key) + return {} + + def delete_objects(self, *, Bucket, Delete): + for item in Delete["Objects"]: + self.path(Bucket, item["Key"]).unlink(missing_ok=True) + self.record("delete", item["Key"]) + return {} + + +def client(name: str, **kwargs): + assert name == "s3" + assert kwargs["endpoint_url"] + assert kwargs["aws_access_key_id"] + assert kwargs["aws_secret_access_key"] + return Client() diff --git a/tests/package b/tests/package index dca898d..6aa3fc0 100755 --- a/tests/package +++ b/tests/package @@ -17,19 +17,24 @@ ln -s "$work/outside" "$work/stage/outside-link" cat > "$work/bin/nfpm" <<'EOF' #!/bin/sh set -eu -[ -z "${GPG_PRIVATE_KEY:-}" ] && [ -z "${APK_PRIVATE_KEY:-}" ] || { +[ -z "${GPG_PRIVATE_KEY:-}" ] && \ +[ -z "${APK_PRIVATE_KEY:-}" ] && \ +[ -z "${SIGNING_PRIVATE_KEY:-}" ] && \ +[ -z "${NFPM_PASSPHRASE:-}" ] || { echo 'raw private key leaked to nFPM' >&2 exit 1 } printf 'ARCH=%s\nVERSION=%s\nGPG_KEY_ID=%s\n' "$ARCH" "$VERSION" "${GPG_KEY_ID:-}" >> "$PACKAGE_TEST_LOG" while [ "$#" -gt 0 ]; do case "$1" in + --config) config=$2; shift 2 ;; --packager) packager=$2; shift 2 ;; --target) target=$2; shift 2 ;; *) shift ;; esac done printf '%s\n' "$packager" >> "$PACKAGE_TEST_LOG" +grep '^key_name:' "$config" >> "$PACKAGE_TEST_LOG" || true : > "$target/test.$packager" EOF @@ -37,8 +42,24 @@ cat > "$work/bin/gpg" <<'EOF' #!/bin/sh set -eu printf 'gpg\n' >> "$PACKAGE_TEST_LOG" +while [ "$#" -gt 0 ]; do + case "$1" in + -o) output=$2; shift 2 ;; + *) shift ;; + esac +done +[ -z "${output:-}" ] || printf signature > "$output" EOF +for command in debsigs rpmsign; do + cat > "$work/bin/$command" <<'EOF' +#!/bin/sh +set -eu +printf '%s\n' "${0##*/}" >> "$PACKAGE_TEST_LOG" +EOF + chmod +x "$work/bin/$command" +done + cat > "$work/bin/openssl" <<'EOF' #!/bin/sh set -eu @@ -91,13 +112,45 @@ APK_PRIVATE_KEY=apk-private \ --arch amd64 \ --output "$work/out" \ --apk-public-key test.rsa.pub \ - deb apk + deb rpm apk test -f "$work/out/test.apk" test -f "$work/out/test.rsa.pub" grep -q '^gpg$' "$work/package.log" +grep -q '^debsigs$' "$work/package.log" +grep -q '^rpmsign$' "$work/package.log" grep -q '^GPG_KEY_ID=89ABCDEF01234567$' "$work/package.log" +# Repository publication exports the organization APK public key, so package +# builds do not need to stage a per-service public key artifact. +APK_PRIVATE_KEY=apk-private \ + "$root/tasks/package" \ + --config "$work/nfpm.yaml" \ + --version 1.2.3 \ + --arch amd64 \ + --output "$work/out" \ + apk + +printf 'key_name: packages.${PACKAGE_KEY_VERSION}\n' > "$work/versioned.yaml" +if APK_PRIVATE_KEY=apk-private "$root/tasks/package" \ + --config "$work/versioned.yaml" \ + --version 1.2.3 \ + --arch amd64 \ + --output "$work/out" \ + apk 2>/dev/null; then + echo 'package test: accepted versioned APK config without key version' >&2 + exit 1 +fi +APK_PRIVATE_KEY=apk-private \ +PACKAGE_KEY_VERSION=0001 \ + "$root/tasks/package" \ + --config "$work/versioned.yaml" \ + --version 1.2.3 \ + --arch amd64 \ + --output "$work/out" \ + apk +grep -q '^key_name: packages.0001$' "$work/package.log" + if "$root/tasks/package" --output "$work/out" deb 2>/dev/null; then echo 'package test: accepted system format without version and architecture' >&2 exit 1 diff --git a/tests/publish-integration b/tests/publish-integration new file mode 100755 index 0000000..5081943 --- /dev/null +++ b/tests/publish-integration @@ -0,0 +1,185 @@ +#!/bin/sh +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD + +set -eu + +root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT HUP INT TERM +remote="$work/remote/integration" +mkdir -p "$work/bin" "$work/input" "$remote" "$work/fixture" + +printf 'publish integration fixture\n' > "$work/fixture/publish-contract" +cat > "$work/fixture/nfpm.yaml" < +description: Shared publish task integration fixture +license: 0BSD +contents: + - src: $work/fixture/publish-contract + dst: /usr/local/bin/publish-contract +deb: + signature: + method: debsign + key_id: \${GPG_KEY_ID} + key_file: \${SIGNING_PRIVATE_KEY} +rpm: + signature: + key_id: \${GPG_KEY_ID} + key_file: \${SIGNING_PRIVATE_KEY} +apk: + signature: + key_file: \${APK_SIGNING_KEY} + key_name: packages.\${PACKAGE_KEY_VERSION} +EOF + +export GNUPGHOME="$work/source-gnupg" +mkdir -m 700 "$GNUPGHOME" +gpg --batch --pinentry-mode loopback --passphrase integration-pass \ + --quick-generate-key 'Publish Integration ' rsa2048 sign 1d >/dev/null +fingerprint=$(gpg --batch --with-colons --list-secret-keys | + awk -F: '$1 == "fpr" { print $10; exit }') +gpg --batch --pinentry-mode loopback --passphrase integration-pass \ + --armor --export-secret-keys "$fingerprint" > "$work/private.gpg" +openssl genrsa -out "$work/private.rsa" 2048 >/dev/null 2>&1 +# Organization keys are provisioned independently under a versioned archive. +key_version=0001 +mkdir -p "$remote/keys" +gpg --batch --armor --export "$fingerprint" > "$remote/packages.gpg" +cp "$remote/packages.gpg" "$remote/keys/packages.$key_version.gpg" +openssl rsa -in "$work/private.rsa" -pubout \ + -out "$remote/keys/packages.$key_version.rsa.pub" >/dev/null 2>&1 + +gpg_private_key=$(cat "$work/private.gpg") +apk_private_key=$(cat "$work/private.rsa") + +package_version() { + version=$1 + GPG_PRIVATE_KEY="$gpg_private_key" \ + GPG_PASSPHRASE=integration-pass \ + GPG_KEY_ID="$fingerprint" \ + APK_PRIVATE_KEY="$apk_private_key" \ + PACKAGE_KEY_VERSION="$key_version" \ + mise --cd "$root" run package -- \ + --config "$work/fixture/nfpm.yaml" \ + --version "$version" \ + --arch amd64 \ + --output "$work/input" \ + deb rpm apk +} + +publish() { + PUBLISH_REMOTE="$work/remote" \ + PUBLISH_TEST_LOG="$work/s3.log" \ + PYTHONPATH="$root/tests/fakes" \ + PATH="$work/bin:$PATH" \ + S3_BUCKET=integration \ + S3_ENDPOINT=https://example.invalid \ + S3_PUBLIC_URL=https://pkg.dimidiumlabs.io \ + S3_ACCESS_KEY_ID=integration \ + S3_SECRET_ACCESS_KEY=integration \ + GPG_PRIVATE_KEY="$gpg_private_key" \ + GPG_PASSPHRASE=integration-pass \ + GPG_KEY_ID="$fingerprint" \ + APK_PRIVATE_KEY="$apk_private_key" \ + PACKAGE_KEY_VERSION="$key_version" \ + "$root/tasks/publish" \ + --service publish-contract \ + --channel nightly \ + --input "$work/input" \ + deb rpm apk +} + +package_version '1.2.3~nightly.42' +publish + +apt_root="$remote/publish-contract/apt" +rpm_root="$remote/publish-contract/rpm/nightly" +apk_root="$remote/publish-contract/apk/nightly/x86_64" + +test -f "$remote/packages.gpg" +test -f "$remote/keys/packages.$key_version.gpg" +test -f "$remote/keys/packages.$key_version.rsa.pub" +test "$(find "$apt_root/pool/nightly" -name '*.deb' | wc -l)" -eq 1 +test "$(find "$rpm_root" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 1 +test "$(find "$apk_root" -maxdepth 1 -name '*.apk' | wc -l)" -eq 1 +grep -q '^Package: publish-contract$' \ + "$apt_root/dists/nightly/main/binary-amd64/Packages" +grep -q 'baseurl=https://pkg.dimidiumlabs.io/publish-contract/rpm/nightly/' \ + "$rpm_root/publish-contract-nightly.repo" + +keyring="$work/packages.gpg" +gpg --batch --dearmor -o "$keyring" "$remote/packages.gpg" +gpgv --keyring "$keyring" "$apt_root/dists/nightly/InRelease" >/dev/null +gpgv --keyring "$keyring" "$rpm_root/repodata/repomd.xml.asc" \ + "$rpm_root/repodata/repomd.xml" >/dev/null + +apk_tool=$(find "$HOME/.cache/mise" "$HOME/.local/share/mise" -type f -name apk.static 2>/dev/null | head -n 1 || true) +if [ -z "$apk_tool" ]; then + archive="$work/apk-tools-static.apk" + curl -fsSL \ + https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/apk-tools-static-2.14.10-r0.apk \ + -o "$archive" + printf '%s %s\n' c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61 "$archive" | + sha256sum -c - >/dev/null + mkdir -p "$work/apk-tools" + tar -xzf "$archive" -C "$work/apk-tools" sbin/apk.static 2>/dev/null + apk_tool="$work/apk-tools/sbin/apk.static" +fi +mkdir "$work/apk-keys" +cp "$remote/keys/packages.$key_version.rsa.pub" "$work/apk-keys/" +"$apk_tool" verify --keys-dir "$work/apk-keys" "$apk_root/APKINDEX.tar.gz" >/dev/null + +# A later publication keeps every package payload, regenerates metadata, and +# removes stale metadata only inside the corresponding repository prefixes. +printf stale > "$apt_root/dists/nightly/stale" +printf stale > "$rpm_root/repodata/stale" +rm -rf "$work/input" +mkdir "$work/input" +package_version '1.2.3~nightly.43' +publish +test "$(find "$apt_root/pool/nightly" -name '*.deb' | wc -l)" -eq 2 +test "$(find "$rpm_root" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 2 +test "$(find "$apk_root" -maxdepth 1 -name '*.apk' | wc -l)" -eq 2 +test ! -e "$apt_root/dists/nightly/stale" +test ! -e "$rpm_root/repodata/stale" +test "$(grep -c '^Package: publish-contract$' \ + "$apt_root/dists/nightly/main/binary-amd64/Packages")" -eq 2 +"$apk_tool" verify --keys-dir "$work/apk-keys" "$apk_root/APKINDEX.tar.gz" >/dev/null + +# Existing package names and organization keys are immutable. +deb=$(find "$work/input" -maxdepth 1 -name '*.deb') +printf '\nchanged\n' >> "$deb" +if publish 2>/dev/null; then + echo 'publish integration: replaced an immutable package payload' >&2 + exit 1 +fi +cp "$apt_root/pool/nightly/${deb##*/}" "$deb" +lock="$remote/publish-contract/_locks/nightly" +mkdir -p "${lock%/*}" +printf '{"expires":9999999999}\n' > "$lock" +if publish 2>/dev/null; then + echo 'publish integration: ignored an active publication lock' >&2 + exit 1 +fi +rm "$lock" +printf 'different key\n' > "$remote/keys/packages.$key_version.gpg" +if publish 2>/dev/null; then + echo 'publish integration: ignored a mismatched versioned key' >&2 + exit 1 +fi + +# Metadata cleanup must never escape generated repository prefixes. +if awk '$1 == "s3" && $2 == "delete" && $3 !~ /^publish-contract\/(apt\/dists|rpm\/nightly\/repodata|_locks\/nightly)/' "$work/s3.log" | grep . >/dev/null; then + echo 'publish integration: deleted outside a metadata prefix' >&2 + exit 1 +fi +if PYTHONPATH="$root/tests/fakes" "$root/tasks/publish" \ + --service '../escape' --channel nightly --input "$work/input" deb 2>/dev/null; then + echo 'publish integration: accepted an unsafe service name' >&2 + exit 1 +fi -- Gilti