diff options
Diffstat
| -rw-r--r-- | .github/workflows/ci.yml | 15 | +6 −9 |
| -rw-r--r-- | README.md | 18 | +10 −8 |
| -rw-r--r-- | mise.lock | 80 | +34 −46 |
| -rw-r--r-- | mise.toml | 2 | +1 −1 |
| -rwxr-xr-x | tasks/chart | 111 | +0 −111 |
| -rwxr-xr-x | tasks/chart.py | 87 | +87 −0 |
| -rwxr-xr-x | tasks/container | 173 | +0 −173 |
| -rwxr-xr-x | tasks/container.py | 105 | +105 −0 |
| -rw-r--r-- | tasks/libs/__init__.py | 2 | +2 −0 |
| -rw-r--r-- | tasks/libs/apk.py (renamed from tasks/_apk.py) | 88 | +43 −45 |
| -rw-r--r-- | tasks/libs/apt.py (renamed from tasks/_apt.py) | 58 | +25 −33 |
| -rw-r--r-- | tasks/libs/common.py (renamed from tasks/_lib.py) | 123 | +74 −49 |
| -rw-r--r-- | tasks/libs/repository.py (renamed from tasks/_repository.py) | 34 | +17 −17 |
| -rw-r--r-- | tasks/libs/rpm.py (renamed from tasks/_rpm.py) | 20 | +10 −10 |
| -rw-r--r-- | tasks/libs/storage.py (renamed from tasks/_storage.py) | 3 | +2 −1 |
| -rwxr-xr-x | tasks/licenses | 50 | +0 −50 |
| -rwxr-xr-x | tasks/licenses.py | 95 | +95 −0 |
| -rwxr-xr-x | tasks/package.py (renamed from tasks/package) | 166 | +77 −89 |
| -rwxr-xr-x | tasks/publish.py (renamed from tasks/publish) | 54 | +22 −32 |
| -rwxr-xr-x | tasks/signoff | 155 | +0 −155 |
| -rwxr-xr-x | tasks/signoff.py | 208 | +208 −0 |
| -rwxr-xr-x | tests/oci-tasks | 87 | +0 −87 |
| -rwxr-xr-x | tests/oci-tasks.py | 147 | +147 −0 |
| -rwxr-xr-x | tests/package | 177 | +0 −177 |
| -rwxr-xr-x | tests/package-integration | 42 | +0 −42 |
| -rwxr-xr-x | tests/package-integration.py | 93 | +93 −0 |
| -rwxr-xr-x | tests/package.py | 270 | +270 −0 |
| -rwxr-xr-x | tests/publish-integration | 185 | +0 −185 |
| -rwxr-xr-x | tests/publish-integration.py | 346 | +346 −0 |
29 files changed, 1674 insertions, 1320 deletions
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5835033..4783219 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,17 +30,14 @@ jobs: - name: Check Python run: | - ruff check tasks/_*.py tasks/package tasks/publish tests/fakes/boto3.py - ruff format --check tasks/_*.py tasks/package tasks/publish tests/fakes/boto3.py - - - name: Check shell - run: sh -n tasks/chart tasks/container tests/oci-tasks + ruff check tasks tests + ruff format --check tasks tests - name: Tests run: | - tests/oci-tasks + tests/oci-tasks.py - tests/package - tests/package-integration + tests/package.py + tests/package-integration.py - tests/publish-integration + tests/publish-integration.py diff --git a/README.md b/README.md index bc1517a..057d394 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,14 @@ Consuming projects pin this repository by commit SHA. ## Packaging Projects build and stage their own binaries and keep their nFPM configuration. -The shared [`package`](tasks/package) task creates only the formats explicitly +The shared [`package`](tasks/package.py) task creates only the formats explicitly requested by a project: nFPM packages (`deb`, `rpm`, or `apk`) and portable archives (`tar.gz` or `zip`). APK configurations may use `${PACKAGE_KEY_VERSION}` in `apk.signature.key_name`; the task renders the four-digit generation before invoking nFPM. DEB and RPM payloads are built by nFPM and then signed through `debsigs` and `rpmsign`, allowing CI to use only an OpenPGP signing subkey while the certification key remains offline. The shared -[`publish`](tasks/publish) task adds +[`publish`](tasks/publish.py) task adds explicitly selected package formats to signed repositories in the organization package bucket. @@ -43,10 +43,10 @@ mise run package -- \ ## OCI artifacts -The shared [`container`](tasks/container) task builds one or more tagged OCI +The shared [`container`](tasks/container.py) task builds one or more tagged OCI images with Docker Buildx. Registry authentication is deliberately left to the calling workflow, so the same build can be pushed to GHCR, Cloudflare, or -another OCI registry. The [`chart`](tasks/chart) task strictly lints a Helm +another OCI registry. The [`chart`](tasks/chart.py) task strictly lints a Helm chart, packages an immutable version, and can push it to one or more OCI repositories. @@ -110,20 +110,22 @@ mise bootstrap ``` Shared tasks declare task-specific tools in their `#MISE tools` metadata, so -`mise run` installs the same pinned versions on demand. System libraries that -cannot be installed as portable tools belong in `[bootstrap.packages]`. +`mise run` installs the same pinned versions on demand. Python tasks declare +their dependencies inline with PEP 723 and run through pipx's standard pip +backend, without a project virtual environment or install step. System libraries +that cannot be installed as portable tools belong in `[bootstrap.packages]`. ## Guardrails ### Licensing policy -`tasks/licenses` runs a pinned REUSE version and verifies the repository's +`tasks/licenses.py` runs a pinned REUSE version and verifies the repository's licensing metadata and canonical SPDX copyright headers. In Rust projects it also runs a pinned `cargo deny check`. ### Sign-off policy -`tasks/signoff` verifies that: +`tasks/signoff.py` verifies that: - authors and co-authors with an email from `config/signoff-approved-emails` are trusted without a trailer; diff --git a/mise.lock b/mise.lock index 04ef64a..9305d16 100644 --- a/mise.lock +++ b/mise.lock @@ -1,5 +1,39 @@ # @generated - this file is auto-generated by `mise lock` https://mise.jdx.dev/dev-tools/mise-lock.html +[[tools.pipx]] +version = "1.16.7" +backend = "aqua:pypa/pipx" + +[tools.pipx."platforms.linux-arm64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.linux-arm64-musl"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.linux-x64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.linux-x64-musl"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.macos-arm64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + +[tools.pipx."platforms.macos-x64"] +checksum = "sha256:302633d0061e0ab4269257501cbe1338cde5f00f22f121543706624aada0145e" +url = "https://github.com/pypa/pipx/releases/download/1.16.7/pipx.pyz" +url_api = "https://api.github.com/repos/pypa/pipx/releases/assets/513588802" + [[tools.ruff]] version = "0.16.3" backend = "aqua:astral-sh/ruff" @@ -45,49 +79,3 @@ checksum = "sha256:f10c709755b393fd9821506b21070bcca969b9966504edd1e490efd08e366 url = "https://github.com/astral-sh/ruff/releases/download/0.16.3/ruff-x86_64-pc-windows-msvc.zip" url_api = "https://api.github.com/repos/astral-sh/ruff/releases/assets/513140346" provenance = "github-attestations" - -[[tools.uv]] -version = "0.12.5" -backend = "aqua:astral-sh/uv" - -[tools.uv."platforms.linux-arm64"] -checksum = "sha256:8767a0e77f2cd45436401b1b42bf7e9ed5a4a91a74a5305d6fe93249d0f6dbc5" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850993" -provenance = "github-attestations" - -[tools.uv."platforms.linux-arm64-musl"] -checksum = "sha256:8767a0e77f2cd45436401b1b42bf7e9ed5a4a91a74a5305d6fe93249d0f6dbc5" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850993" -provenance = "github-attestations" - -[tools.uv."platforms.linux-x64"] -checksum = "sha256:a4742988791c9aeae68c78150d6cba762062ad2a47e53738c2779d2b596bfcdb" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851105" -provenance = "github-attestations" - -[tools.uv."platforms.linux-x64-musl"] -checksum = "sha256:a4742988791c9aeae68c78150d6cba762062ad2a47e53738c2779d2b596bfcdb" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-unknown-linux-musl.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851105" -provenance = "github-attestations" - -[tools.uv."platforms.macos-arm64"] -checksum = "sha256:5bb0e5fe008a773c3dbcb97ff79cd89e1241464fe9d2f986d52ad8f1b037bd62" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-aarch64-apple-darwin.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514850968" -provenance = "github-attestations" - -[tools.uv."platforms.macos-x64"] -checksum = "sha256:b3b2137477cf96c9686ebfb71524614cec780c673fd73e59bce099aef02e70e8" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-apple-darwin.tar.gz" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851078" -provenance = "github-attestations" - -[tools.uv."platforms.windows-x64"] -checksum = "sha256:4c4d49d8738847d9b71ba319e49a5688c93eac0fe6204b1df24e98528dddf39a" -url = "https://github.com/astral-sh/uv/releases/download/0.12.5/uv-x86_64-pc-windows-msvc.zip" -url_api = "https://api.github.com/repos/astral-sh/uv/releases/assets/514851084" -provenance = "github-attestations" diff --git a/mise.toml b/mise.toml index e4a7671..449df01 100644 --- a/mise.toml +++ b/mise.toml @@ -7,8 +7,8 @@ min_version = "2026.7.5" experimental = true [tools] +pipx = "1.16.7" ruff = "0.16.3" -uv = "0.12.5" [bootstrap.packages] # APT diff --git a/tasks/chart b/tasks/chart deleted file mode 100755 --- a/tasks/chart +++ /dev/null @@ -1,111 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Lint, package, and optionally publish a Helm chart" -#MISE tools={"helm"="4.1.1"} - -usage() { - cat <<'EOF' -Usage: mise run chart -- --chart DIR [options] - -Options: - --version VERSION Immutable chart version (required for packaging) - --app-version VERSION Application version written to Chart.yaml - --output DIR Package directory (default: dist/charts) - --push OCI_URL Push package to an OCI registry; repeatable - --lint-only Strictly lint without packaging -EOF -} - -chart= -version= -app_version= -output=dist/charts -push_urls= -lint_only=false - -while [ "$#" -gt 0 ]; do - case "$1" in - --chart) - [ "$#" -ge 2 ] || { echo "chart: --chart requires a value" >&2; exit 2; } - chart=$2 - shift 2 - ;; - --version) - [ "$#" -ge 2 ] || { echo "chart: --version requires a value" >&2; exit 2; } - version=$2 - shift 2 - ;; - --app-version) - [ "$#" -ge 2 ] || { echo "chart: --app-version requires a value" >&2; exit 2; } - app_version=$2 - shift 2 - ;; - --output) - [ "$#" -ge 2 ] || { echo "chart: --output requires a value" >&2; exit 2; } - output=$2 - shift 2 - ;; - --push) - [ "$#" -ge 2 ] || { echo "chart: --push requires a value" >&2; exit 2; } - case $2 in - oci://*) ;; - *) echo "chart: registry must use oci://: $2" >&2; exit 2 ;; - esac - push_urls="${push_urls}${push_urls:+ -}$2" - shift 2 - ;; - --lint-only) - lint_only=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "chart: unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -command -v helm >/dev/null 2>&1 || { echo "chart: helm is not installed" >&2; exit 1; } -[ -n "$chart" ] || { echo "chart: --chart is required" >&2; exit 2; } -[ -d "$chart" ] || { echo "chart: directory not found: $chart" >&2; exit 1; } -[ -f "$chart/Chart.yaml" ] || { echo "chart: Chart.yaml not found in $chart" >&2; exit 1; } - -name=$(sed -n 's/^name:[[:space:]]*\([A-Za-z0-9_.-][A-Za-z0-9_.-]*\)[[:space:]]*$/\1/p' "$chart/Chart.yaml" | head -n 1) -[ -n "$name" ] || { echo "chart: cannot read chart name from $chart/Chart.yaml" >&2; exit 1; } - -helm lint "$chart" --strict - -if [ "$lint_only" = true ]; then - if [ -n "$version" ] || [ -n "$app_version" ] || [ -n "$push_urls" ]; then - echo "chart: --lint-only cannot package or push a chart" >&2 - exit 2 - fi - exit 0 -fi - -[ -n "$version" ] || { echo "chart: --version is required unless --lint-only is used" >&2; exit 2; } -mkdir -p "$output" - -set -- helm package "$chart" --destination "$output" --version "$version" -if [ -n "$app_version" ]; then - set -- "$@" --app-version "$app_version" -fi -"$@" - -package=$output/$name-$version.tgz -[ -f "$package" ] || { echo "chart: Helm did not create expected package: $package" >&2; exit 1; } - -old_ifs=$IFS -IFS=' -' -for registry in $push_urls; do - helm push "$package" "${registry%/}" -done -IFS=$old_ifs diff --git a/tasks/chart.py b/tasks/chart.py new file mode 100755 --- /dev/null +++ b/tasks/chart.py @@ -0,0 +1,87 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Lint, package, and optionally publish a Helm chart" +#MISE tools={"pipx"="1.16.7","python"="3.14.7","helm"="4.1.1"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import re +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, require_command, run, task_main + +TASK = "chart" +CHART_NAME = re.compile(r"^name:\s*([A-Za-z0-9_.-][A-Za-z0-9_.-]*)\s*$") + + +def chart_name(chart: Path) -> str: + for line in (chart / "Chart.yaml").read_text().splitlines(): + if match := CHART_NAME.fullmatch(line): + return match.group(1) + raise TaskError(f"{TASK}: cannot read chart name from {chart}/Chart.yaml") + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser(prog="mise run chart --") + command.add_argument("--chart", required=True, type=Path) + command.add_argument("--version") + command.add_argument("--app-version") + command.add_argument("--output", default=Path("dist/charts"), type=Path) + command.add_argument("--push", action="append", default=[], metavar="OCI_URL") + command.add_argument("--lint-only", action="store_true") + arguments = command.parse_args(args) + + require_command("helm", TASK) + if not arguments.chart.is_dir(): + raise TaskError(f"{TASK}: directory not found: {arguments.chart}") + if not (arguments.chart / "Chart.yaml").is_file(): + raise TaskError(f"{TASK}: Chart.yaml not found in {arguments.chart}") + for registry in arguments.push: + if not registry.startswith("oci://"): + command.error(f"registry must use oci://: {registry}") + + name = chart_name(arguments.chart) + await run("helm", "lint", arguments.chart, "--strict") + + if arguments.lint_only: + if arguments.version or arguments.app_version or arguments.push: + command.error("--lint-only cannot package or push a chart") + return + if not arguments.version: + command.error("--version is required unless --lint-only is used") + + arguments.output.mkdir(parents=True, exist_ok=True) + package_arguments: list[str | Path] = [ + "helm", + "package", + arguments.chart, + "--destination", + arguments.output, + "--version", + arguments.version, + ] + if arguments.app_version: + package_arguments.extend(("--app-version", arguments.app_version)) + await run(package_arguments) + + package = arguments.output / f"{name}-{arguments.version}.tgz" + if not package.is_file(): + raise TaskError(f"{TASK}: Helm did not create expected package: {package}") + for registry in arguments.push: + await run("helm", "push", package, registry.rstrip("/")) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/container b/tasks/container deleted file mode 100755 --- a/tasks/container +++ /dev/null @@ -1,173 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Build and optionally publish an OCI container image" - -usage() { - cat <<'EOF' -Usage: mise run container -- --tag IMAGE [options] - -Options: - --context DIR Build context (default: .) - --file FILE Dockerfile relative to the context (default: Dockerfile) - --platform PLATFORMS Target platforms (default: linux/amd64) - --target TARGET Dockerfile build target - --build-arg VALUE Build argument; repeatable - --label VALUE OCI image label; repeatable - --tag IMAGE Image tag; repeatable and required - --cache-scope SCOPE GitHub Actions cache scope - --provenance BOOL BuildKit provenance (default: true) - --sbom BOOL BuildKit SBOM (default: true) - --push Push the resulting image - --load Load a single-platform image locally -EOF -} - -context=. -file=Dockerfile -platform=linux/amd64 -target= -build_args= -labels= -tags= -cache_scope= -provenance=true -sbom=true -push=false -load=false - -while [ "$#" -gt 0 ]; do - case "$1" in - --context) - [ "$#" -ge 2 ] || { echo "container: --context requires a value" >&2; exit 2; } - context=$2 - shift 2 - ;; - --file) - [ "$#" -ge 2 ] || { echo "container: --file requires a value" >&2; exit 2; } - file=$2 - shift 2 - ;; - --platform) - [ "$#" -ge 2 ] || { echo "container: --platform requires a value" >&2; exit 2; } - platform=$2 - shift 2 - ;; - --target) - [ "$#" -ge 2 ] || { echo "container: --target requires a value" >&2; exit 2; } - target=$2 - shift 2 - ;; - --build-arg) - [ "$#" -ge 2 ] || { echo "container: --build-arg requires a value" >&2; exit 2; } - case $2 in *' -'*) echo "container: build arguments cannot contain newlines" >&2; exit 2 ;; esac - build_args="${build_args}${build_args:+ -}$2" - shift 2 - ;; - --label) - [ "$#" -ge 2 ] || { echo "container: --label requires a value" >&2; exit 2; } - case $2 in *' -'*) echo "container: labels cannot contain newlines" >&2; exit 2 ;; esac - labels="${labels}${labels:+ -}$2" - shift 2 - ;; - --tag) - [ "$#" -ge 2 ] || { echo "container: --tag requires a value" >&2; exit 2; } - case $2 in ''|*[[:space:]]*) echo "container: invalid image tag: $2" >&2; exit 2 ;; esac - tags="${tags}${tags:+ -}$2" - shift 2 - ;; - --cache-scope) - [ "$#" -ge 2 ] || { echo "container: --cache-scope requires a value" >&2; exit 2; } - cache_scope=$2 - shift 2 - ;; - --provenance) - [ "$#" -ge 2 ] || { echo "container: --provenance requires a value" >&2; exit 2; } - case $2 in true|false) provenance=$2 ;; *) echo "container: --provenance must be true or false" >&2; exit 2 ;; esac - shift 2 - ;; - --sbom) - [ "$#" -ge 2 ] || { echo "container: --sbom requires a value" >&2; exit 2; } - case $2 in true|false) sbom=$2 ;; *) echo "container: --sbom must be true or false" >&2; exit 2 ;; esac - shift 2 - ;; - --push) - push=true - shift - ;; - --load) - load=true - shift - ;; - -h|--help) - usage - exit 0 - ;; - *) - echo "container: unknown argument: $1" >&2 - usage >&2 - exit 2 - ;; - esac -done - -command -v docker >/dev/null 2>&1 || { echo "container: docker is not installed" >&2; exit 1; } -[ -d "$context" ] || { echo "container: context directory not found: $context" >&2; exit 1; } -[ -n "$tags" ] || { echo "container: at least one --tag is required" >&2; exit 2; } - -case $file in - /*) dockerfile=$file ;; - *) dockerfile=$context/$file ;; -esac -[ -f "$dockerfile" ] || { echo "container: Dockerfile not found: $dockerfile" >&2; exit 1; } - -if [ "$push" = true ] && [ "$load" = true ]; then - echo "container: --push and --load are mutually exclusive" >&2 - exit 2 -fi -case $platform:$load in - *,*:true) echo "container: --load supports exactly one platform" >&2; exit 2 ;; -esac - -set -- docker buildx build \ - --file "$dockerfile" \ - --platform "$platform" \ - "--provenance=$provenance" \ - "--sbom=$sbom" - -if [ -n "$target" ]; then - set -- "$@" --target "$target" -fi - -old_ifs=$IFS -IFS=' -' -for value in $build_args; do - set -- "$@" --build-arg "$value" -done -for value in $labels; do - set -- "$@" --label "$value" -done -for value in $tags; do - set -- "$@" --tag "$value" -done -IFS=$old_ifs - -if [ -n "$cache_scope" ]; then - set -- "$@" \ - --cache-from "type=gha,scope=$cache_scope" \ - --cache-to "type=gha,mode=max,scope=$cache_scope" -fi -if [ "$push" = true ]; then - set -- "$@" --push -elif [ "$load" = true ]; then - set -- "$@" --load -fi -set -- "$@" "$context" - -exec "$@" diff --git a/tasks/container.py b/tasks/container.py new file mode 100755 --- /dev/null +++ b/tasks/container.py @@ -0,0 +1,105 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Build and optionally publish an OCI container image" +#MISE tools={"pipx"="1.16.7","python"="3.14.7"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, require_command, run, task_main + +TASK = "container" + + +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser(prog="mise run container --") + command.add_argument("--context", default=Path("."), type=Path) + command.add_argument("--file", default=Path("Dockerfile"), type=Path) + command.add_argument("--platform", default="linux/amd64") + command.add_argument("--target") + command.add_argument("--build-arg", action="append", default=[]) + command.add_argument("--label", action="append", default=[]) + command.add_argument("--tag", action="append", required=True) + command.add_argument("--cache-scope") + command.add_argument("--provenance", choices=("true", "false"), default="true") + command.add_argument("--sbom", choices=("true", "false"), default="true") + command.add_argument("--push", action="store_true") + command.add_argument("--load", action="store_true") + arguments = command.parse_args(args) + + require_command("docker", TASK) + if not arguments.context.is_dir(): + raise TaskError(f"{TASK}: context directory not found: {arguments.context}") + dockerfile = ( + arguments.file + if arguments.file.is_absolute() + else arguments.context / arguments.file + ) + if not dockerfile.is_file(): + raise TaskError(f"{TASK}: Dockerfile not found: {dockerfile}") + for value in arguments.build_arg: + if "\n" in value: + command.error("build arguments cannot contain newlines") + for value in arguments.label: + if "\n" in value: + command.error("labels cannot contain newlines") + for tag in arguments.tag: + if not tag or any(character.isspace() for character in tag): + command.error(f"invalid image tag: {tag}") + if arguments.push and arguments.load: + command.error("--push and --load are mutually exclusive") + if arguments.load and "," in arguments.platform: + command.error("--load supports exactly one platform") + + build: list[str | Path] = [ + "docker", + "buildx", + "build", + "--file", + dockerfile, + "--platform", + arguments.platform, + f"--provenance={arguments.provenance}", + f"--sbom={arguments.sbom}", + ] + if arguments.target: + build.extend(("--target", arguments.target)) + for value in arguments.build_arg: + build.extend(("--build-arg", value)) + for value in arguments.label: + build.extend(("--label", value)) + for tag in arguments.tag: + build.extend(("--tag", tag)) + if arguments.cache_scope: + build.extend( + ( + "--cache-from", + f"type=gha,scope={arguments.cache_scope}", + "--cache-to", + f"type=gha,mode=max,scope={arguments.cache_scope}", + ) + ) + if arguments.push: + build.append("--push") + elif arguments.load: + build.append("--load") + build.append(arguments.context) + + await run(*build) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/libs/__init__.py b/tasks/libs/__init__.py new file mode 100644 --- /dev/null +++ b/tasks/libs/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD diff --git a/tasks/_apk.py b/tasks/libs/apk.py index 70d156c..ae2efc3 100644 --- a/tasks/_apk.py +++ b/tasks/libs/apk.py @@ -3,18 +3,20 @@ from __future__ import annotations +import asyncio import gzip import hashlib import io import os import platform import shutil -import subprocess import tarfile import urllib.request from pathlib import Path -from _lib import TaskError, run +from shellous import sh + +from .common import TaskError, capture, run TASK = "publish" APK_TOOLS_VERSION = "2.14.10-r0" @@ -24,19 +26,15 @@ APK_TOOLS_SHA256 = { } -def architecture(package: Path) -> str: - metadata = run( - ["tar", "-xOzf", package, ".PKGINFO"], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - ).stdout +async def architecture(package: Path) -> str: + metadata = await capture("tar", "-xOzf", package, ".PKGINFO").stderr(sh.DEVNULL) for line in metadata.splitlines(): if line.startswith("arch = "): return line.removeprefix("arch = ") raise TaskError(f"{TASK}: cannot read APK architecture: {package}") -def apk_tool(work: Path) -> Path: +async def apk_tool(work: Path) -> Path: configured = os.environ.get("APK_TOOL") if configured: tool = Path(configured) @@ -61,33 +59,29 @@ def apk_tool(work: Path) -> Path: "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/" f"{apk_arch}/apk-tools-static-{APK_TOOLS_VERSION}.apk" ) - urllib.request.urlretrieve(url, archive) + await asyncio.to_thread(urllib.request.urlretrieve, url, archive) if hashlib.sha256(archive.read_bytes()).hexdigest() != APK_TOOLS_SHA256[apk_arch]: raise TaskError(f"{TASK}: apk-tools checksum mismatch") directory = work / "apk-tools" directory.mkdir() - run( - ["tar", "-xzf", archive, "-C", directory, "sbin/apk.static"], - stderr=subprocess.DEVNULL, + await run("tar", "-xzf", archive, "-C", directory, "sbin/apk.static").stderr( + sh.DEVNULL ) return directory / "sbin" / "apk.static" -def sign_index(context, index: Path) -> None: +async def sign_index(context, index: Path) -> None: name = f".SIGN.RSA256.{context.apk_signing.public_key_name}" signature = index.parent / name - run( - [ - "openssl", - "dgst", - "-sha256", - "-sign", - context.apk_signing.private_key_file, - "-out", - signature, - index, - ], - env=context.apk_signing.environment, + await run.set(env=context.apk_signing.environment, inherit_env=False)( + "openssl", + "dgst", + "-sha256", + "-sign", + context.apk_signing.private_key_file, + "-out", + signature, + index, ) data = signature.read_bytes() stream = io.BytesIO() @@ -104,10 +98,17 @@ def sign_index(context, index: Path) -> None: signature.unlink() -def publish(context) -> None: - tool = apk_tool(context.work) +async def publish(context) -> None: + tool = await apk_tool(context.work) packages = context.packages["apk"] - architectures = sorted({architecture(package) for package in packages}) + package_architectures = dict( + zip( + packages, + await asyncio.gather(*(architecture(package) for package in packages)), + strict=True, + ) + ) + architectures = sorted(set(package_architectures.values())) keys = context.work / "apk-keys" keys.mkdir() for public_key in context.apk_public_keys: @@ -119,28 +120,25 @@ def publish(context) -> None: remote = context.storage.service_key("apk", context.channel, apk_arch) context.storage.download_prefix(remote, root, "*.apk") for package in packages: - if architecture(package) == apk_arch: + if package_architectures[package] == apk_arch: context.add_package(package, root) for package in root.glob("*.apk"): - if run( - [tool, "verify", "--keys-dir", keys, package], check=False - ).returncode: + result = await run.result(tool, "verify", "--keys-dir", keys, package) + if result.exit_code: raise TaskError( f"{TASK}: APK signature verification failed: {package.name}" ) index = root / "APKINDEX.tar.gz" - run( - [ - tool, - "--allow-untrusted", - "index", - "--description", - f"Dimidium Labs {context.service} {context.channel}", - "--output", - index, - *root.glob("*.apk"), - ] + await run( + tool, + "--allow-untrusted", + "index", + "--description", + f"Dimidium Labs {context.service} {context.channel}", + "--output", + index, + sorted(root.glob("*.apk")), ) - sign_index(context, index) + await sign_index(context, index) context.storage.upload_payloads(root, remote, "*.apk") context.storage.upload(index, f"{remote}/APKINDEX.tar.gz") diff --git a/tasks/_apt.py b/tasks/libs/apt.py index d153d9b..8952900 100644 --- a/tasks/_apt.py +++ b/tasks/libs/apt.py @@ -3,14 +3,12 @@ from __future__ import annotations -import subprocess - -from _lib import TaskError, require_command, run +from .common import TaskError, capture, require_command, run TASK = "publish" -def publish(context) -> None: +async def publish(context) -> None: require_command("apt-ftparchive", TASK) require_command("dpkg-deb", TASK) root = context.work / "apt" @@ -25,10 +23,7 @@ def publish(context) -> None: architectures = sorted( { - run( - ["dpkg-deb", "-f", package, "Architecture"], - stdout=subprocess.PIPE, - ).stdout.strip() + (await capture("dpkg-deb", "-f", package, "Architecture")).strip() for package in pool.glob("*.deb") } ) @@ -57,31 +52,28 @@ Tree "dists/{context.channel}" {{ }}; ''' ) - run(["apt-ftparchive", "generate", config]) + await run("apt-ftparchive", "generate", config) release = metadata / "Release" - with release.open("wb") as stream: - run( - [ - "apt-ftparchive", - "-o", - "APT::FTPArchive::Release::Origin=Dimidium Labs", - "-o", - f"APT::FTPArchive::Release::Label={context.service} {context.channel}", - "-o", - f"APT::FTPArchive::Release::Suite={context.channel}", - "-o", - f"APT::FTPArchive::Release::Codename={context.channel}", - "-o", - "APT::FTPArchive::Release::Components=main", - "-o", - f"APT::FTPArchive::Release::Architectures={architecture_list}", - "release", - f"{metadata}/", - ], - stdout=stream, - text=False, - ) - context.gpg.sign(metadata / "Release.gpg", "--armor", "--detach-sign", release) - context.gpg.sign(metadata / "InRelease", "--clearsign", release) + await run( + "apt-ftparchive", + "-o", + "APT::FTPArchive::Release::Origin=Dimidium Labs", + "-o", + f"APT::FTPArchive::Release::Label={context.service} {context.channel}", + "-o", + f"APT::FTPArchive::Release::Suite={context.channel}", + "-o", + f"APT::FTPArchive::Release::Codename={context.channel}", + "-o", + "APT::FTPArchive::Release::Components=main", + "-o", + f"APT::FTPArchive::Release::Architectures={architecture_list}", + "release", + f"{metadata}/", + ).stdout(release) + await context.gpg.sign( + metadata / "Release.gpg", "--armor", "--detach-sign", release + ) + await context.gpg.sign(metadata / "InRelease", "--clearsign", release) context.storage.upload_payloads(pool, pool_prefix, "*.deb") context.storage.replace_prefix(metadata, metadata_prefix) diff --git a/tasks/_lib.py b/tasks/libs/common.py index 051f71d..e1373e6 100644 --- a/tasks/_lib.py +++ b/tasks/libs/common.py @@ -3,11 +3,18 @@ from __future__ import annotations +import asyncio import os import shutil -import subprocess +import sys +from collections.abc import Awaitable, Callable, Sequence from pathlib import Path +from shellous import ResultError, sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) + class TaskError(RuntimeError): pass @@ -28,16 +35,25 @@ def required_env(name: str, task: str, purpose: str = "") -> str: raise TaskError(f"{task}: {name} is required{suffix}") -def run(args, **options) -> subprocess.CompletedProcess[str]: - check = options.pop("check", True) - options.setdefault("text", True) - if "input_text" in options: - options["input"] = options.pop("input_text") - return subprocess.run([str(arg) for arg in args], check=check, **options) +def task_main( + task: str, + main: Callable[[Sequence[str]], Awaitable[None]], + args: Sequence[str], +) -> None: + try: + asyncio.run(main(args)) + except TaskError as error: + print(error, file=sys.stderr) + raise SystemExit(1) from None + except ResultError as error: + exit_code = error.result.exit_code + print(f"{task}: command failed with exit code {exit_code}", file=sys.stderr) + raise SystemExit(exit_code) from None class GPGSigning: def __init__(self, task: str, work: Path): + self.task = task require_command("gpg", task) private_key = required_env("GPG_PRIVATE_KEY", task) self.passphrase = required_env("GPG_PASSPHRASE", task) @@ -53,8 +69,14 @@ class GPGSigning: self.environment.pop("GPG_PRIVATE_KEY", None) self.environment.pop("GPG_PASSPHRASE", None) self.environment.pop("APK_PRIVATE_KEY", None) - run( - [ + + @classmethod + async def create(cls, task: str, work: Path) -> GPGSigning: + signing = cls(task, work) + command = run.set(env=signing.environment, inherit_env=False) + await ( + f"{signing.passphrase}\n" + | command( "gpg", "--batch", "--yes", @@ -63,39 +85,44 @@ class GPGSigning: "--passphrase-fd", "0", "--import", - self.private_key_file, - ], - env=self.environment, - input_text=f"{self.passphrase}\n", + signing.private_key_file, + ) ) + return signing def package_environment(self) -> dict[str, str]: environment = dict(self.environment) environment["GPG_KEY_ID"] = self.short_key_id return environment - def prime_agent(self) -> None: + async def prime_agent(self) -> None: signature = self.private_key_file.with_suffix(".sig") - self.sign(signature, "--detach-sign", self.private_key_file) + await self.sign(signature, "--detach-sign", self.private_key_file) signature.unlink() - def export_public_key(self, output: Path) -> None: - with output.open("wb") as stream: - run( - ["gpg", "--batch", "--yes", "--armor", "--export", self.key_id], - env=self.environment, - stdout=stream, - text=False, - ) - - def verify_public_bundle(self, bundle: Path) -> None: + async def export_public_key(self, output: Path) -> None: + command = run.set(env=self.environment, inherit_env=False) + await command( + "gpg", + "--batch", + "--yes", + "--armor", + "--export", + self.key_id, + ).stdout(output) + + async def verify_public_bundle(self, bundle: Path) -> None: + command = capture.set(env=self.environment, inherit_env=False) + output = await command( + "gpg", + "--batch", + "--with-colons", + "--show-keys", + bundle, + ) fingerprints = { line.split(":")[9] - for line in run( - ["gpg", "--batch", "--with-colons", "--show-keys", bundle], - env=self.environment, - stdout=subprocess.PIPE, - ).stdout.splitlines() + for line in output.splitlines() if line.startswith("fpr:") } if self.key_id not in fingerprints: @@ -103,9 +130,11 @@ class GPGSigning: f"{self.task}: packages.gpg does not contain signing key {self.key_id}" ) - def sign(self, output: Path, *arguments: str | Path) -> None: - run( - [ + async def sign(self, output: Path, *arguments: str | Path) -> None: + command = run.set(env=self.environment, inherit_env=False) + await ( + f"{self.passphrase}\n" + | command( "gpg", f"--default-key={self.key_id}", "--batch", @@ -116,10 +145,8 @@ class GPGSigning: "0", "-o", output, - *arguments, - ], - env=self.environment, - input_text=f"{self.passphrase}\n", + arguments, + ) ) @@ -138,16 +165,14 @@ class APKSigning: self.environment.pop("GPG_PASSPHRASE", None) self.environment["APK_SIGNING_KEY"] = str(self.private_key_file) - def export_public_key(self, output: Path) -> None: - run( - [ - "openssl", - "rsa", - "-in", - self.private_key_file, - "-pubout", - "-out", - output, - ], - env=self.environment, + async def export_public_key(self, output: Path) -> None: + command = run.set(env=self.environment, inherit_env=False) + await command( + "openssl", + "rsa", + "-in", + self.private_key_file, + "-pubout", + "-out", + output, ) diff --git a/tasks/_repository.py b/tasks/libs/repository.py index 577b007..772e1ab 100644 --- a/tasks/_repository.py +++ b/tasks/libs/repository.py @@ -7,8 +7,8 @@ import re import shutil from pathlib import Path -from _lib import APKSigning, GPGSigning, TaskError, required_env -from _storage import S3Storage +from .common import APKSigning, GPGSigning, TaskError, required_env +from .storage import S3Storage TASK = "publish" KEY_VERSION = re.compile(r"^[0-9]{4}$") @@ -57,23 +57,23 @@ class Repository: if not destination.exists(): shutil.copy2(source, destination) - def setup_openpgp(self) -> None: - self.gpg = GPGSigning(TASK, self.work) + async def setup_openpgp(self) -> None: + self.gpg = await GPGSigning.create(TASK, self.work) current = self.work / "current-packages.gpg" - self.gpg.export_public_key(current) + await self.gpg.export_public_key(current) self.check_public_key(current, f"keys/packages.{self.key_version}.gpg") bundle = self.work / "packages.gpg" if not self.storage.download("packages.gpg", bundle): raise TaskError(f"{TASK}: organization key packages.gpg is not provisioned") - self.gpg.verify_public_bundle(bundle) + await self.gpg.verify_public_bundle(bundle) self.gpg_public_key = bundle - def setup_rsa(self) -> None: + async def setup_rsa(self) -> None: key_name = f"packages.{self.key_version}" self.apk_signing = APKSigning(TASK, self.work, key_name) current = self.work / self.apk_signing.public_key_name - self.apk_signing.export_public_key(current) + await self.apk_signing.export_public_key(current) self.check_public_key( current, f"keys/{self.apk_signing.public_key_name}", @@ -93,19 +93,19 @@ class Repository: f"{TASK}: current RSA public key is absent from key archive" ) - def setup_signing(self) -> None: + async def setup_signing(self) -> None: if {"deb", "rpm"} & set(self.formats): - self.setup_openpgp() + await self.setup_openpgp() if "apk" in self.formats: - self.setup_rsa() + await self.setup_rsa() - def publish(self) -> None: - from _apk import publish as publish_apk - from _apt import publish as publish_apt - from _rpm import publish as publish_rpm + async def publish(self) -> None: + from .apk import publish as publish_apk + from .apt import publish as publish_apt + from .rpm import publish as publish_rpm publishers = {"deb": publish_apt, "rpm": publish_rpm, "apk": publish_apk} with self.storage.lock(self.channel): - self.setup_signing() + await self.setup_signing() for package_format in self.formats: - publishers[package_format](self) + await publishers[package_format](self) diff --git a/tasks/_rpm.py b/tasks/libs/rpm.py index c2f476f..390446e 100644 --- a/tasks/_rpm.py +++ b/tasks/libs/rpm.py @@ -4,14 +4,13 @@ from __future__ import annotations import shutil -import subprocess -from _lib import TaskError, require_command, run +from .common import TaskError, capture, require_command, run TASK = "publish" -def publish(context) -> None: +async def publish(context) -> None: require_command("createrepo_c", TASK) require_command("rpmkeys", TASK) root = context.work / "rpm" @@ -23,21 +22,22 @@ def publish(context) -> None: rpm_database = context.work / "rpmdb" rpm_database.mkdir() - run(["rpmkeys", "--dbpath", rpm_database, "--import", context.gpg_public_key]) + await run("rpmkeys", "--dbpath", rpm_database, "--import", context.gpg_public_key) for package in root.glob("*.rpm"): - result = run( - ["rpmkeys", "--dbpath", rpm_database, "--checksig", package], - stdout=subprocess.PIPE, - ).stdout + result = await capture( + "rpmkeys", "--dbpath", rpm_database, "--checksig", package + ) if "signatures OK" not in result: raise TaskError( f"{TASK}: RPM is not signed by a trusted key: {package.name}" ) shutil.rmtree(root / "repodata", ignore_errors=True) - run(["createrepo_c", root]) + await run("createrepo_c", root) repomd = root / "repodata" / "repomd.xml" - context.gpg.sign(repomd.with_suffix(".xml.asc"), "--armor", "--detach-sign", repomd) + await context.gpg.sign( + repomd.with_suffix(".xml.asc"), "--armor", "--detach-sign", repomd + ) definition = root / f"{context.service}-{context.channel}.repo" definition.write_text( f"""[{context.service}-{context.channel}] diff --git a/tasks/_storage.py b/tasks/libs/storage.py index 1fc163e..7bd89d2 100644 --- a/tasks/_storage.py +++ b/tasks/libs/storage.py @@ -12,9 +12,10 @@ from contextlib import contextmanager from pathlib import Path import boto3 -from _lib import TaskError, required_env from botocore.exceptions import ClientError +from .common import TaskError, required_env + class S3Storage: def __init__(self, task: str, service: str): diff --git a/tasks/licenses b/tasks/licenses deleted file mode 100755 --- a/tasks/licenses +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Verify repository licensing metadata" -#MISE tools={"pipx:reuse"="6.2.0","aqua:EmbarkStudios/cargo-deny"="0.19.0"} - -check_copyright_headers() { - invalid_headers=$( - git grep -n -I -E \ - '^((<!--|#|//|/\*|\*)[[:space:]]*)?(Copyright[[:space:]]+(\([cC]\)|©)|SPDX-FileCopyrightText:)' \ - -- . \ - ':(exclude)*.md' \ - ':(exclude)LICENSE' \ - ':(exclude)LICENSES/**' \ - ':(exclude)COPYING*' | - awk 'match($0, /:[0-9]+:/) { - line = substr($0, RSTART + 1, RLENGTH - 2) - if (line > 10) { - next - } - - text = substr($0, RSTART + RLENGTH) - reason = "" - if (text ~ /^[[:space:]]*((<!--|#|\/\/|\/\*|\*)[[:space:]]*)?Copyright[[:space:]]+(\([cC]\)|©)/) { - reason = "legacy copyright header" - } else if (text ~ /SPDX-FileCopyrightText:($|[^ ]| [[:space:]])/) { - reason = "expected exactly one space after colon" - } else if (text ~ /SPDX-FileCopyrightText:.*Nikolay Govorov/ && - text !~ /SPDX-FileCopyrightText: 2026 Nikolay Govorov([[:space:]]*(\*\/|-->))?$/) { - reason = "expected 2026 Nikolay Govorov" - } - - if (reason != "") { - print reason ": " $0 - } - }' - ) - - if [ -n "$invalid_headers" ]; then - printf '%s\n%s\n' 'Invalid copyright headers:' "$invalid_headers" >&2 - return 1 - fi -} - -check_copyright_headers -reuse lint - -if [ -f Cargo.toml ]; then - cargo-deny check -fi diff --git a/tasks/licenses.py b/tasks/licenses.py new file mode 100755 --- /dev/null +++ b/tasks/licenses.py @@ -0,0 +1,95 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Verify repository licensing metadata" +#MISE tools={"pipx"="1.16.7","python"="3.14.7","pipx:reuse"="6.2.0","aqua:EmbarkStudios/cargo-deny"="0.19.0"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import re +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, capture, run, task_main + +TASK = "licenses" +HEADER = ( + r"^((<!--|#|//|/\*|\*)[[:space:]]*)?" + r"(Copyright[[:space:]]+(\([cC]\)|©)|SPDX-FileCopyrightText:)" +) +LEGACY_COPYRIGHT = re.compile(r"^\s*((<!--|#|//|/\*|\*)\s*)?Copyright\s+(\([cC]\)|©)") +CANONICAL_COPYRIGHT = re.compile( + r"SPDX-FileCopyrightText: 2026 Nikolay Govorov(?:\s*(?:\*/|-->))?$" +) + + +async def check_copyright_headers() -> None: + result = await capture.result( + "git", + "grep", + "-n", + "-I", + "-E", + HEADER, + "--", + ".", + ":(exclude)*.md", + ":(exclude)LICENSE", + ":(exclude)LICENSES/**", + ":(exclude)COPYING*", + ) + if result.exit_code not in {0, 1}: + raise TaskError(f"{TASK}: git grep failed with exit code {result.exit_code}") + + invalid: list[str] = [] + for line in result.output.splitlines(): + match = re.search(r":([0-9]+):", line) + if match is None or int(match.group(1)) > 10: + continue + text = line[match.end() :] + reason = "" + if LEGACY_COPYRIGHT.match(text): + reason = "legacy copyright header" + elif (position := text.find("SPDX-FileCopyrightText:")) >= 0: + suffix = text[position + len("SPDX-FileCopyrightText:") :] + if ( + not suffix + or not suffix.startswith(" ") + or (len(suffix) > 1 and suffix[1].isspace()) + ): + reason = "expected exactly one space after colon" + if ( + not reason + and "SPDX-FileCopyrightText:" in text + and "Nikolay Govorov" in text + and not CANONICAL_COPYRIGHT.search(text) + ): + reason = "expected 2026 Nikolay Govorov" + if reason: + invalid.append(f"{reason}: {line}") + + if invalid: + print("Invalid copyright headers:", *invalid, sep="\n", file=sys.stderr) + raise TaskError(f"{TASK}: invalid copyright headers") + + +async def main(args: Sequence[str]) -> None: + argparse.ArgumentParser(prog="mise run licenses --").parse_args(args) + await check_copyright_headers() + await run("reuse", "lint") + if Path("Cargo.toml").is_file(): + await run("cargo-deny", "check") + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/package b/tasks/package.py index 6db5f86..9856b26 100755 --- a/tasks/package +++ b/tasks/package.py @@ -1,10 +1,14 @@ -#!/usr/bin/env -S python3 -B +#!/usr/bin/env -S pipx run --backend pip # SPDX-FileCopyrightText: 2026 Nikolay Govorov # SPDX-License-Identifier: 0BSD # fmt: off #MISE description="Build release archives and signed Linux packages" -#MISE tools={"python"="3.14.7","nfpm"="2.47.0"} +#MISE tools={"pipx"="1.16.7","python"="3.14.7","nfpm"="2.47.0"} # fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// from __future__ import annotations @@ -12,12 +16,21 @@ import argparse import os import re import shutil -import subprocess import sys import tempfile +from collections.abc import Sequence from pathlib import Path -from _lib import APKSigning, GPGSigning, TaskError, require_command, run +sys.dont_write_bytecode = True + +from libs.common import ( + APKSigning, + GPGSigning, + TaskError, + require_command, + run, + task_main, +) TASK = "package" SYSTEM_FORMATS = {"deb", "rpm", "apk"} @@ -26,29 +39,6 @@ SAFE_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") SAFE_KEY_VERSION = re.compile(r"^[0-9]{4}$") -def parser() -> argparse.ArgumentParser: - result = argparse.ArgumentParser( - prog="mise run package --", - usage=( - "%(prog)s --output DIR [--config FILE] " - "[--version VERSION --arch ARCH] [--apk-public-key FILE] " - "[--archive-root DIR --archive-name NAME] " - "deb|rpm|apk|tar.gz|zip..." - ), - ) - result.add_argument("--config", default="nfpm.yaml", type=Path) - result.add_argument("--version") - result.add_argument("--arch") - result.add_argument("--output", required=True, type=Path) - result.add_argument("--archive-root", type=Path) - result.add_argument("--archive-name") - result.add_argument("--apk-public-key") - result.add_argument( - "formats", nargs="+", choices=sorted(SYSTEM_FORMATS | ARCHIVE_FORMATS) - ) - return result - - def validate(arguments: argparse.Namespace, command: argparse.ArgumentParser) -> None: formats = set(arguments.formats) if formats & SYSTEM_FORMATS: @@ -71,7 +61,7 @@ def validate(arguments: argparse.Namespace, command: argparse.ArgumentParser) -> ) -def create_archive( +async def create_archive( archive_format: str, output: Path, root: Path, @@ -79,47 +69,60 @@ def create_archive( ) -> None: if archive_format == "tar.gz": require_command("tar", TASK) - run(["tar", "-czf", output / f"{name}.tar.gz", "-C", root, "."]) + await run("tar", "-czf", output / f"{name}.tar.gz", "-C", root, ".") return require_command("zip", TASK) destination = output / f"{name}.zip" destination.unlink(missing_ok=True) - run(["zip", "-qry", destination, "."], cwd=root) + await run.set(cwd=root)("zip", "-qry", destination, ".") -def sign_package( +async def sign_package( package_format: str, package: Path, signing: GPGSigning | None ) -> None: if signing is None or package_format not in {"deb", "rpm"}: return + command = run.set(env=signing.environment, inherit_env=False) if package_format == "deb": require_command("debsigs", TASK) - run( - [ - "debsigs", - "--sign=origin", - f"--default-key={signing.key_id}", - package, - ], - env=signing.environment, + await command( + "debsigs", + "--sign=origin", + f"--default-key={signing.key_id}", + package, ) else: require_command("rpmsign", TASK) - run( - [ - "rpmsign", - "--define", - f"_gpg_name {signing.key_id}", - "--addsign", - package, - ], - env=signing.environment, + await command( + "rpmsign", + "--define", + f"_gpg_name {signing.key_id}", + "--addsign", + package, ) -def main() -> None: - command = parser() - arguments = command.parse_args() +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser( + prog="mise run package --", + usage=( + "%(prog)s --output DIR [--config FILE] " + "[--version VERSION --arch ARCH] [--apk-public-key FILE] " + "[--archive-root DIR --archive-name NAME] " + "deb|rpm|apk|tar.gz|zip..." + ), + ) + command.add_argument("--config", default="nfpm.yaml", type=Path) + command.add_argument("--version") + command.add_argument("--arch") + command.add_argument("--output", required=True, type=Path) + command.add_argument("--archive-root", type=Path) + command.add_argument("--archive-name") + command.add_argument("--apk-public-key") + command.add_argument( + "formats", nargs="+", choices=sorted(SYSTEM_FORMATS | ARCHIVE_FORMATS) + ) + arguments = command.parse_args(args) validate(arguments, command) arguments.output.mkdir(parents=True, exist_ok=True) @@ -154,8 +157,8 @@ def main() -> None: gpg_signing: GPGSigning | None = None if formats & {"deb", "rpm"} and os.environ.get("GPG_PRIVATE_KEY"): - gpg_signing = GPGSigning(TASK, work) - gpg_signing.prime_agent() + gpg_signing = await GPGSigning.create(TASK, work) + await gpg_signing.prime_agent() environment = gpg_signing.package_environment() apk_signing: APKSigning | None = None @@ -168,20 +171,17 @@ def main() -> None: if "apk" in formats and arguments.apk_public_key: public_key = output / arguments.apk_public_key if apk_signing is not None: - apk_signing.export_public_key(public_key) + await apk_signing.export_public_key(public_key) elif environment.get("APK_SIGNING_KEY"): require_command("openssl", TASK) - run( - [ - "openssl", - "rsa", - "-in", - environment["APK_SIGNING_KEY"], - "-pubout", - "-out", - public_key, - ], - env=environment, + await run.set(env=environment, inherit_env=False)( + "openssl", + "rsa", + "-in", + environment["APK_SIGNING_KEY"], + "-pubout", + "-out", + public_key, ) for package_format in arguments.formats: @@ -194,28 +194,25 @@ def main() -> None: ) package_output = work / f"output-{package_format}" package_output.mkdir() - run( - [ - "nfpm", - "package", - "--config", - config, - "--packager", - package_format, - "--target", - f"{package_output}/", - ], - env=package_environment, + await run.set(env=package_environment, inherit_env=False)( + "nfpm", + "package", + "--config", + config, + "--packager", + package_format, + "--target", + f"{package_output}/", ) packages = list(package_output.iterdir()) if len(packages) != 1 or not packages[0].is_file(): raise TaskError( f"{TASK}: nFPM produced an unexpected number of packages" ) - sign_package(package_format, packages[0], gpg_signing) + await sign_package(package_format, packages[0], gpg_signing) shutil.move(packages[0], output / packages[0].name) else: - create_archive( + await create_archive( package_format, output, arguments.archive_root, @@ -224,13 +221,4 @@ def main() -> None: if __name__ == "__main__": - try: - main() - except TaskError as error: - print(error, file=sys.stderr) - raise SystemExit(1) from None - except subprocess.CalledProcessError as error: - print( - f"{TASK}: command failed with exit code {error.returncode}", file=sys.stderr - ) - raise SystemExit(error.returncode) from None + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/publish b/tasks/publish.py index 74cf2db..75c5299 100755 --- a/tasks/publish +++ b/tasks/publish.py @@ -1,49 +1,49 @@ -#!/usr/bin/env -S uv run --script --python 3.14.7 +#!/usr/bin/env -S pipx run --backend pip # SPDX-FileCopyrightText: 2026 Nikolay Govorov # SPDX-License-Identifier: 0BSD # fmt: off #MISE description="Publish signed package repositories to shared S3 storage" -#MISE tools={"uv"="0.12.5"} +#MISE tools={"pipx"="1.16.7","python"="3.14.7"} # fmt: on # /// script -# requires-python = ">=3.14" -# dependencies = ["boto3==1.43.75"] +# requires-python = ">=3.11" +# dependencies = ["boto3==1.43.75", "shellous==0.42.0"] # /// from __future__ import annotations import argparse +import os import re -import subprocess import sys import tempfile +from collections.abc import Sequence from pathlib import Path sys.dont_write_bytecode = True -from _lib import TaskError -from _repository import Repository +from libs.common import TaskError, task_main + +if test_path := os.environ.get("PUBLISH_TEST_PYTHONPATH"): + sys.path.insert(0, test_path) + +from libs.repository import Repository TASK = "publish" FORMATS = {"deb", "rpm", "apk"} SAFE_SLUG = re.compile(r"^[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$") -def parser() -> argparse.ArgumentParser: - result = argparse.ArgumentParser( +async def main(args: Sequence[str]) -> None: + command = argparse.ArgumentParser( prog="mise run publish --", - usage=("%(prog)s --service NAME --channel CHANNEL --input DIR deb|rpm|apk..."), + usage="%(prog)s --service NAME --channel CHANNEL --input DIR deb|rpm|apk...", ) - result.add_argument("--service", required=True) - result.add_argument("--channel", required=True) - result.add_argument("--input", required=True, type=Path) - result.add_argument("formats", nargs="+", choices=sorted(FORMATS)) - return result - - -def main() -> None: - command = parser() - arguments = command.parse_args() + command.add_argument("--service", required=True) + command.add_argument("--channel", required=True) + command.add_argument("--input", required=True, type=Path) + command.add_argument("formats", nargs="+", choices=sorted(FORMATS)) + arguments = command.parse_args(args) for label, value in ( ("service name", arguments.service), ("channel", arguments.channel), @@ -53,7 +53,7 @@ def main() -> None: if not arguments.input.is_dir(): raise TaskError(f"{TASK}: {arguments.input} not found") with tempfile.TemporaryDirectory(prefix="publish-") as directory: - Repository( + await Repository( arguments.service, arguments.channel, arguments.input.resolve(), @@ -63,14 +63,4 @@ def main() -> None: if __name__ == "__main__": - try: - main() - except TaskError as error: - print(error, file=sys.stderr) - raise SystemExit(1) from None - except subprocess.CalledProcessError as error: - print( - f"{TASK}: command failed with exit code {error.returncode}", - file=sys.stderr, - ) - raise SystemExit(error.returncode) from None + task_main(TASK, main, sys.argv[1:]) diff --git a/tasks/signoff b/tasks/signoff deleted file mode 100755 --- a/tasks/signoff +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD -#MISE description="Verify contributor identities and CLA acceptance trailers" - -root=$(git rev-parse --show-toplevel) -cd "$root" - -task_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) -approved_emails_file="$task_dir/../config/signoff-approved-emails" -unsupported_commits_file="$task_dir/../config/cla-unsupported-commits" - -if [ ! -f CLA.md ]; then - echo "CLA.md is missing" >&2 - exit 1 -fi - -head_cla_version_count=$(grep -c '^Version ' CLA.md || :) -if [ "$head_cla_version_count" -ne 1 ]; then - echo "CLA.md must declare exactly one version" >&2 - exit 1 -fi - -head_cla_version=$(sed -n 's/^Version //p' CLA.md) -if [ -z "$head_cla_version" ]; then - echo "CLA.md declares an empty version" >&2 - exit 1 -fi - -is_approved_email() { - grep -v '^[[:space:]]*#' "$approved_emails_file" | - grep -Fqx -- "$1" -} - -is_unsupported_commit() { - grep -v '^[[:space:]]*#' "$unsupported_commits_file" | - grep -Fqx -- "$1" -} - -check_identity() { - identity=$1 - role=$2 - email=$(printf '%s\n' "$identity" | sed -n 's/^.*<\([^<>]*\)>$/\1/p') - - if [ -z "$email" ]; then - echo "Commit $short_sha has an invalid $role identity: $identity" - bad=1 - return - fi - - if is_approved_email "$email"; then - return - fi - - requires_cla=1 - if ! printf '%s\n' "$signoffs" | grep -Fqx -- "$identity"; then - echo "Commit $short_sha $role $identity is missing a matching Signed-off-by" - bad=1 - fi -} - -if [ ! -f "$approved_emails_file" ]; then - echo "Approved email configuration is missing: $approved_emails_file" >&2 - exit 1 -fi - -if [ ! -f "$unsupported_commits_file" ]; then - echo "Unsupported commit configuration is missing: $unsupported_commits_file" >&2 - exit 1 -fi - -bad=0 -for sha in $(git log --no-merges --format=%H); do - short_sha=$(printf '%.8s' "$sha") - signoffs=$(git show -s --format='%(trailers:key=Signed-off-by,valueonly)' "$sha") - requires_cla=0 - - author=$(git show -s --format='%an <%ae>' "$sha") - check_identity "$author" "author" - - coauthors=$(git show -s --format='%(trailers:key=Co-authored-by,valueonly)' "$sha") - if [ -n "$coauthors" ]; then - old_ifs=$IFS - IFS=' -' - for coauthor in $coauthors; do - check_identity "$coauthor" "co-author" - done - IFS=$old_ifs - fi - - if [ "$requires_cla" -eq 1 ] && ! is_unsupported_commit "$sha"; then - expected_cla_version_count=$( - git show "$sha:CLA.md" 2>/dev/null | grep -c '^Version ' || : - ) - if [ "$expected_cla_version_count" -ne 1 ]; then - echo "Commit $short_sha does not contain a CLA.md with exactly one version" - bad=1 - continue - fi - - expected_cla_version=$( - git show "$sha:CLA.md" | sed -n 's/^Version //p' - ) - if [ -z "$expected_cla_version" ]; then - echo "Commit $short_sha contains an empty CLA version" - bad=1 - continue - fi - - commit_cla_version=$( - git show -s --format='%(trailers:key=CLA-Version,valueonly)' "$sha" - ) - if [ "$commit_cla_version" != "$expected_cla_version" ]; then - if [ -z "$commit_cla_version" ]; then - echo "Commit $short_sha is missing CLA-Version: $expected_cla_version" - else - echo "Commit $short_sha has invalid CLA-Version: $commit_cla_version" - echo "Expected CLA-Version: $expected_cla_version" - fi - bad=1 - fi - fi -done - -if [ "$bad" -ne 0 ]; then - echo "Every non-approved author and co-author must accept the CLA in their commit" - echo "Required trailers:" - echo " CLA-Version: <version from CLA.md>" - echo " Signed-off-by: Name <email>" - echo "See CLA.md" - exit 1 -fi - -if [ ! -f .mailmap ]; then - echo "Contributor registry .mailmap is missing" >&2 - exit 1 -fi - -missing=0 -for email in $(git log --no-merges --format='%ae%n%ce' | sort -u); do - if is_approved_email "$email"; then - continue - fi - - if ! grep -v '^[[:space:]]*#' .mailmap | grep -qF -- "<${email}>"; then - echo "Email <${email}> is not in .mailmap" - missing=1 - fi -done - -if [ "$missing" -ne 0 ]; then - echo "All authors and committers must be listed in .mailmap" - exit 1 -fi diff --git a/tasks/signoff.py b/tasks/signoff.py new file mode 100755 --- /dev/null +++ b/tasks/signoff.py @@ -0,0 +1,208 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# fmt: off +#MISE description="Verify contributor identities and CLA acceptance trailers" +#MISE tools={"pipx"="1.16.7","python"="3.14.7"} +# fmt: on +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import argparse +import os +import re +import sys +from collections.abc import Sequence +from pathlib import Path + +sys.dont_write_bytecode = True + +from libs.common import TaskError, capture, task_main +from shellous import sh + +TASK = "signoff" + + +def configured_values(path: Path) -> set[str]: + return { + line for line in path.read_text().splitlines() if not re.match(r"^\s*#", line) + } + + +def cla_version(document: str) -> tuple[int, str]: + versions = [ + line.removeprefix("Version ") + for line in document.splitlines() + if line.startswith("Version ") + ] + return len(versions), versions[0] if len(versions) == 1 else "" + + +def check_identity( + identity: str, + role: str, + short_sha: str, + approved_emails: set[str], + signoffs: set[str], +) -> tuple[bool, bool]: + match = re.fullmatch(r".*<([^<>]*)>", identity) + if match is None or not match.group(1): + print(f"Commit {short_sha} has an invalid {role} identity: {identity}") + return True, False + if match.group(1) in approved_emails: + return False, False + if identity not in signoffs: + print( + f"Commit {short_sha} {role} {identity} is missing a matching Signed-off-by" + ) + return True, True + return False, True + + +async def git(*arguments: str) -> str: + return await capture("git", arguments) + + +async def main(args: Sequence[str]) -> None: + argparse.ArgumentParser(prog="mise run signoff --").parse_args(args) + root = Path((await git("rev-parse", "--show-toplevel")).strip()) + os.chdir(root) + + task_directory = Path(__file__).resolve().parent + approved_emails_file = task_directory.parent / "config/signoff-approved-emails" + unsupported_commits_file = task_directory.parent / "config/cla-unsupported-commits" + + cla_file = Path("CLA.md") + if not cla_file.is_file(): + raise TaskError("CLA.md is missing") + version_count, head_version = cla_version(cla_file.read_text()) + if version_count != 1: + raise TaskError("CLA.md must declare exactly one version") + if not head_version: + raise TaskError("CLA.md declares an empty version") + if not approved_emails_file.is_file(): + raise TaskError( + f"Approved email configuration is missing: {approved_emails_file}" + ) + if not unsupported_commits_file.is_file(): + raise TaskError( + f"Unsupported commit configuration is missing: {unsupported_commits_file}" + ) + + approved_emails = configured_values(approved_emails_file) + unsupported_commits = configured_values(unsupported_commits_file) + bad = False + + commits = (await git("log", "--no-merges", "--format=%H")).splitlines() + for sha in commits: + short_sha = sha[:8] + signoffs = set( + ( + await git( + "show", + "-s", + "--format=%(trailers:key=Signed-off-by,valueonly)", + sha, + ) + ).splitlines() + ) + requires_cla = False + + author = (await git("show", "-s", "--format=%an <%ae>", sha)).rstrip("\n") + invalid, required = check_identity( + author, "author", short_sha, approved_emails, signoffs + ) + bad |= invalid + requires_cla |= required + + coauthor_output = ( + await git( + "show", + "-s", + "--format=%(trailers:key=Co-authored-by,valueonly)", + sha, + ) + ).rstrip("\n") + for coauthor in coauthor_output.splitlines() if coauthor_output else (): + invalid, required = check_identity( + coauthor, "co-author", short_sha, approved_emails, signoffs + ) + bad |= invalid + requires_cla |= required + + if requires_cla and sha not in unsupported_commits: + result = await capture.result("git", "show", f"{sha}:CLA.md").stderr( + sh.DEVNULL + ) + document = result.output if result.exit_code == 0 else "" + expected_count, expected_version = cla_version(document) + if expected_count != 1: + print( + f"Commit {short_sha} does not contain a CLA.md with exactly one version" + ) + bad = True + continue + if not expected_version: + print(f"Commit {short_sha} contains an empty CLA version") + bad = True + continue + + commit_version = ( + await git( + "show", + "-s", + "--format=%(trailers:key=CLA-Version,valueonly)", + sha, + ) + ).rstrip("\n") + if commit_version != expected_version: + if not commit_version: + print( + f"Commit {short_sha} is missing CLA-Version: {expected_version}" + ) + else: + print( + f"Commit {short_sha} has invalid CLA-Version: {commit_version}" + ) + print(f"Expected CLA-Version: {expected_version}") + bad = True + + if bad: + print( + "Every non-approved author and co-author must accept the CLA in their commit" + ) + print("Required trailers:") + print(" CLA-Version: <version from CLA.md>") + print(" Signed-off-by: Name <email>") + print("See CLA.md") + raise SystemExit(1) + + mailmap = Path(".mailmap") + if not mailmap.is_file(): + raise TaskError("Contributor registry .mailmap is missing") + mailmap_lines = [ + line + for line in mailmap.read_text().splitlines() + if not re.match(r"^\s*#", line) + ] + emails = sorted( + set((await git("log", "--no-merges", "--format=%ae%n%ce")).splitlines()) + ) + missing = False + for email in emails: + if email in approved_emails: + continue + if not any(f"<{email}>" in line for line in mailmap_lines): + print(f"Email <{email}> is not in .mailmap") + missing = True + if missing: + print("All authors and committers must be listed in .mailmap") + raise SystemExit(1) + + +if __name__ == "__main__": + task_main(TASK, main, sys.argv[1:]) diff --git a/tests/oci-tasks b/tests/oci-tasks deleted file mode 100755 --- a/tests/oci-tasks +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/sh -eu -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT -mkdir -p "$work/bin" "$work/project/chart/templates" -: > "$work/project/Dockerfile" -cat > "$work/project/chart/Chart.yaml" <<'EOF' -apiVersion: v2 -name: fixture -version: 0.0.0 -EOF -cat > "$work/project/chart/values.yaml" <<'EOF' -image: fixture -EOF -cat > "$work/project/chart/templates/configmap.yaml" <<'EOF' -apiVersion: v1 -kind: ConfigMap -metadata: - name: fixture -EOF - -cat > "$work/bin/docker" <<'EOF' -#!/bin/sh -printf '%s\n' "$@" > "$OCI_TEST_DOCKER_LOG" -EOF -cat > "$work/bin/helm" <<'EOF' -#!/bin/sh -printf '%s\n' "$@" >> "$OCI_TEST_HELM_LOG" -if [ "$1" = package ]; then - destination= - version= - shift - while [ "$#" -gt 0 ]; do - case "$1" in - --destination) destination=$2; shift 2 ;; - --version) version=$2; shift 2 ;; - *) shift ;; - esac - done - : > "$destination/fixture-$version.tgz" -fi -EOF -chmod +x "$work/bin/docker" "$work/bin/helm" -export PATH="$work/bin:$PATH" -export OCI_TEST_DOCKER_LOG="$work/docker.log" -export OCI_TEST_HELM_LOG="$work/helm.log" - -"$root/tasks/container" \ - --context "$work/project" \ - --platform linux/amd64 \ - --target site \ - --build-arg APP=site \ - --build-arg 'TITLE=hello world' \ - --label 'org.example.title=Example site' \ - --tag ghcr.io/example/site:sha-abc \ - --tag ghcr.io/example/site:latest \ - --cache-scope site \ - --provenance false \ - --sbom false \ - --push - -grep -qx -- 'buildx' "$work/docker.log" -grep -qx -- '--target' "$work/docker.log" -grep -qx -- 'site' "$work/docker.log" -grep -qx -- 'TITLE=hello world' "$work/docker.log" -grep -qx -- 'org.example.title=Example site' "$work/docker.log" -grep -qx -- 'ghcr.io/example/site:sha-abc' "$work/docker.log" -grep -qx -- 'ghcr.io/example/site:latest' "$work/docker.log" -grep -qx -- 'type=gha,mode=max,scope=site' "$work/docker.log" -grep -qx -- '--push' "$work/docker.log" - -"$root/tasks/chart" \ - --chart "$work/project/chart" \ - --version 1.2.3 \ - --app-version sha-abc \ - --output "$work/output" \ - --push oci://ghcr.io/example/charts - -grep -qx -- 'lint' "$work/helm.log" -grep -qx -- 'package' "$work/helm.log" -grep -qx -- 'push' "$work/helm.log" -grep -qx -- 'oci://ghcr.io/example/charts' "$work/helm.log" - -echo 'oci tasks: ok' diff --git a/tests/oci-tasks.py b/tests/oci-tasks.py new file mode 100755 --- /dev/null +++ b/tests/oci-tasks.py @@ -0,0 +1,147 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from shellous import sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) + + +def executable(path: Path, source: str) -> None: + path.write_text("#!/usr/bin/env python3\n" + source) + path.chmod(0o755) + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + binary = work / "bin" + project = work / "project" + chart = project / "chart" + (chart / "templates").mkdir(parents=True) + (project / "Dockerfile").touch() + (chart / "Chart.yaml").write_text( + "apiVersion: v2\nname: fixture\nversion: 0.0.0\n" + ) + (chart / "values.yaml").write_text("image: fixture\n") + (chart / "templates/configmap.yaml").write_text( + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: fixture\n" + ) + binary.mkdir() + executable( + binary / "docker", + """import os +import sys +from pathlib import Path +Path(os.environ["OCI_TEST_DOCKER_LOG"]).write_text("\\n".join(sys.argv[1:]) + "\\n") +""", + ) + executable( + binary / "helm", + """import os +import sys +from pathlib import Path +arguments = sys.argv[1:] +log = Path(os.environ["OCI_TEST_HELM_LOG"]) +with log.open("a") as stream: + stream.write("\\n".join(arguments) + "\\n") +if arguments and arguments[0] == "package": + destination = Path(arguments[arguments.index("--destination") + 1]) + version = arguments[arguments.index("--version") + 1] + (destination / f"fixture-{version}.tgz").touch() +""", + ) + + docker_log = work / "docker.log" + helm_log = work / "helm.log" + environment = dict(os.environ) + environment.update( + PATH=f"{binary}:{environment['PATH']}", + OCI_TEST_DOCKER_LOG=str(docker_log), + OCI_TEST_HELM_LOG=str(helm_log), + ) + command = run.set(env=environment, inherit_env=False) + + await command( + root / "tasks/container.py", + "--context", + project, + "--platform", + "linux/amd64", + "--target", + "site", + "--build-arg", + "APP=site", + "--build-arg", + "TITLE=hello world", + "--label", + "org.example.title=Example site", + "--tag", + "ghcr.io/example/site:sha-abc", + "--tag", + "ghcr.io/example/site:latest", + "--cache-scope", + "site", + "--provenance", + "false", + "--sbom", + "false", + "--push", + ) + docker_arguments = docker_log.read_text().splitlines() + for expected in ( + "buildx", + "--target", + "site", + "TITLE=hello world", + "org.example.title=Example site", + "ghcr.io/example/site:sha-abc", + "ghcr.io/example/site:latest", + "type=gha,mode=max,scope=site", + "--push", + ): + assert expected in docker_arguments + + await command( + root / "tasks/chart.py", + "--chart", + chart, + "--version", + "1.2.3", + "--app-version", + "sha-abc", + "--output", + work / "output", + "--push", + "oci://ghcr.io/example/charts", + ) + helm_arguments = helm_log.read_text().splitlines() + for expected in ( + "lint", + "package", + "push", + "oci://ghcr.io/example/charts", + ): + assert expected in helm_arguments + + print("oci tasks: ok") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/tests/package b/tests/package deleted file mode 100755 --- a/tests/package +++ /dev/null @@ -1,177 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -set -eu - -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM - -mkdir -p "$work/bin" "$work/stage/sub" "$work/out" -printf 'payload\n' > "$work/stage/tool" -printf 'nested\n' > "$work/stage/sub/file" -printf 'outside\n' > "$work/outside" -ln -s "$work/outside" "$work/stage/outside-link" - -cat > "$work/bin/nfpm" <<'EOF' -#!/bin/sh -set -eu -[ -z "${GPG_PRIVATE_KEY:-}" ] && \ -[ -z "${APK_PRIVATE_KEY:-}" ] && \ -[ -z "${SIGNING_PRIVATE_KEY:-}" ] && \ -[ -z "${NFPM_PASSPHRASE:-}" ] || { - echo 'raw private key leaked to nFPM' >&2 - exit 1 -} -printf 'ARCH=%s\nVERSION=%s\nGPG_KEY_ID=%s\n' "$ARCH" "$VERSION" "${GPG_KEY_ID:-}" >> "$PACKAGE_TEST_LOG" -while [ "$#" -gt 0 ]; do - case "$1" in - --config) config=$2; shift 2 ;; - --packager) packager=$2; shift 2 ;; - --target) target=$2; shift 2 ;; - *) shift ;; - esac -done -printf '%s\n' "$packager" >> "$PACKAGE_TEST_LOG" -grep '^key_name:' "$config" >> "$PACKAGE_TEST_LOG" || true -: > "$target/test.$packager" -EOF - -cat > "$work/bin/gpg" <<'EOF' -#!/bin/sh -set -eu -printf 'gpg\n' >> "$PACKAGE_TEST_LOG" -while [ "$#" -gt 0 ]; do - case "$1" in - -o) output=$2; shift 2 ;; - *) shift ;; - esac -done -[ -z "${output:-}" ] || printf signature > "$output" -EOF - -for command in debsigs rpmsign; do - cat > "$work/bin/$command" <<'EOF' -#!/bin/sh -set -eu -printf '%s\n' "${0##*/}" >> "$PACKAGE_TEST_LOG" -EOF - chmod +x "$work/bin/$command" -done - -cat > "$work/bin/openssl" <<'EOF' -#!/bin/sh -set -eu -while [ "$#" -gt 0 ]; do - case "$1" in - -out) output=$2; shift 2 ;; - *) shift ;; - esac -done -printf 'public key\n' > "$output" -EOF - -chmod +x "$work/bin/nfpm" "$work/bin/gpg" "$work/bin/openssl" -printf 'name: test\n' > "$work/nfpm.yaml" -export PACKAGE_TEST_LOG="$work/package.log" -PATH="$work/bin:$PATH" -export PATH - -"$root/tasks/package" \ - --output "$work/out" \ - --archive-root "$work/stage" \ - --archive-name test-linux-amd64 \ - tar.gz zip - -test -f "$work/out/test-linux-amd64.tar.gz" -test -f "$work/out/test-linux-amd64.zip" -tar -tzf "$work/out/test-linux-amd64.tar.gz" | grep -q './tool' -unzip -l "$work/out/test-linux-amd64.zip" | grep -q 'sub/file' -test "$(unzip -p "$work/out/test-linux-amd64.zip" outside-link)" = "$work/outside" - -"$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version '1.2.3~nightly.42' \ - --arch arm64 \ - --output "$work/out" \ - deb rpm - -test -f "$work/out/test.deb" -test -f "$work/out/test.rpm" -grep -q '^ARCH=arm64$' "$work/package.log" -grep -q '^VERSION=1.2.3~nightly.42$' "$work/package.log" - -GPG_PRIVATE_KEY=private \ -GPG_PASSPHRASE=passphrase \ -GPG_KEY_ID=0123456789ABCDEF0123456789ABCDEF01234567 \ -APK_PRIVATE_KEY=apk-private \ - "$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - --apk-public-key test.rsa.pub \ - deb rpm apk - -test -f "$work/out/test.apk" -test -f "$work/out/test.rsa.pub" -grep -q '^gpg$' "$work/package.log" -grep -q '^debsigs$' "$work/package.log" -grep -q '^rpmsign$' "$work/package.log" -grep -q '^GPG_KEY_ID=89ABCDEF01234567$' "$work/package.log" - -# Repository publication exports the organization APK public key, so package -# builds do not need to stage a per-service public key artifact. -APK_PRIVATE_KEY=apk-private \ - "$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - apk - -printf 'key_name: packages.${PACKAGE_KEY_VERSION}\n' > "$work/versioned.yaml" -if APK_PRIVATE_KEY=apk-private "$root/tasks/package" \ - --config "$work/versioned.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - apk 2>/dev/null; then - echo 'package test: accepted versioned APK config without key version' >&2 - exit 1 -fi -APK_PRIVATE_KEY=apk-private \ -PACKAGE_KEY_VERSION=0001 \ - "$root/tasks/package" \ - --config "$work/versioned.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - apk -grep -q '^key_name: packages.0001$' "$work/package.log" - -if "$root/tasks/package" --output "$work/out" deb 2>/dev/null; then - echo 'package test: accepted system format without version and architecture' >&2 - exit 1 -fi - -if "$root/tasks/package" \ - --output "$work/out" \ - --archive-root "$work/stage" \ - --archive-name '../escape' \ - zip 2>/dev/null; then - echo 'package test: accepted unsafe archive name' >&2 - exit 1 -fi - -if APK_SIGNING_KEY="$work/apk.rsa" "$root/tasks/package" \ - --config "$work/nfpm.yaml" \ - --version 1.2.3 \ - --arch amd64 \ - --output "$work/out" \ - --apk-public-key '../escape' \ - apk 2>/dev/null; then - echo 'package test: accepted unsafe APK public key name' >&2 - exit 1 -fi diff --git a/tests/package-integration b/tests/package-integration deleted file mode 100755 --- a/tests/package-integration +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -set -eu - -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM - -mkdir -p "$work/stage" -printf 'package contract\n' > "$work/stage/package-contract" -cat > "$work/nfpm.yaml" <<'EOF' -name: package-contract -arch: ${ARCH} -version: ${VERSION} -platform: linux -maintainer: Dimidium Labs <me@govorov.online> -description: Shared package task integration fixture -license: 0BSD -EOF - -mise --cd "$root" run package -- \ - --config "$work/nfpm.yaml" \ - --version '1.2.3~nightly.42' \ - --arch amd64 \ - --output "$work/out" \ - --archive-root "$work/stage" \ - --archive-name package-contract-linux-amd64 \ - deb rpm apk tar.gz zip - -test "$(find "$work/out" -maxdepth 1 -name '*.deb' | wc -l)" -eq 1 -test "$(find "$work/out" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 1 -test "$(find "$work/out" -maxdepth 1 -name '*.apk' | wc -l)" -eq 1 -test -f "$work/out/package-contract-linux-amd64.tar.gz" -test -f "$work/out/package-contract-linux-amd64.zip" - -test "$(dpkg-deb -f "$work/out"/*.deb Package)" = package-contract -test "$(dpkg-deb -f "$work/out"/*.deb Version)" = '1.2.3~nightly.42' -test "$(dpkg-deb -f "$work/out"/*.deb Architecture)" = amd64 -tar -tzf "$work/out/package-contract-linux-amd64.tar.gz" | grep -q './package-contract' -unzip -l "$work/out/package-contract-linux-amd64.zip" | grep -q 'package-contract' diff --git a/tests/package-integration.py b/tests/package-integration.py new file mode 100755 --- /dev/null +++ b/tests/package-integration.py @@ -0,0 +1,93 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from shellous import sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + stage = work / "stage" + stage.mkdir() + (stage / "package-contract").write_text("package contract\n") + config = work / "nfpm.yaml" + config.write_text( + """name: package-contract +arch: ${ARCH} +version: ${VERSION} +platform: linux +maintainer: Dimidium Labs <me@govorov.online> +description: Shared package task integration fixture +license: 0BSD +""" + ) + output = work / "out" + await run( + "mise", + "--cd", + root, + "run", + "package", + "--", + "--config", + config, + "--version", + "1.2.3~nightly.42", + "--arch", + "amd64", + "--output", + output, + "--archive-root", + stage, + "--archive-name", + "package-contract-linux-amd64", + "deb", + "rpm", + "apk", + "tar.gz", + "zip", + ) + + assert len(list(output.glob("*.deb"))) == 1 + assert len(list(output.glob("*.rpm"))) == 1 + assert len(list(output.glob("*.apk"))) == 1 + tarball = output / "package-contract-linux-amd64.tar.gz" + zipfile = output / "package-contract-linux-amd64.zip" + assert tarball.is_file() + assert zipfile.is_file() + + package = next(output.glob("*.deb")) + assert (await capture("dpkg-deb", "-f", package, "Package")).strip() == ( + "package-contract" + ) + assert (await capture("dpkg-deb", "-f", package, "Version")).strip() == ( + "1.2.3~nightly.42" + ) + assert (await capture("dpkg-deb", "-f", package, "Architecture")).strip() == ( + "amd64" + ) + assert "./package-contract" in await capture("tar", "-tzf", tarball) + assert "package-contract" in await capture("unzip", "-l", zipfile) + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/tests/package.py b/tests/package.py new file mode 100755 --- /dev/null +++ b/tests/package.py @@ -0,0 +1,270 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import os +import sys +import tempfile +from collections.abc import Sequence +from pathlib import Path + +from shellous import sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) + + +def executable(path: Path, source: str) -> None: + path.write_text("#!/usr/bin/env python3\n" + source) + path.chmod(0o755) + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + binary = work / "bin" + stage = work / "stage" + output = work / "out" + binary.mkdir() + (stage / "sub").mkdir(parents=True) + output.mkdir() + (stage / "tool").write_text("payload\n") + (stage / "sub/file").write_text("nested\n") + outside = work / "outside" + outside.write_text("outside\n") + (stage / "outside-link").symlink_to(outside) + + executable( + binary / "nfpm", + """import os +import sys +from pathlib import Path +secrets = ( + "GPG_PRIVATE_KEY", "APK_PRIVATE_KEY", "SIGNING_PRIVATE_KEY", + "NFPM_PASSPHRASE", +) +if any(os.environ.get(name) for name in secrets): + print("raw private key leaked to nFPM", file=sys.stderr) + raise SystemExit(1) +arguments = sys.argv[1:] +config = Path(arguments[arguments.index("--config") + 1]) +packager = arguments[arguments.index("--packager") + 1] +target = Path(arguments[arguments.index("--target") + 1]) +with Path(os.environ["PACKAGE_TEST_LOG"]).open("a") as stream: + stream.write(f"ARCH={os.environ['ARCH']}\\n") + stream.write(f"VERSION={os.environ['VERSION']}\\n") + stream.write(f"GPG_KEY_ID={os.environ.get('GPG_KEY_ID', '')}\\n") + stream.write(f"{packager}\\n") + for line in config.read_text().splitlines(): + if line.startswith("key_name:"): + stream.write(f"{line}\\n") +(target / f"test.{packager}").touch() +""", + ) + executable( + binary / "gpg", + """import os +import sys +from pathlib import Path +with Path(os.environ["PACKAGE_TEST_LOG"]).open("a") as stream: + stream.write("gpg\\n") +arguments = sys.argv[1:] +if "-o" in arguments: + Path(arguments[arguments.index("-o") + 1]).write_text("signature") +""", + ) + for name in ("debsigs", "rpmsign"): + executable( + binary / name, + """import os +import sys +from pathlib import Path +with Path(os.environ["PACKAGE_TEST_LOG"]).open("a") as stream: + stream.write(f"{Path(sys.argv[0]).name}\\n") +""", + ) + executable( + binary / "openssl", + """import sys +from pathlib import Path +arguments = sys.argv[1:] +Path(arguments[arguments.index("-out") + 1]).write_text("public key\\n") +""", + ) + + config = work / "nfpm.yaml" + config.write_text("name: test\n") + log = work / "package.log" + environment = dict(os.environ) + environment.update( + PATH=f"{binary}:{environment['PATH']}", + PACKAGE_TEST_LOG=str(log), + ) + command = run.set(env=environment, inherit_env=False) + package = root / "tasks/package.py" + + await command( + package, + "--output", + output, + "--archive-root", + stage, + "--archive-name", + "test-linux-amd64", + "tar.gz", + "zip", + ) + tarball = output / "test-linux-amd64.tar.gz" + zipfile = output / "test-linux-amd64.zip" + assert tarball.is_file() + assert zipfile.is_file() + assert "./tool" in await capture("tar", "-tzf", tarball) + assert "sub/file" in await capture("unzip", "-l", zipfile) + assert (await capture("unzip", "-p", zipfile, "outside-link")) == str(outside) + + await command( + package, + "--config", + config, + "--version", + "1.2.3~nightly.42", + "--arch", + "arm64", + "--output", + output, + "deb", + "rpm", + ) + assert (output / "test.deb").is_file() + assert (output / "test.rpm").is_file() + assert "ARCH=arm64" in log.read_text().splitlines() + assert "VERSION=1.2.3~nightly.42" in log.read_text().splitlines() + + signing_environment = dict(environment) + signing_environment.update( + GPG_PRIVATE_KEY="private", + GPG_PASSPHRASE="passphrase", + GPG_KEY_ID="0123456789ABCDEF0123456789ABCDEF01234567", + APK_PRIVATE_KEY="apk-private", + ) + await run.set(env=signing_environment, inherit_env=False)( + package, + "--config", + config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "--apk-public-key", + "test.rsa.pub", + "deb", + "rpm", + "apk", + ) + assert (output / "test.apk").is_file() + assert (output / "test.rsa.pub").is_file() + log_lines = log.read_text().splitlines() + for expected in ( + "gpg", + "debsigs", + "rpmsign", + "GPG_KEY_ID=89ABCDEF01234567", + ): + assert expected in log_lines + + apk_environment = dict(environment, APK_PRIVATE_KEY="apk-private") + await run.set(env=apk_environment, inherit_env=False)( + package, + "--config", + config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "apk", + ) + + versioned_config = work / "versioned.yaml" + versioned_config.write_text("key_name: packages.${PACKAGE_KEY_VERSION}\n") + result = await run.result.set(env=apk_environment, inherit_env=False)( + package, + "--config", + versioned_config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "apk", + ).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + versioned_environment = dict(apk_environment, PACKAGE_KEY_VERSION="0001") + await run.set(env=versioned_environment, inherit_env=False)( + package, + "--config", + versioned_config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "apk", + ) + assert "key_name: packages.0001" in log.read_text().splitlines() + + invalid_commands = ( + (package, "--output", output, "deb"), + ( + package, + "--output", + output, + "--archive-root", + stage, + "--archive-name", + "../escape", + "zip", + ), + ) + for arguments in invalid_commands: + result = await command.result(*arguments).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + external_key_environment = dict( + environment, APK_SIGNING_KEY=str(work / "apk.rsa") + ) + result = await run.result.set(env=external_key_environment, inherit_env=False)( + package, + "--config", + config, + "--version", + "1.2.3", + "--arch", + "amd64", + "--output", + output, + "--apk-public-key", + "../escape", + "apk", + ).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/tests/publish-integration b/tests/publish-integration deleted file mode 100755 --- a/tests/publish-integration +++ /dev/null @@ -1,185 +0,0 @@ -#!/bin/sh -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: 0BSD - -set -eu - -root=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT HUP INT TERM -remote="$work/remote/integration" -mkdir -p "$work/bin" "$work/input" "$remote" "$work/fixture" - -printf 'publish integration fixture\n' > "$work/fixture/publish-contract" -cat > "$work/fixture/nfpm.yaml" <<EOF -name: publish-contract -arch: \${ARCH} -version: \${VERSION} -platform: linux -maintainer: Dimidium Labs <me@govorov.online> -description: Shared publish task integration fixture -license: 0BSD -contents: - - src: $work/fixture/publish-contract - dst: /usr/local/bin/publish-contract -deb: - signature: - method: debsign - key_id: \${GPG_KEY_ID} - key_file: \${SIGNING_PRIVATE_KEY} -rpm: - signature: - key_id: \${GPG_KEY_ID} - key_file: \${SIGNING_PRIVATE_KEY} -apk: - signature: - key_file: \${APK_SIGNING_KEY} - key_name: packages.\${PACKAGE_KEY_VERSION} -EOF - -export GNUPGHOME="$work/source-gnupg" -mkdir -m 700 "$GNUPGHOME" -gpg --batch --pinentry-mode loopback --passphrase integration-pass \ - --quick-generate-key 'Publish Integration <publish@example.invalid>' rsa2048 sign 1d >/dev/null -fingerprint=$(gpg --batch --with-colons --list-secret-keys | - awk -F: '$1 == "fpr" { print $10; exit }') -gpg --batch --pinentry-mode loopback --passphrase integration-pass \ - --armor --export-secret-keys "$fingerprint" > "$work/private.gpg" -openssl genrsa -out "$work/private.rsa" 2048 >/dev/null 2>&1 -# Organization keys are provisioned independently under a versioned archive. -key_version=0001 -mkdir -p "$remote/keys" -gpg --batch --armor --export "$fingerprint" > "$remote/packages.gpg" -cp "$remote/packages.gpg" "$remote/keys/packages.$key_version.gpg" -openssl rsa -in "$work/private.rsa" -pubout \ - -out "$remote/keys/packages.$key_version.rsa.pub" >/dev/null 2>&1 - -gpg_private_key=$(cat "$work/private.gpg") -apk_private_key=$(cat "$work/private.rsa") - -package_version() { - version=$1 - GPG_PRIVATE_KEY="$gpg_private_key" \ - GPG_PASSPHRASE=integration-pass \ - GPG_KEY_ID="$fingerprint" \ - APK_PRIVATE_KEY="$apk_private_key" \ - PACKAGE_KEY_VERSION="$key_version" \ - mise --cd "$root" run package -- \ - --config "$work/fixture/nfpm.yaml" \ - --version "$version" \ - --arch amd64 \ - --output "$work/input" \ - deb rpm apk -} - -publish() { - PUBLISH_REMOTE="$work/remote" \ - PUBLISH_TEST_LOG="$work/s3.log" \ - PYTHONPATH="$root/tests/fakes" \ - PATH="$work/bin:$PATH" \ - S3_BUCKET=integration \ - S3_ENDPOINT=https://example.invalid \ - S3_PUBLIC_URL=https://pkg.dimidiumlabs.io \ - S3_ACCESS_KEY_ID=integration \ - S3_SECRET_ACCESS_KEY=integration \ - GPG_PRIVATE_KEY="$gpg_private_key" \ - GPG_PASSPHRASE=integration-pass \ - GPG_KEY_ID="$fingerprint" \ - APK_PRIVATE_KEY="$apk_private_key" \ - PACKAGE_KEY_VERSION="$key_version" \ - "$root/tasks/publish" \ - --service publish-contract \ - --channel nightly \ - --input "$work/input" \ - deb rpm apk -} - -package_version '1.2.3~nightly.42' -publish - -apt_root="$remote/publish-contract/apt" -rpm_root="$remote/publish-contract/rpm/nightly" -apk_root="$remote/publish-contract/apk/nightly/x86_64" - -test -f "$remote/packages.gpg" -test -f "$remote/keys/packages.$key_version.gpg" -test -f "$remote/keys/packages.$key_version.rsa.pub" -test "$(find "$apt_root/pool/nightly" -name '*.deb' | wc -l)" -eq 1 -test "$(find "$rpm_root" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 1 -test "$(find "$apk_root" -maxdepth 1 -name '*.apk' | wc -l)" -eq 1 -grep -q '^Package: publish-contract$' \ - "$apt_root/dists/nightly/main/binary-amd64/Packages" -grep -q 'baseurl=https://pkg.dimidiumlabs.io/publish-contract/rpm/nightly/' \ - "$rpm_root/publish-contract-nightly.repo" - -keyring="$work/packages.gpg" -gpg --batch --dearmor -o "$keyring" "$remote/packages.gpg" -gpgv --keyring "$keyring" "$apt_root/dists/nightly/InRelease" >/dev/null -gpgv --keyring "$keyring" "$rpm_root/repodata/repomd.xml.asc" \ - "$rpm_root/repodata/repomd.xml" >/dev/null - -apk_tool=$(find "$HOME/.cache/mise" "$HOME/.local/share/mise" -type f -name apk.static 2>/dev/null | head -n 1 || true) -if [ -z "$apk_tool" ]; then - archive="$work/apk-tools-static.apk" - curl -fsSL \ - https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/apk-tools-static-2.14.10-r0.apk \ - -o "$archive" - printf '%s %s\n' c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61 "$archive" | - sha256sum -c - >/dev/null - mkdir -p "$work/apk-tools" - tar -xzf "$archive" -C "$work/apk-tools" sbin/apk.static 2>/dev/null - apk_tool="$work/apk-tools/sbin/apk.static" -fi -mkdir "$work/apk-keys" -cp "$remote/keys/packages.$key_version.rsa.pub" "$work/apk-keys/" -"$apk_tool" verify --keys-dir "$work/apk-keys" "$apk_root/APKINDEX.tar.gz" >/dev/null - -# A later publication keeps every package payload, regenerates metadata, and -# removes stale metadata only inside the corresponding repository prefixes. -printf stale > "$apt_root/dists/nightly/stale" -printf stale > "$rpm_root/repodata/stale" -rm -rf "$work/input" -mkdir "$work/input" -package_version '1.2.3~nightly.43' -publish -test "$(find "$apt_root/pool/nightly" -name '*.deb' | wc -l)" -eq 2 -test "$(find "$rpm_root" -maxdepth 1 -name '*.rpm' | wc -l)" -eq 2 -test "$(find "$apk_root" -maxdepth 1 -name '*.apk' | wc -l)" -eq 2 -test ! -e "$apt_root/dists/nightly/stale" -test ! -e "$rpm_root/repodata/stale" -test "$(grep -c '^Package: publish-contract$' \ - "$apt_root/dists/nightly/main/binary-amd64/Packages")" -eq 2 -"$apk_tool" verify --keys-dir "$work/apk-keys" "$apk_root/APKINDEX.tar.gz" >/dev/null - -# Existing package names and organization keys are immutable. -deb=$(find "$work/input" -maxdepth 1 -name '*.deb') -printf '\nchanged\n' >> "$deb" -if publish 2>/dev/null; then - echo 'publish integration: replaced an immutable package payload' >&2 - exit 1 -fi -cp "$apt_root/pool/nightly/${deb##*/}" "$deb" -lock="$remote/publish-contract/_locks/nightly" -mkdir -p "${lock%/*}" -printf '{"expires":9999999999}\n' > "$lock" -if publish 2>/dev/null; then - echo 'publish integration: ignored an active publication lock' >&2 - exit 1 -fi -rm "$lock" -printf 'different key\n' > "$remote/keys/packages.$key_version.gpg" -if publish 2>/dev/null; then - echo 'publish integration: ignored a mismatched versioned key' >&2 - exit 1 -fi - -# Metadata cleanup must never escape generated repository prefixes. -if awk '$1 == "s3" && $2 == "delete" && $3 !~ /^publish-contract\/(apt\/dists|rpm\/nightly\/repodata|_locks\/nightly)/' "$work/s3.log" | grep . >/dev/null; then - echo 'publish integration: deleted outside a metadata prefix' >&2 - exit 1 -fi -if PYTHONPATH="$root/tests/fakes" "$root/tasks/publish" \ - --service '../escape' --channel nightly --input "$work/input" deb 2>/dev/null; then - echo 'publish integration: accepted an unsafe service name' >&2 - exit 1 -fi diff --git a/tests/publish-integration.py b/tests/publish-integration.py new file mode 100755 --- /dev/null +++ b/tests/publish-integration.py @@ -0,0 +1,346 @@ +#!/usr/bin/env -S pipx run --backend pip +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: 0BSD +# /// script +# requires-python = ">=3.11" +# dependencies = ["shellous==0.42.0"] +# /// + +from __future__ import annotations + +import asyncio +import hashlib +import os +import re +import shutil +import sys +import tempfile +import urllib.request +from collections.abc import Sequence +from pathlib import Path + +from shellous import Result, sh + +run = sh.stdout(sh.INHERIT).stderr(sh.INHERIT) +capture = sh.stderr(sh.INHERIT) +APK_TOOLS_URL = ( + "https://dl-cdn.alpinelinux.org/alpine/v3.22/main/x86_64/" + "apk-tools-static-2.14.10-r0.apk" +) +APK_TOOLS_SHA256 = "c86e3822764e5fe19f41ce2e13553e48cac1ea4e74f858338e8d44bf0b616b61" + + +async def main(args: Sequence[str]) -> None: + if args: + raise SystemExit(f"unexpected arguments: {' '.join(args)}") + root = Path(__file__).resolve().parent.parent + with tempfile.TemporaryDirectory() as directory: + work = Path(directory) + remote = work / "remote/integration" + binary = work / "bin" + package_input = work / "input" + fixture = work / "fixture" + for path in (remote, binary, package_input, fixture): + path.mkdir(parents=True) + + payload = fixture / "publish-contract" + payload.write_text("publish integration fixture\n") + config = fixture / "nfpm.yaml" + config.write_text( + f"""name: publish-contract +arch: ${{ARCH}} +version: ${{VERSION}} +platform: linux +maintainer: Dimidium Labs <me@govorov.online> +description: Shared publish task integration fixture +license: 0BSD +contents: + - src: {payload} + dst: /usr/local/bin/publish-contract +deb: + signature: + method: debsign + key_id: ${{GPG_KEY_ID}} + key_file: ${{SIGNING_PRIVATE_KEY}} +rpm: + signature: + key_id: ${{GPG_KEY_ID}} + key_file: ${{SIGNING_PRIVATE_KEY}} +apk: + signature: + key_file: ${{APK_SIGNING_KEY}} + key_name: packages.${{PACKAGE_KEY_VERSION}} +""" + ) + + source_gnupg = work / "source-gnupg" + source_gnupg.mkdir(mode=0o700) + source_environment = dict(os.environ, GNUPGHOME=str(source_gnupg)) + gpg = run.set(env=source_environment, inherit_env=False) + gpg_capture = capture.set(env=source_environment, inherit_env=False) + await gpg( + "gpg", + "--batch", + "--pinentry-mode", + "loopback", + "--passphrase", + "integration-pass", + "--quick-generate-key", + "Publish Integration <publish@example.invalid>", + "rsa2048", + "sign", + "1d", + ).stdout(sh.DEVNULL) + key_listing = await gpg_capture( + "gpg", "--batch", "--with-colons", "--list-secret-keys" + ) + fingerprint = next( + fields[9] + for line in key_listing.splitlines() + if (fields := line.split(":"))[0] == "fpr" + ) + private_gpg = work / "private.gpg" + await gpg( + "gpg", + "--batch", + "--pinentry-mode", + "loopback", + "--passphrase", + "integration-pass", + "--armor", + "--export-secret-keys", + fingerprint, + ).stdout(private_gpg) + private_rsa = work / "private.rsa" + await ( + run("openssl", "genrsa", "-out", private_rsa, "2048") + .stdout(sh.DEVNULL) + .stderr(sh.DEVNULL) + ) + + key_version = "0001" + keys = remote / "keys" + keys.mkdir() + public_gpg = remote / "packages.gpg" + await gpg("gpg", "--batch", "--armor", "--export", fingerprint).stdout( + public_gpg + ) + shutil.copy2(public_gpg, keys / f"packages.{key_version}.gpg") + await ( + run( + "openssl", + "rsa", + "-in", + private_rsa, + "-pubout", + "-out", + keys / f"packages.{key_version}.rsa.pub", + ) + .stdout(sh.DEVNULL) + .stderr(sh.DEVNULL) + ) + gpg_private_key = private_gpg.read_text() + apk_private_key = private_rsa.read_text() + + async def package_version(version: str) -> None: + environment = dict(os.environ) + environment.update( + GPG_PRIVATE_KEY=gpg_private_key, + GPG_PASSPHRASE="integration-pass", + GPG_KEY_ID=fingerprint, + APK_PRIVATE_KEY=apk_private_key, + PACKAGE_KEY_VERSION=key_version, + ) + await run.set(env=environment, inherit_env=False)( + "mise", + "--cd", + root, + "run", + "package", + "--", + "--config", + config, + "--version", + version, + "--arch", + "amd64", + "--output", + package_input, + "deb", + "rpm", + "apk", + ) + + publish_environment = dict(os.environ) + publish_environment.update( + PUBLISH_REMOTE=str(work / "remote"), + PUBLISH_TEST_LOG=str(work / "s3.log"), + PUBLISH_TEST_PYTHONPATH=str(root / "tests/fakes"), + PATH=f"{binary}:{os.environ['PATH']}", + S3_BUCKET="integration", + S3_ENDPOINT="https://example.invalid", + S3_PUBLIC_URL="https://pkg.dimidiumlabs.io", + S3_ACCESS_KEY_ID="integration", + S3_SECRET_ACCESS_KEY="integration", + GPG_PRIVATE_KEY=gpg_private_key, + GPG_PASSPHRASE="integration-pass", + GPG_KEY_ID=fingerprint, + APK_PRIVATE_KEY=apk_private_key, + PACKAGE_KEY_VERSION=key_version, + ) + publish_command = run.result.set(env=publish_environment, inherit_env=False) + + async def publish(*, quiet: bool = False) -> Result: + command = publish_command( + root / "tasks/publish.py", + "--service", + "publish-contract", + "--channel", + "nightly", + "--input", + package_input, + "deb", + "rpm", + "apk", + ) + if quiet: + command = command.stderr(sh.DEVNULL) + return await command + + await package_version("1.2.3~nightly.42") + assert (await publish()).exit_code == 0 + + apt_root = remote / "publish-contract/apt" + rpm_root = remote / "publish-contract/rpm/nightly" + apk_root = remote / "publish-contract/apk/nightly/x86_64" + assert public_gpg.is_file() + assert (keys / f"packages.{key_version}.gpg").is_file() + assert (keys / f"packages.{key_version}.rsa.pub").is_file() + assert len(list((apt_root / "pool/nightly").glob("*.deb"))) == 1 + assert len(list(rpm_root.glob("*.rpm"))) == 1 + assert len(list(apk_root.glob("*.apk"))) == 1 + packages_file = apt_root / "dists/nightly/main/binary-amd64/Packages" + assert "Package: publish-contract" in packages_file.read_text().splitlines() + repository_file = rpm_root / "publish-contract-nightly.repo" + assert ( + "baseurl=https://pkg.dimidiumlabs.io/publish-contract/rpm/nightly/" + in repository_file.read_text().splitlines() + ) + + keyring = work / "packages.gpg" + await run("gpg", "--batch", "--dearmor", "-o", keyring, public_gpg) + await run( + "gpgv", + "--keyring", + keyring, + apt_root / "dists/nightly/InRelease", + ).stdout(sh.DEVNULL) + await run( + "gpgv", + "--keyring", + keyring, + rpm_root / "repodata/repomd.xml.asc", + rpm_root / "repodata/repomd.xml", + ).stdout(sh.DEVNULL) + + apk_tool = next( + ( + path + for base in ( + Path.home() / ".cache/mise", + Path.home() / ".local/share/mise", + ) + if base.is_dir() + for path in base.rglob("apk.static") + if path.is_file() + ), + None, + ) + if apk_tool is None: + archive = work / "apk-tools-static.apk" + await asyncio.to_thread(urllib.request.urlretrieve, APK_TOOLS_URL, archive) + assert hashlib.sha256(archive.read_bytes()).hexdigest() == APK_TOOLS_SHA256 + apk_directory = work / "apk-tools" + apk_directory.mkdir() + await run( + "tar", + "-xzf", + archive, + "-C", + apk_directory, + "sbin/apk.static", + ).stderr(sh.DEVNULL) + apk_tool = apk_directory / "sbin/apk.static" + apk_keys = work / "apk-keys" + apk_keys.mkdir() + shutil.copy2(keys / f"packages.{key_version}.rsa.pub", apk_keys) + await run( + apk_tool, + "verify", + "--keys-dir", + apk_keys, + apk_root / "APKINDEX.tar.gz", + ).stdout(sh.DEVNULL) + + (apt_root / "dists/nightly/stale").write_text("stale") + (rpm_root / "repodata/stale").write_text("stale") + shutil.rmtree(package_input) + package_input.mkdir() + await package_version("1.2.3~nightly.43") + assert (await publish()).exit_code == 0 + assert len(list((apt_root / "pool/nightly").glob("*.deb"))) == 2 + assert len(list(rpm_root.glob("*.rpm"))) == 2 + assert len(list(apk_root.glob("*.apk"))) == 2 + assert not (apt_root / "dists/nightly/stale").exists() + assert not (rpm_root / "repodata/stale").exists() + assert ( + packages_file.read_text().splitlines().count("Package: publish-contract") + == 2 + ) + await run( + apk_tool, + "verify", + "--keys-dir", + apk_keys, + apk_root / "APKINDEX.tar.gz", + ).stdout(sh.DEVNULL) + + deb = next(package_input.glob("*.deb")) + with deb.open("a") as stream: + stream.write("\nchanged\n") + assert (await publish(quiet=True)).exit_code != 0 + shutil.copy2(apt_root / "pool/nightly" / deb.name, deb) + lock = remote / "publish-contract/_locks/nightly" + lock.parent.mkdir(parents=True, exist_ok=True) + lock.write_text('{"expires":9999999999}\n') + assert (await publish(quiet=True)).exit_code != 0 + lock.unlink() + (keys / f"packages.{key_version}.gpg").write_text("different key\n") + assert (await publish(quiet=True)).exit_code != 0 + + allowed_delete = re.compile( + r"^publish-contract/(apt/dists|rpm/nightly/repodata|_locks/nightly)" + ) + for line in (work / "s3.log").read_text().splitlines(): + fields = line.split(maxsplit=2) + if len(fields) == 3 and fields[:2] == ["s3", "delete"]: + assert allowed_delete.match(fields[2]) + + invalid_environment = dict( + os.environ, PUBLISH_TEST_PYTHONPATH=str(root / "tests/fakes") + ) + result = await run.result.set(env=invalid_environment, inherit_env=False)( + root / "tasks/publish.py", + "--service", + "../escape", + "--channel", + "nightly", + "--input", + package_input, + "deb", + ).stderr(sh.DEVNULL) + assert result.exit_code != 0 + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) |
