diff options
Diffstat
| -rw-r--r-- | .gitignore | 9 | +5 −4 |
| -rw-r--r-- | CONCEPT.md | 149 | +0 −149 |
| -rw-r--r-- | LICENSES/CC-BY-4.0.txt | 156 | +156 −0 |
| -rw-r--r-- | README.md | 130 | +123 −7 |
| -rw-r--r-- | REUSE.toml | 14 | +10 −4 |
| -rw-r--r-- | Taskfile.yml | 72 | +39 −33 |
| -rw-r--r-- | buf.lock (renamed from proto/buf.lock) | 0 | +0 −0 |
| -rw-r--r-- | buf.yaml (renamed from proto/buf.yaml) | 3 | +3 −0 |
| -rw-r--r-- | cmd/mirum-server/actor.go (renamed from cmd/mirumd/actor.go) | 52 | +26 −26 |
| -rw-r--r-- | cmd/mirum-server/api_cli.go (renamed from cmd/mirumd/api_cli.go) | 38 | +19 −19 |
| -rw-r--r-- | cmd/mirum-server/cert.go (renamed from cmd/mirumd/cert.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/cert_test.go (renamed from cmd/mirumd/cert_test.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/config.go (renamed from cmd/mirumd/config.go) | 2 | +1 −1 |
| -rw-r--r-- | cmd/mirum-server/database.go (renamed from cmd/mirumd/database.go) | 32 | +16 −16 |
| -rw-r--r-- | cmd/mirum-server/id.go (renamed from cmd/mirumd/id.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/id_test.go (renamed from cmd/mirumd/id_test.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/main.go (renamed from cmd/mirumd/main.go) | 26 | +13 −13 |
| -rw-r--r-- | cmd/mirum-server/proto/api.proto (renamed from proto/admin.proto) | 6 | +3 −3 |
| -rw-r--r-- | cmd/mirum-server/proto/buf.gen.yaml (renamed from proto/buf.gen.yaml) | 6 | +3 −3 |
| -rw-r--r-- | cmd/mirum-server/server.go (renamed from cmd/mirumd/server.go) | 6 | +3 −3 |
| -rw-r--r-- | cmd/mirum-server/server_admin.go (renamed from cmd/mirumd/server_admin.go) | 192 | +96 −96 |
| -rw-r--r-- | cmd/mirum-server/server_grpc.go (renamed from cmd/mirumd/server_grpc.go) | 14 | +7 −7 |
| -rw-r--r-- | cmd/mirum-server/server_web.go (renamed from cmd/mirumd/server_web.go) | 18 | +9 −9 |
| -rw-r--r-- | cmd/mirum-server/static.go (renamed from cmd/mirumd/static.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/static_dev.go (renamed from cmd/mirumd/static_dev.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/static_prod.go (renamed from cmd/mirumd/static_prod.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/api/client.ts (renamed from cmd/mirumd/web/api/client.ts) | 6 | +3 −3 |
| -rw-r--r-- | cmd/mirum-server/web/components.json (renamed from cmd/mirumd/web/components.json) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/pages/dashboard.tsx (renamed from cmd/mirumd/web/components/pages/dashboard.tsx) | 6 | +3 −3 |
| -rw-r--r-- | cmd/mirum-server/web/components/pages/error.tsx (renamed from cmd/mirumd/web/components/pages/error.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/pages/login.tsx (renamed from cmd/mirumd/web/components/pages/login.tsx) | 2 | +1 −1 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/alert.tsx (renamed from cmd/mirumd/web/components/ui/alert.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/button.tsx (renamed from cmd/mirumd/web/components/ui/button.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/card.tsx (renamed from cmd/mirumd/web/components/ui/card.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/field.tsx (renamed from cmd/mirumd/web/components/ui/field.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/input.tsx (renamed from cmd/mirumd/web/components/ui/input.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/label.tsx (renamed from cmd/mirumd/web/components/ui/label.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/components/ui/separator.tsx (renamed from cmd/mirumd/web/components/ui/separator.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/entries/dashboard.tsx (renamed from cmd/mirumd/web/entries/dashboard.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/entries/error.tsx (renamed from cmd/mirumd/web/entries/error.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/entries/login.tsx (renamed from cmd/mirumd/web/entries/login.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/index.css (renamed from cmd/mirumd/web/index.css) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/lib/errors.ts (renamed from cmd/mirumd/web/lib/errors.ts) | 2 | +1 −1 |
| -rw-r--r-- | cmd/mirum-server/web/lib/mount.tsx (renamed from cmd/mirumd/web/lib/mount.tsx) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/lib/utils.ts (renamed from cmd/mirumd/web/lib/utils.ts) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/package-lock.json (renamed from cmd/mirumd/web/package-lock.json) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/package.json (renamed from cmd/mirumd/web/package.json) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/shell.html (renamed from cmd/mirumd/web/shell.html) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/tsconfig.json (renamed from cmd/mirumd/web/tsconfig.json) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-server/web/vite.config.ts (renamed from cmd/mirumd/web/vite.config.ts) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum-worker/client.go (renamed from cmd/mirumw/client.go) | 12 | +6 −6 |
| -rw-r--r-- | cmd/mirum-worker/config.go (renamed from cmd/mirumw/config.go) | 2 | +1 −1 |
| -rw-r--r-- | cmd/mirum-worker/main.go (renamed from cmd/mirumw/main.go) | 0 | +0 −0 |
| -rw-r--r-- | cmd/mirum/main.go | 33 | +33 −0 |
| -rw-r--r-- | docs/mirumfile.md | 590 | +590 −0 |
| -rw-r--r-- | docs/whitepaper.md | 594 | +594 −0 |
| -rw-r--r-- | internal/config/config.go | 2 | +1 −1 |
| -rw-r--r-- | internal/protocol/platform.go | 76 | +38 −38 |
| -rw-r--r-- | internal/protocol/proto/buf.gen.yaml | 11 | +11 −0 |
| -rw-r--r-- | internal/protocol/proto/wire.proto (renamed from proto/mirum.proto) | 6 | +3 −3 |
| -rw-r--r-- | nfpm.yaml | 45 | +25 −20 |
| -rw-r--r-- | pkg/mirum-server.service (renamed from pkg/mirumd.service) | 10 | +5 −5 |
| -rw-r--r-- | pkg/mirum-worker@.service (renamed from pkg/mirumw@.service) | 8 | +4 −4 |
| -rw-r--r-- | pkg/scripts/postinstall.sh | 6 | +3 −3 |
| -rw-r--r-- | pkg/scripts/preinstall.sh | 2 | +1 −1 |
| -rw-r--r-- | pkg/scripts/preremove.sh | 8 | +4 −4 |
| -rw-r--r-- | pkg/server/config.yaml (renamed from pkg/mirumd.yaml) | 4 | +2 −2 |
| -rw-r--r-- | pkg/worker/default.yaml (renamed from pkg/mirumw-default.yaml) | 10 | +5 −5 |
68 files changed, 1866 insertions, 494 deletions
diff --git a/.gitignore b/.gitignore index cec9f17..e9e681e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,9 +4,10 @@ /dev /build /.task -cmd/mirumd/static -cmd/mirumd/web/node_modules +cmd/mirum-server/static +cmd/mirum-server/web/node_modules # grpc generated code -/internal/protocol/pb -/cmd/mirumd/web/gen +/internal/protocol/wirepb +/cmd/mirum-server/apipb +/cmd/mirum-server/web/gen diff --git a/CONCEPT.md b/CONCEPT.md deleted file mode 100644 --- a/CONCEPT.md +++ /dev/null @@ -1,149 +0,0 @@ -# The Concept Document - -> [!CAUTION] -> This document describes how I would like to design the system; -> it doesn't reflect the current state. Almost everything is still unimplemented. - -The really portable CI platform with VM-first isolation, -programmable pipelines, and local execution parity. - -## [Why](https://xkcd.com/927/) - -The goal of this project is to build a CI system that's both convenient for tiny -projects and suitable for gigantic C++ codebases like Chromium or llvm. - -For this to work, several key decisions need to be made: - -**Open source.** -You can and should build SaaS, but the user must be able to deploy the entire -system themselves. Self-hosted runners aren't enough. - -**Portability and cross-platform support.** -You should be able to run on at least x64, arm64, riscv64, powerpc, s390x, and -loongarch64, as well as Linux, {Free,Open,Net}BSD, Windows, and macOS. Users -should be able to add exotic features like Haiku and Plan9, or even a custom kernel. - -**Hermitic builds.** -All official platforms should support sealed builds with declarative environment -descriptions (like Docker, yes). Hosted builds should work everywhere. - -**Different deployment models.** -Not everyone can deploy themselves, and not everyone wants to. Offer open-source -SaaS, cloud and self-hosted runners, and fully autonomous solutions. The degree -of autonomy is the user's choice. - -**Dynamic pipelines.** -Don't assume that all tasks are described by a static YAML/TOML configuration. -This is often the case, but you should have a path for dynamic pipelines when they -are needed. - -**Security and Isolation.** -CI is the most security-sensitive platform, affecting testing, deployment to production, -and releases. -CI has access to both the code and the production environment. -Since the "just hide it behind a VPN" option doesn't work for either SaaS or open -source projects where CI must be public, you can't be overly paranoid about architectural -decisions. Consider that the code for your pacemaker might be tested here. - -**Local debugging.** -You should be able to run the build locally, get the console into the sandbox, and -attach a debugger. There's nothing more pointless and merciless than trying to fix -automation than the cycle of "test commit -> push to Git Forge -> hope it works." - -- `mirum run` — run the pipeline locally with the same VMs but with a local copy - of your code and a debugger -- `mirum ssh` — connect to a failed VM over SSH and debug on real hardware -- `mirum try` — run a build from local changes on the cluster without creating - throwaway commits. - -## How - -The architecture is built on two key solutions: -[Virtual Machines](https://en.wikipedia.org/wiki/Virtualization#Hardware_virtualization) -and [Starlark](https://starlark-lang.org/). - -### Virtualization vs. Containers vs. Host - -Modern CI must provide a reproducible and hermetic environment for every task. -Despite the popularity of container isolation, it's a Linux-specific technology -with many limitations. As soon as you need Windows, a specific kernel version, -or even systemd, you're forced to revert to bare, stateful runners you've manually -configured. - -A solution was proposed in Sourcehut: use ephemeral VMs from a snapshot for isolated -builds. Unlike a container, a VM can run any guest system, can emulate inaccessible -architectures, provides a full stack including the kernel, and provides sufficient -isolation to allow a user to access the build machine via SSH. - -The idea is to split the runner into two layers: - -- mirum-agent, a highly portable statically linked C binary that can copy files, - execute bash commands, and collect logs and resources. -- mirum-worker, a full-fledged runtime that launches a disposable VM for each task, - launching and managing mirum-agent within the VM. - -This architecture allows for full support on official platforms, including isolation, -SSH access, and so on. On the other hand, if you're testing an exotic system -(for example, on bare metal without any OS at all), port mirum-agent and you'll -be able to connect to a regular mirum server. You can also offer specialized versions -of mirum-worker for containers/clouds/lambdas, or any custom environment. -Simply run mirum-agent in a sandbox and issue commands to it. - -### Starlark - -Today, there are two ways to describe pipelines: statically in yaml or by writing -a script in a scripting language like JavaScript/Python/Ruby. - -Mirum occupies a niche between the two and offers Starlark as a configuration language. -Starlark is a specialized embedded programming language developed for the Bazel -build system. On the one hand, you have variable conditions and loops, imports, -objects, and arrays. - -Unlike yaml, you don't have to reinvent the wheel. Settings are variables, tasks -are functions, build matrices are loops, and the reusable actions library is -a simple import. Python syntax is well-known and doesn't require learning your DSL. - -On the other hand, it's not an algorithmically complete language. There are no side -effects, no need for a separate sandbox, and no need to drag in a runtime. The CI -server has total control over Starlark execution. But unlike Kotlin DSL (TeamCity), -Groovy (Jenkins), or Python (Buildbot), Starlark forbids side effects: no network, -no filesystem, no arbitrary imports. Eval is safe for untrusted code -(PRs from external contributors), deterministic, and cacheable. - -A quick example pipeline, all in one starlark file: - -```python -# /.mirum/main.star — complete CI -# A single pipeline describes a matrix of multiple operating systems and architectures. -# A single Linux host with KVM serves Linux, Windows, and \*BSD guests. A macOS host serves macOS, Linux, and \*BSD. -# Tasks adapt to the platform via `ctx.os` — they don't choose it. - -# ctx.run takes an optional setup function to indicate which steps -# are environment setup only, so the resulting image can be cached -def setup(ctx): - ctx.shell("apt-get update && apt-get install -y cargo") - -def build(ctx): - ctx.checkout() - ctx.shell("cargo build --release") - ctx.upload("target/release/myapp", artifact="bin") - -def test(ctx): - # test knows nothing about build — only that it needs an artifact. - # that artifact could have been built right here, come from cache - # or a registry, or even uploaded from a developer's laptop - ctx.download("bin", dest=".") - ctx.shell("cargo test") - -# Write a function that describes your pipeline. -def build_pipeline(ctx): - src = ctx.source() - - # Build steps are simply functions that the pipeline calls in the VM sandbox. - b = ctx.run(build, setup=setup, source=src, image="mirum/ubuntu-24.04") - ctx.run(test, setup=setup, depends=b, image="mirum/ubuntu-24.04") - -# Describe your project, what pipelines exist, and how to launch them. -def project(ctx): - ctx.pipeline("build", fn=build_pipeline, watch=ctx.watch(events=["push"], manual=True)) -``` diff --git a/LICENSES/CC-BY-4.0.txt b/LICENSES/CC-BY-4.0.txt new file mode 100644 --- /dev/null +++ b/LICENSES/CC-BY-4.0.txt @@ -0,0 +1,156 @@ +Creative Commons Attribution 4.0 International + + Creative Commons Corporation (“Creative Commons”) is not a law firm and does not provide legal services or legal advice. Distribution of Creative Commons public licenses does not create a lawyer-client or other relationship. Creative Commons makes its licenses and related information available on an “as-is” basis. Creative Commons gives no warranties regarding its licenses, any material licensed under their terms and conditions, or any related information. Creative Commons disclaims all liability for damages resulting from their use to the fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and conditions that creators and other rights holders may use to share original works of authorship and other material subject to copyright and certain other rights specified in the public license below. The following considerations are for informational purposes only, are not exhaustive, and do not form part of our licenses. + +Considerations for licensors: Our public licenses are intended for use by those authorized to give the public permission to use material in ways otherwise restricted by copyright and certain other rights. Our licenses are irrevocable. Licensors should read and understand the terms and conditions of the license they choose before applying it. Licensors should also secure all rights necessary before applying our licenses so that the public can reuse the material as expected. Licensors should clearly mark any material not subject to the license. This includes other CC-licensed material, or material used under an exception or limitation to copyright. More considerations for licensors. + +Considerations for the public: By using one of our public licenses, a licensor grants the public permission to use the licensed material under specified terms and conditions. If the licensor’s permission is not necessary for any reason–for example, because of any applicable exception or limitation to copyright–then that use is not regulated by the license. Our licenses grant only permissions under copyright and certain other rights that a licensor has authority to grant. Use of the licensed material may still be restricted for other reasons, including because others have copyright or other rights in the material. A licensor may make special requests, such as asking that all changes be marked or described. Although not required by our licenses, you are encouraged to respect those requests where reasonable. More considerations for the public. + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree to be bound by the terms and conditions of this Creative Commons Attribution 4.0 International Public License ("Public License"). To the extent this Public License may be interpreted as a contract, You are granted the Licensed Rights in consideration of Your acceptance of these terms and conditions, and the Licensor grants You such rights in consideration of benefits the Licensor receives from making the Licensed Material available under these terms and conditions. + +Section 1 – Definitions. + + a. Adapted Material means material subject to Copyright and Similar Rights that is derived from or based upon the Licensed Material and in which the Licensed Material is translated, altered, arranged, transformed, or otherwise modified in a manner requiring permission under the Copyright and Similar Rights held by the Licensor. For purposes of this Public License, where the Licensed Material is a musical work, performance, or sound recording, Adapted Material is always produced where the Licensed Material is synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright and Similar Rights in Your contributions to Adapted Material in accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights closely related to copyright including, without limitation, performance, broadcast, sound recording, and Sui Generis Database Rights, without regard to how the rights are labeled or categorized. For purposes of this Public License, the rights specified in Section 2(b)(1)-(2) are not Copyright and Similar Rights. + + d. Effective Technological Measures means those measures that, in the absence of proper authority, may not be circumvented under laws fulfilling obligations under Article 11 of the WIPO Copyright Treaty adopted on December 20, 1996, and/or similar international agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or any other exception or limitation to Copyright and Similar Rights that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, or other material to which the Licensor applied this Public License. + + g. Licensed Rights means the rights granted to You subject to the terms and conditions of this Public License, which are limited to all Copyright and Similar Rights that apply to Your use of the Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights under this Public License. + + i. Share means to provide material to the public by any means or process that requires permission under the Licensed Rights, such as reproduction, public display, public performance, distribution, dissemination, communication, or importation, and to make material available to the public including in ways that members of the public may access the material from a place and at a time individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright resulting from Directive 96/9/EC of the European Parliament and of the Council of 11 March 1996 on the legal protection of databases, as amended and/or succeeded, as well as other essentially equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights under this Public License. Your has a corresponding meaning. + +Section 2 – Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, the Licensor hereby grants You a worldwide, royalty-free, non-sublicensable, non-exclusive, irrevocable license to exercise the Licensed Rights in the Licensed Material to: + + A. reproduce and Share the Licensed Material, in whole or in part; and + + B. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where Exceptions and Limitations apply to Your use, this Public License does not apply, and You do not need to comply with its terms and conditions. + + 3. Term. The term of this Public License is specified in Section 6(a). + + 4. Media and formats; technical modifications allowed. The Licensor authorizes You to exercise the Licensed Rights in all media and formats whether now known or hereafter created, and to make technical modifications necessary to do so. The Licensor waives and/or agrees not to assert any right or authority to forbid You from making technical modifications necessary to exercise the Licensed Rights, including technical modifications necessary to circumvent Effective Technological Measures. For purposes of this Public License, simply making modifications authorized by this Section 2(a)(4) never produces Adapted Material. + + 5. Downstream recipients. + + A. Offer from the Licensor – Licensed Material. Every recipient of the Licensed Material automatically receives an offer from the Licensor to exercise the Licensed Rights under the terms and conditions of this Public License. + + B. No downstream restrictions. You may not offer or impose any additional or different terms or conditions on, or apply any Effective Technological Measures to, the Licensed Material if doing so restricts exercise of the Licensed Rights by any recipient of the Licensed Material. + + 6. No endorsement. Nothing in this Public License constitutes or may be construed as permission to assert or imply that You are, or that Your use of the Licensed Material is, connected with, or sponsored, endorsed, or granted official status by, the Licensor or others designated to receive attribution as provided in Section 3(a)(1)(A)(i). + +b. Other rights. + + 1. Moral rights, such as the right of integrity, are not licensed under this Public License, nor are publicity, privacy, and/or other similar personality rights; however, to the extent possible, the Licensor waives and/or agrees not to assert any such rights held by the Licensor to the limited extent necessary to allow You to exercise the Licensed Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this Public License. + + 3. To the extent possible, the Licensor waives any right to collect royalties from You for the exercise of the Licensed Rights, whether directly or through a collecting society under any voluntary or waivable statutory or compulsory licensing scheme. In all other cases the Licensor expressly reserves any right to collect such royalties. + +Section 3 – License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified form), You must: + + A. retain the following if it is supplied by the Licensor with the Licensed Material: + + i. identification of the creator(s) of the Licensed Material and any others designated to receive attribution, in any reasonable manner requested by the Licensor (including by pseudonym if designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of warranties; + + v. a URI or hyperlink to the Licensed Material to the extent reasonably practicable; + + B. indicate if You modified the Licensed Material and retain an indication of any previous modifications; and + + C. indicate the Licensed Material is licensed under this Public License, and include the text of, or the URI or hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any reasonable manner based on the medium, means, and context in which You Share the Licensed Material. For example, it may be reasonable to satisfy the conditions by providing a URI or hyperlink to a resource that includes the required information. + + 3. If requested by the Licensor, You must remove any of the information required by Section 3(a)(1)(A) to the extent reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's License You apply must not prevent recipients of the Adapted Material from complying with this Public License. + +Section 4 – Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right to extract, reuse, reproduce, and Share all or a substantial portion of the contents of the database; + + b. if You include all or a substantial portion of the database contents in a database in which You have Sui Generis Database Rights, then the database in which You have Sui Generis Database Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share all or a substantial portion of the contents of the database. +For the avoidance of doubt, this Section 4 supplements and does not replace Your obligations under this Public License where the Licensed Rights include other Copyright and Similar Rights. + +Section 5 – Disclaimer of Warranties and Limitation of Liability. + + a. Unless otherwise separately undertaken by the Licensor, to the extent possible, the Licensor offers the Licensed Material as-is and as-available, and makes no representations or warranties of any kind concerning the Licensed Material, whether express, implied, statutory, or other. This includes, without limitation, warranties of title, merchantability, fitness for a particular purpose, non-infringement, absence of latent or other defects, accuracy, or the presence or absence of errors, whether or not known or discoverable. Where disclaimers of warranties are not allowed in full or in part, this disclaimer may not apply to You. + + b. To the extent possible, in no event will the Licensor be liable to You on any legal theory (including, without limitation, negligence) or otherwise for any direct, special, indirect, incidental, consequential, punitive, exemplary, or other losses, costs, expenses, or damages arising out of this Public License or use of the Licensed Material, even if the Licensor has been advised of the possibility of such losses, costs, expenses, or damages. Where a limitation of liability is not allowed in full or in part, this limitation may not apply to You. + + c. The disclaimer of warranties and limitation of liability provided above shall be interpreted in a manner that, to the extent possible, most closely approximates an absolute disclaimer and waiver of all liability. + +Section 6 – Term and Termination. + + a. This Public License applies for the term of the Copyright and Similar Rights licensed here. However, if You fail to comply with this Public License, then Your rights under this Public License terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided it is cured within 30 days of Your discovery of the violation; or + + 2. upon express reinstatement by the Licensor. + + c. For the avoidance of doubt, this Section 6(b) does not affect any right the Licensor may have to seek remedies for Your violations of this Public License. + + d. For the avoidance of doubt, the Licensor may also offer the Licensed Material under separate terms or conditions or stop distributing the Licensed Material at any time; however, doing so will not terminate this Public License. + + e. Sections 1, 5, 6, 7, and 8 survive termination of this Public License. + +Section 7 – Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the Licensed Material not stated herein are separate from and independent of the terms and conditions of this Public License. + +Section 8 – Interpretation. + + a. For the avoidance of doubt, this Public License does not, and shall not be interpreted to, reduce, limit, restrict, or impose conditions on any use of the Licensed Material that could lawfully be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is deemed unenforceable, it shall be automatically reformed to the minimum extent necessary to make it enforceable. If the provision cannot be reformed, it shall be severed from this Public License without affecting the enforceability of the remaining terms and conditions. + + c. No term or condition of this Public License will be waived and no failure to comply consented to unless expressly agreed to by the Licensor. + + d. Nothing in this Public License constitutes or may be interpreted as a limitation upon, or waiver of, any privileges and immunities that apply to the Licensor or You, including from the legal processes of any jurisdiction or authority. + +Creative Commons is not a party to its public licenses. Notwithstanding, Creative Commons may elect to apply one of its public licenses to material it publishes and in those instances will be considered the “Licensor.” Except for the limited purpose of indicating that material is shared under a Creative Commons public license or as otherwise permitted by the Creative Commons policies published at creativecommons.org/policies, Creative Commons does not authorize the use of the trademark “Creative Commons” or any other trademark or logo of Creative Commons without its prior written consent including, without limitation, in connection with any unauthorized modifications to any of its public licenses or any other arrangements, understandings, or agreements concerning use of licensed material. For the avoidance of doubt, this paragraph does not form part of the public licenses. + +Creative Commons may be contacted at creativecommons.org. diff --git a/README.md b/README.md index 4e30f73..56684af 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,122 @@ # Mirum -An experimental CI built around virtual machines and Starlark +> [!CAUTION] +> This document describes the design and rationale of the system. It does +> not reflect the current state — almost everything is still +> unimplemented. For the canonical user-facing API of the configuration +> file, see [`mirumfile.md`](./docs/mirumfile.md). + +An experimental portable CI platform with VM-first isolation, +programmable pipelines, local execution parity and Starlark configs. + +## [Why](https://xkcd.com/927/) + +The goal of this project is to build a CI system that's both convenient for tiny +projects and suitable for gigantic C++ codebases like Chromium or llvm. + +For this to work, several key decisions need to be made: + +**Open source.** +You can and should build SaaS, but the user must be able to deploy the entire +system themselves. Self-hosted runners aren't enough. + +**Portability and cross-platform support.** +You should be able to run on at least x64, arm64, riscv64, powerpc, s390x, and +loongarch64, as well as Linux, {Free,Open,Net}BSD, Windows, and macOS. Users +should be able to add exotic features like Haiku and Plan9, or even a custom kernel. + +**Hermitic builds.** +All official platforms should support sealed builds with declarative environment +descriptions (like Docker, yes). Hosted builds should work everywhere. + +**Different deployment models.** +Not everyone can deploy themselves, and not everyone wants to. Offer open-source +SaaS, cloud and self-hosted runners, and fully autonomous solutions. The degree +of autonomy is the user's choice. + +**Dynamic pipelines.** +Don't assume that all tasks are described by a static YAML/TOML configuration. +This is often the case, but you should have a path for dynamic pipelines when they +are needed. + +**Security and Isolation.** +CI is the most security-sensitive platform, affecting testing, deployment to production, +and releases. +CI has access to both the code and the production environment. +Since the "just hide it behind a VPN" option doesn't work for either SaaS or open +source projects where CI must be public, you can't be overly paranoid about architectural +decisions. Consider that the code for your pacemaker might be tested here. + +**Local debugging.** +You should be able to run the build locally, get the console into the sandbox, and +attach a debugger. There's nothing more pointless and merciless than trying to fix +automation than the cycle of "test commit -> push to Git Forge -> hope it works." + +- `mirum task` — run a single task on the host without spinning up a VM, + for fast iteration during development +- `mirum run` — run the pipeline locally with the same VMs but with a local copy + of your code and a debugger +- `mirum ssh` — connect to a failed VM over SSH and debug on real hardware +- `mirum try` — run a build from local changes on the cluster without creating + throwaway commits + +## How + +The architecture is built on two key solutions: +[Virtual Machines](https://en.wikipedia.org/wiki/Virtualization#Hardware_virtualization) +and [Starlark](https://starlark-lang.org/). + +### Virtualization vs. Containers vs. Host + +Modern CI must provide a reproducible and hermetic environment for every task. +Despite the popularity of container isolation, it's a Linux-specific technology +with many limitations. As soon as you need Windows, a specific kernel version, +or even systemd, you're forced to revert to bare, stateful runners you've manually +configured. + +A solution was proposed in Sourcehut: use ephemeral VMs from a snapshot for isolated +builds. Unlike a container, a VM can run any guest system, can emulate inaccessible +architectures, provides a full stack including the kernel, and provides sufficient +isolation to allow a user to access the build machine via SSH. + +The idea is to split the runner into two layers: + +- mirum-agent, a highly portable statically linked C binary that can copy files, + execute bash commands, and collect logs and resources. +- mirum-worker, a full-fledged runtime that launches a disposable VM for each task, + launching and managing mirum-agent within the VM. + +This architecture allows for full support on official platforms, including isolation, +SSH access, and so on. On the other hand, if you're testing an exotic system +(for example, on bare metal without any OS at all), port mirum-agent and you'll +be able to connect to a regular mirum server. You can also offer specialized versions +of mirum-worker for containers/clouds/lambdas, or any custom environment. +Simply run mirum-agent in a sandbox and issue commands to it. + +Using VMs also significantly simplifies infrastructure. +One x64 host can run Linux, Windows, and *BSD, +while one arm64 mac mini can run macOS, Linux, and Windows on arm64. + +### Starlark + +Today, there are two ways to describe pipelines: statically in yaml or by writing +a script in a scripting language like JavaScript/Python/Ruby. + +Mirum occupies a niche between the two and offers Starlark as a configuration language. +Starlark is a specialized embedded programming language developed for the Bazel +build system. On the one hand, you have variable conditions and loops, imports, +objects, and arrays. + +Unlike yaml, you don't have to reinvent the wheel. Settings are variables, tasks +are functions, build matrices are loops, and the reusable actions library is +a simple import. Python syntax is well-known and doesn't require learning your DSL. + +On the other hand, it's not an algorithmically complete language. There are no side +effects, no need for a separate sandbox, and no need to drag in a runtime. The CI +server has total control over Starlark execution. But unlike Kotlin DSL (TeamCity), +Groovy (Jenkins), or Python (Buildbot), Starlark forbids side effects: no network, +no filesystem, no arbitrary imports. Eval is safe for untrusted code +(PRs from external contributors), deterministic, and cacheable. ## Installation @@ -16,10 +132,10 @@ echo "deb [signed-by=/usr/share/keyrings/mirum.gpg] https://dl.mirum.dev/apt/ ni sudo apt update && sudo apt install mirum # Start the server -sudo systemctl enable --now mirumd +sudo systemctl enable --now mirum-server # Start a worker (optional, can run on a different host) -sudo systemctl enable --now mirumw@default +sudo systemctl enable --now mirum-worker@default ``` **Fedora/RHEL:** @@ -34,10 +150,10 @@ sudo curl -o /etc/yum.repos.d/mirum-nightly.repo https://dl.mirum.dev/rpm/nightl sudo dnf install mirum # Start the server -sudo systemctl enable --now mirumd +sudo systemctl enable --now mirum-server # Start a worker (optional, can run on a different host) -sudo systemctl enable --now mirumw@default +sudo systemctl enable --now mirum-worker@default ``` **openSUSE:** @@ -49,10 +165,10 @@ sudo zypper refresh sudo zypper install mirum # Start the server -sudo systemctl enable --now mirumd +sudo systemctl enable --now mirum-server # Start a worker (optional, can run on a different host) -sudo systemctl enable --now mirumw@default +sudo systemctl enable --now mirum-worker@default ``` ## Contributing diff --git a/REUSE.toml b/REUSE.toml index 455fc7e..8cd1ea7 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -7,15 +7,21 @@ version = 1 path = [ ".mailmap", "CLA.md", - "CONCEPT.md", - "README.md", "VERSION", - "cmd/mirumd/web/*.json", + "cmd/mirum-server/web/*.json", "go.mod", "go.sum", "pkg/dl/*", "pkg/logo.svg", - "proto/buf.*", + "buf.*", ] SPDX-FileCopyrightText = "2026 Nikolay Govorov <me@govorov.online>" SPDX-License-Identifier = "AGPL-3.0-or-later" + +[[annotations]] +path = [ + "README.md", + "docs/**.md", +] +SPDX-FileCopyrightText = "2026 Nikolay Govorov <me@govorov.online>" +SPDX-License-Identifier = "CC-BY-4.0" diff --git a/Taskfile.yml b/Taskfile.yml index 05ffcdf..936e885 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -71,22 +71,26 @@ tasks: proto: desc: Generate ConnectRPC code from proto files sources: - - proto/*.proto - - proto/buf.gen.yaml - - proto/buf.yaml + - internal/protocol/proto/*.proto + - internal/protocol/proto/buf.gen.yaml + - cmd/mirum-server/proto/*.proto + - cmd/mirum-server/proto/buf.gen.yaml + - buf.yaml generates: - - internal/protocol/pb/**/*.go - - cmd/mirumd/web/gen/**/*.ts + - internal/protocol/wirepb/**/*.go + - cmd/mirum-server/apipb/**/*.go + - cmd/mirum-server/web/gen/**/*.ts cmds: - - rm -rf internal/protocol/pb cmd/mirumd/web/gen - - cd proto && buf generate - # admin_pb.ts imports file_buf_validate_validate from buf/validate/validate_pb.js. + - rm -rf internal/protocol/wirepb cmd/mirum-server/apipb cmd/mirum-server/web/gen + - cd internal/protocol/proto && buf generate + - cd cmd/mirum-server/proto && buf generate + # api_pb.ts imports file_buf_validate_validate from buf/validate/validate_pb.js. # buf.validate carries only server-side field annotations that the browser # never decodes, so replace the 200+ KB generated schema with an empty # descriptor stub that satisfies the GenFile type. - - mkdir -p cmd/mirumd/web/gen/buf/validate + - mkdir -p cmd/mirum-server/web/gen/buf/validate - | - cat > cmd/mirumd/web/gen/buf/validate/validate_pb.ts <<'EOF' + cat > cmd/mirum-server/web/gen/buf/validate/validate_pb.ts <<'EOF' // Stub: see Taskfile.yml proto task. buf.validate is server-only. import { fileDesc, type GenFile } from "@bufbuild/protobuf/codegenv2" export const file_buf_validate_validate: GenFile = /*@__PURE__*/ fileDesc("") @@ -94,7 +98,7 @@ tasks: web:install: desc: Install frontend dependencies - dir: cmd/mirumd/web + dir: cmd/mirum-server/web sources: - package.json - package-lock.json @@ -104,8 +108,8 @@ tasks: - npm ci web:build: - desc: Build the frontend into cmd/mirumd/static - dir: cmd/mirumd/web + desc: Build the frontend into cmd/mirum-server/static + dir: cmd/mirum-server/web deps: [web:install, proto] sources: - api/**/* @@ -124,7 +128,7 @@ tasks: web:size: desc: Check frontend bundle size budgets - dir: cmd/mirumd/web + dir: cmd/mirum-server/web deps: [web:build] sources: - ../static/assets/* @@ -133,8 +137,8 @@ tasks: - npm run size web:dev: - desc: Run Vite dev server (use with `go run -tags dev ./cmd/mirumd`) - dir: cmd/mirumd/web + desc: Run Vite dev server (use with `go run -tags dev ./cmd/mirum-server`) + dir: cmd/mirum-server/web deps: [web:install, proto] prefix: vite env: @@ -155,8 +159,8 @@ tasks: test -f dev/server.crt || openssl req -x509 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -keyout dev/server.key -out dev/server.crt - -days 365 -nodes -subj "/CN=mirumd" - -addext "subjectAltName=DNS:mirumd,DNS:localhost" + -days 365 -nodes -subj "/CN=mirum-server" + -addext "subjectAltName=DNS:mirum-server,DNS:localhost" - >- test -f dev/worker.key || openssl genpkey -algorithm ed25519 -out dev/worker.key @@ -165,13 +169,13 @@ tasks: openssl pkey -in dev/worker.key -pubout -out dev/worker.pub devenv:config: - desc: Write dev/mirumd.yaml if missing + desc: Write dev/mirum-server.yaml if missing status: - - test -f dev/mirumd.yaml + - test -f dev/mirum-server.yaml cmds: - mkdir -p dev - | - cat > dev/mirumd.yaml <<EOF + cat > dev/mirum-server.yaml <<EOF web_addr: ":3000" grpc_addr: ":2026" admin_socket: "/tmp/mirum-admin.sock" @@ -224,18 +228,18 @@ tasks: deps: [devenv:keys, devenv:config, devenv:db] dev:server: - desc: Build mirumd with -tags dev and run it against the dev environment + desc: Build mirum-server with -tags dev and run it against the dev environment deps: [proto, devenv] - prefix: mirumd + prefix: mirum-server env: CGO_ENABLED: "0" cmds: - mkdir -p {{.BUILD_DIR}} - - go build -tags dev -o {{.BUILD_DIR}}/mirumd ./cmd/mirumd - - "{{.BUILD_DIR}}/mirumd daemon --config dev/mirumd.yaml" + - go build -tags dev -o {{.BUILD_DIR}}/mirum-server ./cmd/mirum-server + - "{{.BUILD_DIR}}/mirum-server daemon --config dev/mirum-server.yaml" dev: - desc: Run Vite dev server and mirumd together (HMR-enabled) + desc: Run Vite dev server and mirum-server together (HMR-enabled) deps: [web:dev, dev:server] build: @@ -250,16 +254,18 @@ tasks: GOARCH: "{{.GOARCH}}" cmds: - mkdir -p {{.BUILD_DIR}} - - go build -o {{.BUILD_DIR}}/mirumd-{{.GOOS}}-{{.GOARCH}} ./cmd/mirumd - - go build -o {{.BUILD_DIR}}/mirumw-{{.GOOS}}-{{.GOARCH}} ./cmd/mirumw - - cp {{.BUILD_DIR}}/mirumd-{{.GOOS}}-{{.GOARCH}} {{.BUILD_DIR}}/mirumd - - cp {{.BUILD_DIR}}/mirumw-{{.GOOS}}-{{.GOARCH}} {{.BUILD_DIR}}/mirumw + - go build -o {{.BUILD_DIR}}/mirum-server-{{.GOOS}}-{{.GOARCH}} ./cmd/mirum-server + - go build -o {{.BUILD_DIR}}/mirum-worker-{{.GOOS}}-{{.GOARCH}} ./cmd/mirum-worker + - go build -o {{.BUILD_DIR}}/mirum-{{.GOOS}}-{{.GOARCH}} ./cmd/mirum + - cp {{.BUILD_DIR}}/mirum-server-{{.GOOS}}-{{.GOARCH}} {{.BUILD_DIR}}/mirum-server + - cp {{.BUILD_DIR}}/mirum-worker-{{.GOOS}}-{{.GOARCH}} {{.BUILD_DIR}}/mirum-worker + - cp {{.BUILD_DIR}}/mirum-{{.GOOS}}-{{.GOARCH}} {{.BUILD_DIR}}/mirum run: - desc: Build and run mirumd against the dev environment + desc: Build and run mirum-server against the dev environment deps: [build, devenv] cmds: - - "{{.BUILD_DIR}}/mirumd daemon --config dev/mirumd.yaml" + - "{{.BUILD_DIR}}/mirum-server daemon --config dev/mirum-server.yaml" test: desc: Run all tests @@ -307,7 +313,7 @@ tasks: task build GOOS=linux GOARCH=$arch ARCH=$arch nfpm package --packager deb --target {{.DIST_DIR}}/ ARCH=$arch nfpm package --packager rpm --target {{.DIST_DIR}}/ - rm {{.BUILD_DIR}}/mirumd {{.BUILD_DIR}}/mirumw + rm {{.BUILD_DIR}}/mirum-server {{.BUILD_DIR}}/mirum-worker done printf 'VERSION={{.VERSION}}\nCHANNEL={{.CHANNEL}}\n' > {{.DIST_DIR}}/build.env diff --git a/proto/buf.lock b/buf.lock index d15a117..d15a117 100644 --- a/proto/buf.lock +++ b/buf.lock diff --git a/proto/buf.yaml b/buf.yaml index 8d594f7..ddd7a3c 100644 --- a/proto/buf.yaml +++ b/buf.yaml @@ -2,5 +2,8 @@ # SPDX-License-Identifier: AGPL-3.0-or-later version: v2 +modules: + - path: internal/protocol/proto + - path: cmd/mirum-server/proto deps: - buf.build/bufbuild/protovalidate diff --git a/cmd/mirumd/actor.go b/cmd/mirum-server/actor.go index bdd49c2..03d30ba 100644 --- a/cmd/mirumd/actor.go +++ b/cmd/mirum-server/actor.go @@ -8,7 +8,7 @@ import ( "errors" "slices" - "dimidiumlabs/mirum/internal/protocol/pb" + "dimidiumlabs/mirum/cmd/mirum-server/apipb" "github.com/google/uuid" "github.com/jackc/pgx/v5" @@ -19,40 +19,40 @@ var ( ErrUnauthenticated = errors.New("database: authentication required") ) -var anonPermissions = []pb.Perm{ - pb.Perm_PERM_ORG_READ, +var anonPermissions = []apipb.Perm{ + apipb.Perm_PERM_ORG_READ, } -var userGlobalPermissions = []pb.Perm{ - pb.Perm_PERM_ORG_READ, - pb.Perm_PERM_ORG_WRITE, - pb.Perm_PERM_USER_READ, +var userGlobalPermissions = []apipb.Perm{ + apipb.Perm_PERM_ORG_READ, + apipb.Perm_PERM_ORG_WRITE, + apipb.Perm_PERM_USER_READ, } // rolePermissions is the single source of truth for role → perm bundles. // RLS checks only tenancy (membership); action authz lives here. -var rolePermissions = map[string][]pb.Perm{ +var rolePermissions = map[string][]apipb.Perm{ "owner": { - pb.Perm_PERM_ORG_READ, - pb.Perm_PERM_ORG_WRITE, - pb.Perm_PERM_ORG_DELETE, - pb.Perm_PERM_ORG_MEMBER_READ, - pb.Perm_PERM_ORG_MEMBER_WRITE, - pb.Perm_PERM_WORKER_READ, - pb.Perm_PERM_WORKER_WRITE, + apipb.Perm_PERM_ORG_READ, + apipb.Perm_PERM_ORG_WRITE, + apipb.Perm_PERM_ORG_DELETE, + apipb.Perm_PERM_ORG_MEMBER_READ, + apipb.Perm_PERM_ORG_MEMBER_WRITE, + apipb.Perm_PERM_WORKER_READ, + apipb.Perm_PERM_WORKER_WRITE, }, "admin": { - pb.Perm_PERM_ORG_READ, - pb.Perm_PERM_ORG_WRITE, - pb.Perm_PERM_ORG_MEMBER_READ, - pb.Perm_PERM_ORG_MEMBER_WRITE, - pb.Perm_PERM_WORKER_READ, - pb.Perm_PERM_WORKER_WRITE, + apipb.Perm_PERM_ORG_READ, + apipb.Perm_PERM_ORG_WRITE, + apipb.Perm_PERM_ORG_MEMBER_READ, + apipb.Perm_PERM_ORG_MEMBER_WRITE, + apipb.Perm_PERM_WORKER_READ, + apipb.Perm_PERM_WORKER_WRITE, }, "member": { - pb.Perm_PERM_ORG_READ, - pb.Perm_PERM_ORG_MEMBER_READ, - pb.Perm_PERM_WORKER_READ, + apipb.Perm_PERM_ORG_READ, + apipb.Perm_PERM_ORG_MEMBER_READ, + apipb.Perm_PERM_WORKER_READ, }, } @@ -176,7 +176,7 @@ func (a Actor) kindString() string { } // checkGlobal checks a global-scope perm (no specific org). Pure, no DB. -func checkGlobal(actor Actor, perm pb.Perm) error { +func checkGlobal(actor Actor, perm apipb.Perm) error { switch actor.kind { case actorOperator, actorSystem: return nil @@ -199,7 +199,7 @@ func checkGlobal(actor Actor, perm pb.Perm) error { } // checkPerm checks an org-scoped perm within an existing transaction. -func checkPerm(ctx context.Context, tx pgx.Tx, actor Actor, orgID OrgID, perm pb.Perm) error { +func checkPerm(ctx context.Context, tx pgx.Tx, actor Actor, orgID OrgID, perm apipb.Perm) error { switch actor.kind { case actorOperator, actorSystem: return nil diff --git a/cmd/mirumd/api_cli.go b/cmd/mirum-server/api_cli.go index 6d53511..8220c65 100644 --- a/cmd/mirumd/api_cli.go +++ b/cmd/mirum-server/api_cli.go @@ -3,10 +3,10 @@ package main -// Admin CLI is generated from admin.proto at startup via protoreflect. +// Console CLI is generated from api.proto at startup via protoreflect. // RPC name is camelCase-split into a cobra path: UserCreate -> "user create", // OrgMemberAdd -> "org member add". Flags come from request fields, dispatch -// goes through reflect on pbconnect.AdminClient. +// goes through reflect on apipbconnect.ConsoleClient. import ( "context" @@ -26,14 +26,14 @@ import ( "google.golang.org/protobuf/reflect/protoregistry" "google.golang.org/protobuf/types/known/timestamppb" - "dimidiumlabs/mirum/internal/protocol/pb" - "dimidiumlabs/mirum/internal/protocol/pb/pbconnect" + "dimidiumlabs/mirum/cmd/mirum-server/apipb" + "dimidiumlabs/mirum/cmd/mirum-server/apipb/apipbconnect" ) // mkClient is called per-invocation so persistent flags (e.g. --socket) are // already parsed by the time it runs. -func buildAdminCLI(root *cobra.Command, mkClient func() pbconnect.AdminClient) { - methods := pb.File_admin_proto.Services().ByName("Admin").Methods() +func buildConsoleCLI(root *cobra.Command, mkClient func() apipbconnect.ConsoleClient) { + methods := apipb.File_api_proto.Services().ByName("Console").Methods() for i := 0; i < methods.Len(); i++ { md := methods.Get(i) path := splitCamel(string(md.Name())) @@ -77,7 +77,7 @@ func ensureGroups(root *cobra.Command, path []string) *cobra.Command { // fieldSetter writes one flag value into the request message. type fieldSetter func(*pflag.FlagSet, protoreflect.Message) error -func buildMethodCmd(md protoreflect.MethodDescriptor, leaf string, mkClient func() pbconnect.AdminClient) *cobra.Command { +func buildMethodCmd(md protoreflect.MethodDescriptor, leaf string, mkClient func() apipbconnect.ConsoleClient) *cobra.Command { reqDesc := md.Input() rpcName := string(md.Name()) cmd := &cobra.Command{ @@ -91,7 +91,7 @@ func buildMethodCmd(md protoreflect.MethodDescriptor, leaf string, mkClient func fmt.Fprintln(os.Stderr, err) os.Exit(1) } - resp, err := dispatchAdmin(mkClient(), rpcName, req) + resp, err := dispatchConsole(mkClient(), rpcName, req) if err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -187,7 +187,7 @@ func registerField(cmd *cobra.Command, fd protoreflect.FieldDescriptor) fieldSet } if required { - panic(fmt.Sprintf("admincli: unhandled required field %s (kind=%s)", fd.FullName(), fd.Kind())) + panic(fmt.Sprintf("consolecli: unhandled required field %s (kind=%s)", fd.FullName(), fd.Kind())) } return nil } @@ -204,7 +204,7 @@ func registerMessageField(cmd *cobra.Command, fd protoreflect.FieldDescriptor, f } switch fd.Message().FullName() { - case "mirum.UserRef": + case "mirum.api.UserRef": flags.String(flagName, "", "user email") markRequired() return func(fs *pflag.FlagSet, m protoreflect.Message) error { @@ -212,12 +212,12 @@ func registerMessageField(cmd *cobra.Command, fd protoreflect.FieldDescriptor, f if v == "" { return nil } - ref := &pb.UserRef{Ref: &pb.UserRef_Email{Email: v}} + ref := &apipb.UserRef{Ref: &apipb.UserRef_Email{Email: v}} m.Set(fd, protoreflect.ValueOfMessage(ref.ProtoReflect())) return nil } - case "mirum.OrgRef": + case "mirum.api.OrgRef": flags.String(flagName, "", "org slug") markRequired() return func(fs *pflag.FlagSet, m protoreflect.Message) error { @@ -225,22 +225,22 @@ func registerMessageField(cmd *cobra.Command, fd protoreflect.FieldDescriptor, f if v == "" { return nil } - ref := &pb.OrgRef{Ref: &pb.OrgRef_Slug{Slug: v}} + ref := &apipb.OrgRef{Ref: &apipb.OrgRef_Slug{Slug: v}} m.Set(fd, protoreflect.ValueOfMessage(ref.ProtoReflect())) return nil } - case "mirum.PageRequest": + case "mirum.api.PageRequest": return nil } if required { - panic(fmt.Sprintf("admincli: unhandled required message field %s (type=%s)", fd.FullName(), fd.Message().FullName())) + panic(fmt.Sprintf("consolecli: unhandled required message field %s (type=%s)", fd.FullName(), fd.Message().FullName())) } return nil } -// Admin schema uses bytes only for UUIDs (id / *_id) and ed25519 PKIX keys. +// Console schema uses bytes only for UUIDs (id / *_id) and ed25519 PKIX keys. func parseBytesFlag(fieldName, v string) ([]byte, error) { switch { case fieldName == "id" || strings.HasSuffix(fieldName, "_id"): @@ -297,11 +297,11 @@ func buildRequest(desc protoreflect.MessageDescriptor, flags *pflag.FlagSet, set // reflect.New on *connect.Request[T] is equivalent to connect.NewRequest(req): // Msg is the only public field, the rest are initialised lazily at send time. -func dispatchAdmin(client pbconnect.AdminClient, name string, req proto.Message) (proto.Message, error) { +func dispatchConsole(client apipbconnect.ConsoleClient, name string, req proto.Message) (proto.Message, error) { cv := reflect.ValueOf(client) method := cv.MethodByName(name) if !method.IsValid() { - return nil, fmt.Errorf("unknown admin method %q", name) + return nil, fmt.Errorf("unknown console method %q", name) } // method signature: // func(context.Context, *connect.Request[T]) (*connect.Response[U], error) @@ -329,7 +329,7 @@ func printResponse(resp proto.Message) { var meaningful []protoreflect.FieldDescriptor for i := 0; i < fields.Len(); i++ { f := fields.Get(i) - if f.Kind() == protoreflect.MessageKind && f.Message().FullName() == "mirum.PageResponse" { + if f.Kind() == protoreflect.MessageKind && f.Message().FullName() == "mirum.api.PageResponse" { continue } meaningful = append(meaningful, f) diff --git a/cmd/mirumd/cert.go b/cmd/mirum-server/cert.go index 67128d5..67128d5 100644 --- a/cmd/mirumd/cert.go +++ b/cmd/mirum-server/cert.go diff --git a/cmd/mirumd/cert_test.go b/cmd/mirum-server/cert_test.go index 794b2f6..794b2f6 100644 --- a/cmd/mirumd/cert_test.go +++ b/cmd/mirum-server/cert_test.go diff --git a/cmd/mirumd/config.go b/cmd/mirum-server/config.go index 854e7ec..48d6ed2 100644 --- a/cmd/mirumd/config.go +++ b/cmd/mirum-server/config.go @@ -35,7 +35,7 @@ func getConfig(filename string) (*appConfig, error) { cfg := &appConfig{ GrpcAddr: ":2026", WebAddr: ":3000", - AdminSocket: "/run/mirumd/admin.sock", + AdminSocket: "/run/mirum-server/admin.sock", } data, err := os.ReadFile(filename) diff --git a/cmd/mirumd/database.go b/cmd/mirum-server/database.go index 6c03f7a..8655dbc 100644 --- a/cmd/mirumd/database.go +++ b/cmd/mirum-server/database.go @@ -18,8 +18,8 @@ import ( "strings" "time" + "dimidiumlabs/mirum/cmd/mirum-server/apipb" "dimidiumlabs/mirum/internal/config" - "dimidiumlabs/mirum/internal/protocol/pb" sb "github.com/huandu/go-sqlbuilder" "github.com/jackc/pgerrcode" @@ -469,7 +469,7 @@ func (db *DB) UserGet(ctx context.Context, actor Actor, ref UserRef) (*User, err var u User err := db.apicall( ctx, actor, - func(tx pgx.Tx) error { return checkGlobal(actor, pb.Perm_PERM_USER_READ) }, + func(tx pgx.Tx) error { return checkGlobal(actor, apipb.Perm_PERM_USER_READ) }, nil, func(tx pgx.Tx) error { col, val := ref.where() @@ -498,7 +498,7 @@ func (db *DB) UserList(ctx context.Context, actor Actor, cursor UserID, limit in var users []User var total int err := db.apicall(ctx, actor, - func(tx pgx.Tx) error { return checkGlobal(actor, pb.Perm_PERM_USER_READ) }, + func(tx pgx.Tx) error { return checkGlobal(actor, apipb.Perm_PERM_USER_READ) }, func(tx pgx.Tx) error { if filter != "" { return ErrNotImplemented @@ -750,7 +750,7 @@ func (db *DB) UserSessionPurgeExpired(ctx context.Context) error { func (db *DB) OrgGet(ctx context.Context, actor Actor, ref OrgRef) (*Organization, error) { var o Organization err := db.apicall(ctx, actor, - func(tx pgx.Tx) error { return checkGlobal(actor, pb.Perm_PERM_ORG_READ) }, + func(tx pgx.Tx) error { return checkGlobal(actor, apipb.Perm_PERM_ORG_READ) }, nil, func(tx pgx.Tx) error { col, val := ref.where() @@ -777,7 +777,7 @@ func (db *DB) OrgGet(ctx context.Context, actor Actor, ref OrgRef) (*Organizatio func (db *DB) OrgCreate(ctx context.Context, actor Actor, name, slug string, public bool, owner UserRef) (OrgID, error) { var orgID OrgID err := db.apicall(ctx, actor, - func(tx pgx.Tx) error { return checkGlobal(actor, pb.Perm_PERM_ORG_WRITE) }, + func(tx pgx.Tx) error { return checkGlobal(actor, apipb.Perm_PERM_ORG_WRITE) }, nil, func(tx pgx.Tx) error { userID, err := resolveUser(ctx, tx, owner) @@ -820,7 +820,7 @@ func (db *DB) OrgUpdate(ctx context.Context, actor Actor, ref OrgRef, name *stri if err != nil { return err } - return checkPerm(ctx, tx, actor, id, pb.Perm_PERM_ORG_WRITE) + return checkPerm(ctx, tx, actor, id, apipb.Perm_PERM_ORG_WRITE) }, nil, func(tx pgx.Tx) error { @@ -864,7 +864,7 @@ func (db *DB) OrgDelete(ctx context.Context, actor Actor, ref OrgRef) error { if err != nil { return err } - return checkPerm(ctx, tx, actor, id, pb.Perm_PERM_ORG_DELETE) + return checkPerm(ctx, tx, actor, id, apipb.Perm_PERM_ORG_DELETE) }, nil, func(tx pgx.Tx) error { @@ -890,7 +890,7 @@ func (db *DB) OrgList(ctx context.Context, actor Actor, cursor OrgID, limit int, var orgs []Organization var total int err := db.apicall(ctx, actor, - func(tx pgx.Tx) error { return checkGlobal(actor, pb.Perm_PERM_ORG_READ) }, + func(tx pgx.Tx) error { return checkGlobal(actor, apipb.Perm_PERM_ORG_READ) }, func(tx pgx.Tx) error { if filter != "" { return ErrNotImplemented @@ -946,7 +946,7 @@ func (db *DB) OrgMemberGet(ctx context.Context, actor Actor, org OrgRef, user Us if err != nil { return err } - return checkPerm(ctx, tx, actor, orgID, pb.Perm_PERM_ORG_MEMBER_READ) + return checkPerm(ctx, tx, actor, orgID, apipb.Perm_PERM_ORG_MEMBER_READ) }, nil, func(tx pgx.Tx) error { @@ -985,7 +985,7 @@ func (db *DB) OrgMembersList(ctx context.Context, actor Actor, org OrgRef, curso if err != nil { return err } - return checkPerm(ctx, tx, actor, orgID, pb.Perm_PERM_ORG_MEMBER_READ) + return checkPerm(ctx, tx, actor, orgID, apipb.Perm_PERM_ORG_MEMBER_READ) }, func(tx pgx.Tx) error { if filter != "" { @@ -1041,7 +1041,7 @@ func (db *DB) OrgMemberAdd(ctx context.Context, actor Actor, org OrgRef, user Us if err != nil { return err } - return checkPerm(ctx, tx, actor, orgID, pb.Perm_PERM_ORG_MEMBER_WRITE) + return checkPerm(ctx, tx, actor, orgID, apipb.Perm_PERM_ORG_MEMBER_WRITE) }, nil, func(tx pgx.Tx) error { @@ -1075,7 +1075,7 @@ func (db *DB) OrgMemberUpdateRole(ctx context.Context, actor Actor, org OrgRef, if err != nil { return err } - return checkPerm(ctx, tx, actor, orgID, pb.Perm_PERM_ORG_MEMBER_WRITE) + return checkPerm(ctx, tx, actor, orgID, apipb.Perm_PERM_ORG_MEMBER_WRITE) }, nil, func(tx pgx.Tx) error { @@ -1128,7 +1128,7 @@ func (db *DB) OrgMemberRemove(ctx context.Context, actor Actor, org OrgRef, user if err != nil { return err } - return checkPerm(ctx, tx, actor, orgID, pb.Perm_PERM_ORG_MEMBER_WRITE) + return checkPerm(ctx, tx, actor, orgID, apipb.Perm_PERM_ORG_MEMBER_WRITE) }, nil, func(tx pgx.Tx) error { @@ -1185,7 +1185,7 @@ func (db *DB) WorkerGet(ctx context.Context, actor Actor, id WorkerID) (*Worker, return err } if orgID != nil { - return checkPerm(ctx, tx, actor, *orgID, pb.Perm_PERM_WORKER_READ) + return checkPerm(ctx, tx, actor, *orgID, apipb.Perm_PERM_WORKER_READ) } if !actor.IsSuperuser() { return ErrPermissionDenied @@ -1214,7 +1214,7 @@ func (db *DB) WorkerCreate(ctx context.Context, actor Actor, publicKey []byte, o return err } orgID = &id - return checkPerm(ctx, tx, actor, id, pb.Perm_PERM_WORKER_WRITE) + return checkPerm(ctx, tx, actor, id, apipb.Perm_PERM_WORKER_WRITE) } if !actor.IsSuperuser() { return ErrPermissionDenied @@ -1246,7 +1246,7 @@ func (db *DB) WorkerDelete(ctx context.Context, actor Actor, id WorkerID) error return err } if orgID != nil { - return checkPerm(ctx, tx, actor, *orgID, pb.Perm_PERM_WORKER_WRITE) + return checkPerm(ctx, tx, actor, *orgID, apipb.Perm_PERM_WORKER_WRITE) } if !actor.IsSuperuser() { return ErrPermissionDenied diff --git a/cmd/mirumd/id.go b/cmd/mirum-server/id.go index 09ec141..09ec141 100644 --- a/cmd/mirumd/id.go +++ b/cmd/mirum-server/id.go diff --git a/cmd/mirumd/id_test.go b/cmd/mirum-server/id_test.go index 0d06fc1..0d06fc1 100644 --- a/cmd/mirumd/id_test.go +++ b/cmd/mirum-server/id_test.go diff --git a/cmd/mirumd/main.go b/cmd/mirum-server/main.go index 73aa3f9..c592f71 100644 --- a/cmd/mirumd/main.go +++ b/cmd/mirum-server/main.go @@ -12,10 +12,10 @@ import ( "net/http" "os" + "dimidiumlabs/mirum/cmd/mirum-server/apipb/apipbconnect" "dimidiumlabs/mirum/internal/config" "dimidiumlabs/mirum/internal/forges" - "dimidiumlabs/mirum/internal/protocol/pb" - "dimidiumlabs/mirum/internal/protocol/pb/pbconnect" + "dimidiumlabs/mirum/internal/protocol/wirepb" "dimidiumlabs/mirum/internal/supervisor" "github.com/coreos/go-systemd/v22/activation" @@ -32,8 +32,8 @@ func hardenServer(s *http.Server) *http.Server { func main() { var socketPath string - root := &cobra.Command{Use: "mirumd", Short: "Mirum CI server"} - root.PersistentFlags().StringVar(&socketPath, "socket", "", "admin socket path (default from config or /run/mirumd/admin.sock)") + root := &cobra.Command{Use: "mirum-server", Short: "Mirum CI server"} + root.PersistentFlags().StringVar(&socketPath, "socket", "", "admin socket path (default from config or /run/mirum-server/admin.sock)") root.AddGroup(&cobra.Group{ID: "main", Title: "Commands:"}) daemonCmd := &cobra.Command{ @@ -51,8 +51,8 @@ func main() { _ = daemonCmd.MarkFlagRequired("config") root.AddCommand(daemonCmd) - // Admin subcommands are generated from admin.proto via reflection. - buildAdminCLI(root, func() pbconnect.AdminClient { return adminClient(socketPath) }) + // Console subcommands are generated from api.proto via reflection. + buildConsoleCLI(root, func() apipbconnect.ConsoleClient { return consoleClient(socketPath) }) for _, c := range root.Commands() { if c.GroupID == "" { c.GroupID = "main" @@ -91,7 +91,7 @@ func daemon(configFile, socketFlag string) error { db: db, cfg: cfg, forge: &forges.GitHub{Secret: cfg.WebhookSecret, Token: cfg.GitHubToken}, - queue: make(chan *pb.Task, config.TaskQueueCapacity), + queue: make(chan *wirepb.Task, config.TaskQueueCapacity), } defer srv.Close() @@ -104,13 +104,13 @@ func daemon(configFile, socketFlag string) error { go srv.PurgeSessions(ctx) - adminPath, adminHandler := NewAdminHandler(srv) + consolePath, consoleHandler := NewConsoleHandler(srv) - webSrv := hardenServer(NewWebServer(ctx, srv, adminPath, adminHandler)) + webSrv := hardenServer(NewWebServer(ctx, srv, consolePath, consoleHandler)) grpcSrv := hardenServer(NewGrpcServer(ctx, srv)) adminMux := http.NewServeMux() - adminMux.Handle(adminPath, adminHandler) + adminMux.Handle(consolePath, consoleHandler) adminSrv := hardenServer(&http.Server{ Handler: adminMux, ConnContext: func(ctx context.Context, _ net.Conn) context.Context { @@ -237,11 +237,11 @@ func listeners(cfg *appConfig) (grpcLn, webLn, adminLn net.Listener, err error) return grpcLn, webLn, adminLn, nil } -func adminClient(socketPath string) pbconnect.AdminClient { +func consoleClient(socketPath string) apipbconnect.ConsoleClient { if socketPath == "" { - socketPath = "/run/mirumd/admin.sock" + socketPath = "/run/mirum-server/admin.sock" } - return pbconnect.NewAdminClient( + return apipbconnect.NewConsoleClient( &http.Client{ Transport: &http.Transport{ DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { diff --git a/proto/admin.proto b/cmd/mirum-server/proto/api.proto index 58f4e64..16a635e 100644 --- a/proto/admin.proto +++ b/cmd/mirum-server/proto/api.proto @@ -3,14 +3,14 @@ syntax = "proto3"; -package mirum; +package mirum.api; import "buf/validate/validate.proto"; import "google/protobuf/timestamp.proto"; -option go_package = "dimidiumlabs/mirum/internal/protocol/pb"; +option go_package = "dimidiumlabs/mirum/cmd/mirum-server/apipb"; -service Admin { +service Console { rpc UserCreate(UserCreateRequest) returns (UserCreateResponse); rpc UserGet(UserGetRequest) returns (UserGetResponse); rpc UserList(UserListRequest) returns (UserListResponse); diff --git a/proto/buf.gen.yaml b/cmd/mirum-server/proto/buf.gen.yaml index 0f24cdc..a9a94e2 100644 --- a/proto/buf.gen.yaml +++ b/cmd/mirum-server/proto/buf.gen.yaml @@ -4,13 +4,13 @@ version: v2 plugins: - local: protoc-gen-go - out: ../internal/protocol/pb + out: ../apipb opt: paths=source_relative - local: protoc-gen-connect-go - out: ../internal/protocol/pb + out: ../apipb opt: paths=source_relative - remote: buf.build/bufbuild/es:v2.11.0 - out: ../cmd/mirumd/web/gen + out: ../web/gen opt: - target=ts - import_extension=js diff --git a/cmd/mirumd/server.go b/cmd/mirum-server/server.go index 11585d9..5b4bbf0 100644 --- a/cmd/mirumd/server.go +++ b/cmd/mirum-server/server.go @@ -13,7 +13,7 @@ import ( "dimidiumlabs/mirum/internal/config" "dimidiumlabs/mirum/internal/forges" - "dimidiumlabs/mirum/internal/protocol/pb" + "dimidiumlabs/mirum/internal/protocol/wirepb" ) // server holds the shared application state. @@ -22,7 +22,7 @@ type server struct { db *DB forge forges.Forge - queue chan *pb.Task + queue chan *wirepb.Task tasks sync.Map // task_id → *forges.PushEvent taskCounter atomic.Int64 } @@ -60,7 +60,7 @@ func (s *server) enqueue(ev *forges.PushEvent) string { s.tasks.Store(id, ev) _ = s.forge.SetStatus(context.Background(), ev, forges.StatusPending, "Queued") - s.queue <- &pb.Task{ + s.queue <- &wirepb.Task{ Id: id, CloneUrl: s.forge.AuthURL(ev.CloneURL), Branch: ev.Branch, diff --git a/cmd/mirumd/server_admin.go b/cmd/mirum-server/server_admin.go index 7e1dae1..a79c377 100644 --- a/cmd/mirumd/server_admin.go +++ b/cmd/mirum-server/server_admin.go @@ -13,21 +13,21 @@ import ( "connectrpc.com/validate" "google.golang.org/protobuf/types/known/timestamppb" - "dimidiumlabs/mirum/internal/protocol/pb" - "dimidiumlabs/mirum/internal/protocol/pb/pbconnect" + "dimidiumlabs/mirum/cmd/mirum-server/apipb" + "dimidiumlabs/mirum/cmd/mirum-server/apipb/apipbconnect" ) -// NewAdminHandler creates the ConnectRPC handler with validation. +// NewConsoleHandler creates the ConnectRPC handler with validation. // Authorization is handled inside DB methods, not by an interceptor. -func NewAdminHandler(srv *server) (string, http.Handler) { - as := &adminService{srv: srv} - return pbconnect.NewAdminHandler(as, +func NewConsoleHandler(srv *server) (string, http.Handler) { + as := &consoleService{srv: srv} + return apipbconnect.NewConsoleHandler(as, connect.WithInterceptors(validate.NewInterceptor()), ) } -type adminService struct { - pbconnect.UnimplementedAdminHandler +type consoleService struct { + apipbconnect.UnimplementedConsoleHandler srv *server } @@ -36,9 +36,9 @@ type adminService struct { // newAPIError builds a ConnectError with an empty message string and attaches // an ErrorInfo detail carrying the domain reason. Clients switch on reason to // pick user-facing text; the wire never carries human-readable strings. -func newAPIError(code connect.Code, reason pb.ErrorReason, metadata map[string]string) error { +func newAPIError(code connect.Code, reason apipb.ErrorReason, metadata map[string]string) error { e := connect.NewError(code, nil) - if d, err := connect.NewErrorDetail(&pb.ErrorInfo{Reason: reason, Metadata: metadata}); err == nil { + if d, err := connect.NewErrorDetail(&apipb.ErrorInfo{Reason: reason, Metadata: metadata}); err == nil { e.AddDetail(d) } return e @@ -47,23 +47,23 @@ func newAPIError(code connect.Code, reason pb.ErrorReason, metadata map[string]s var errSpecs = []struct { err error code connect.Code - reason pb.ErrorReason + reason apipb.ErrorReason }{ - {ErrUserNotFound, connect.CodeNotFound, pb.ErrorReason_ERROR_REASON_USER_NOT_FOUND}, - {ErrOrgNotFound, connect.CodeNotFound, pb.ErrorReason_ERROR_REASON_ORG_NOT_FOUND}, - {ErrWorkerNotFound, connect.CodeNotFound, pb.ErrorReason_ERROR_REASON_WORKER_NOT_FOUND}, - {ErrNotMember, connect.CodeNotFound, pb.ErrorReason_ERROR_REASON_MEMBER_NOT_FOUND}, - {ErrEmailTaken, connect.CodeAlreadyExists, pb.ErrorReason_ERROR_REASON_EMAIL_TAKEN}, - {ErrSlugTaken, connect.CodeAlreadyExists, pb.ErrorReason_ERROR_REASON_SLUG_TAKEN}, - {ErrAlreadyMember, connect.CodeAlreadyExists, pb.ErrorReason_ERROR_REASON_ALREADY_MEMBER}, - {ErrLastOwner, connect.CodeFailedPrecondition, pb.ErrorReason_ERROR_REASON_LAST_OWNER}, - {ErrSoleOwner, connect.CodeFailedPrecondition, pb.ErrorReason_ERROR_REASON_SOLE_OWNER}, - {ErrInvalidSlug, connect.CodeInvalidArgument, pb.ErrorReason_ERROR_REASON_INVALID_SLUG}, - {ErrInvalidRole, connect.CodeInvalidArgument, pb.ErrorReason_ERROR_REASON_INVALID_ROLE}, - {ErrReservedEmail, connect.CodeInvalidArgument, pb.ErrorReason_ERROR_REASON_RESERVED_EMAIL}, - {ErrPermissionDenied, connect.CodePermissionDenied, pb.ErrorReason_ERROR_REASON_PERMISSION_DENIED}, - {ErrUnauthenticated, connect.CodeUnauthenticated, pb.ErrorReason_ERROR_REASON_UNAUTHENTICATED}, - {ErrNotImplemented, connect.CodeUnimplemented, pb.ErrorReason_ERROR_REASON_UNIMPLEMENTED}, + {ErrUserNotFound, connect.CodeNotFound, apipb.ErrorReason_ERROR_REASON_USER_NOT_FOUND}, + {ErrOrgNotFound, connect.CodeNotFound, apipb.ErrorReason_ERROR_REASON_ORG_NOT_FOUND}, + {ErrWorkerNotFound, connect.CodeNotFound, apipb.ErrorReason_ERROR_REASON_WORKER_NOT_FOUND}, + {ErrNotMember, connect.CodeNotFound, apipb.ErrorReason_ERROR_REASON_MEMBER_NOT_FOUND}, + {ErrEmailTaken, connect.CodeAlreadyExists, apipb.ErrorReason_ERROR_REASON_EMAIL_TAKEN}, + {ErrSlugTaken, connect.CodeAlreadyExists, apipb.ErrorReason_ERROR_REASON_SLUG_TAKEN}, + {ErrAlreadyMember, connect.CodeAlreadyExists, apipb.ErrorReason_ERROR_REASON_ALREADY_MEMBER}, + {ErrLastOwner, connect.CodeFailedPrecondition, apipb.ErrorReason_ERROR_REASON_LAST_OWNER}, + {ErrSoleOwner, connect.CodeFailedPrecondition, apipb.ErrorReason_ERROR_REASON_SOLE_OWNER}, + {ErrInvalidSlug, connect.CodeInvalidArgument, apipb.ErrorReason_ERROR_REASON_INVALID_SLUG}, + {ErrInvalidRole, connect.CodeInvalidArgument, apipb.ErrorReason_ERROR_REASON_INVALID_ROLE}, + {ErrReservedEmail, connect.CodeInvalidArgument, apipb.ErrorReason_ERROR_REASON_RESERVED_EMAIL}, + {ErrPermissionDenied, connect.CodePermissionDenied, apipb.ErrorReason_ERROR_REASON_PERMISSION_DENIED}, + {ErrUnauthenticated, connect.CodeUnauthenticated, apipb.ErrorReason_ERROR_REASON_UNAUTHENTICATED}, + {ErrNotImplemented, connect.CodeUnimplemented, apipb.ErrorReason_ERROR_REASON_UNIMPLEMENTED}, } func mapErr(err error) error { @@ -76,35 +76,35 @@ func mapErr(err error) error { } } slog.Error("unmapped handler error", "err", err) - return newAPIError(connect.CodeInternal, pb.ErrorReason_ERROR_REASON_INTERNAL, nil) + return newAPIError(connect.CodeInternal, apipb.ErrorReason_ERROR_REASON_INTERNAL, nil) } // --- Ref converters --- -func userRef(r *pb.UserRef) (UserRef, error) { +func userRef(r *apipb.UserRef) (UserRef, error) { switch v := r.GetRef().(type) { - case *pb.UserRef_Id: + case *apipb.UserRef_Id: id, err := IDFromBytes[UserKind](v.Id) if err != nil { return UserRef{}, err } return UserByID(id), nil - case *pb.UserRef_Email: + case *apipb.UserRef_Email: return UserByEmail(v.Email), nil default: return UserRef{}, nil } } -func orgRef(r *pb.OrgRef) (OrgRef, error) { +func orgRef(r *apipb.OrgRef) (OrgRef, error) { switch v := r.GetRef().(type) { - case *pb.OrgRef_Id: + case *apipb.OrgRef_Id: id, err := IDFromBytes[OrgKind](v.Id) if err != nil { return OrgRef{}, err } return OrgByID(id), nil - case *pb.OrgRef_Slug: + case *apipb.OrgRef_Slug: return OrgBySlug(v.Slug), nil default: return OrgRef{}, nil @@ -113,16 +113,16 @@ func orgRef(r *pb.OrgRef) (OrgRef, error) { // --- Role converters --- -var roleToString = map[pb.Role]string{ - pb.Role_ROLE_OWNER: "owner", - pb.Role_ROLE_ADMIN: "admin", - pb.Role_ROLE_MEMBER: "member", +var roleToString = map[apipb.Role]string{ + apipb.Role_ROLE_OWNER: "owner", + apipb.Role_ROLE_ADMIN: "admin", + apipb.Role_ROLE_MEMBER: "member", } -var roleToProto = map[string]pb.Role{ - "owner": pb.Role_ROLE_OWNER, - "admin": pb.Role_ROLE_ADMIN, - "member": pb.Role_ROLE_MEMBER, +var roleToProto = map[string]apipb.Role{ + "owner": apipb.Role_ROLE_OWNER, + "admin": apipb.Role_ROLE_ADMIN, + "member": apipb.Role_ROLE_MEMBER, } // --- Page helpers --- @@ -132,7 +132,7 @@ const ( maxPageSize = 200 ) -func pageParams[K IDKind](p *pb.PageRequest) (cursor ID[K], limit int, err error) { +func pageParams[K IDKind](p *apipb.PageRequest) (cursor ID[K], limit int, err error) { limit = defaultPageSize if p != nil { if p.PageSize > 0 && int(p.PageSize) < maxPageSize { @@ -150,8 +150,8 @@ func pageParams[K IDKind](p *pb.PageRequest) (cursor ID[K], limit int, err error return } -func pageResponse[K IDKind](items int, limit int, lastID ID[K], total int) *pb.PageResponse { - resp := &pb.PageResponse{TotalCount: int32(total)} +func pageResponse[K IDKind](items int, limit int, lastID ID[K], total int) *apipb.PageResponse { + resp := &apipb.PageResponse{TotalCount: int32(total)} if items == limit { resp.NextCursor = lastID.Bytes() } @@ -160,28 +160,28 @@ func pageResponse[K IDKind](items int, limit int, lastID ID[K], total int) *pb.P // --- Proto converters --- -func userToProto(u User) *pb.User { - return &pb.User{ +func userToProto(u User) *apipb.User { + return &apipb.User{ Id: u.ID.Bytes(), Email: u.Email, CreatedAt: timestamppb.New(u.CreatedAt), } } -func orgToProto(o Organization) *pb.Org { - return &pb.Org{ +func orgToProto(o Organization) *apipb.Org { + return &apipb.Org{ Id: o.ID.Bytes(), Name: o.Name, Slug: o.Slug, Public: o.Public, CreatedAt: timestamppb.New(o.CreatedAt), } } -func memberToProto(m OrgMember) *pb.OrgMemberInfo { - return &pb.OrgMemberInfo{ +func memberToProto(m OrgMember) *apipb.OrgMemberInfo { + return &apipb.OrgMemberInfo{ User: userToProto(m.User), Role: roleToProto[m.Role], JoinedAt: timestamppb.New(m.JoinedAt), } } -func workerToProto(w Worker) *pb.Worker { - pw := &pb.Worker{ +func workerToProto(w Worker) *apipb.Worker { + pw := &apipb.Worker{ Id: w.ID.Bytes(), PublicKey: w.PublicKey, CreatedAt: timestamppb.New(w.CreatedAt), } if w.OrgID != nil { @@ -192,15 +192,15 @@ func workerToProto(w Worker) *pb.Worker { // --- User handlers --- -func (a *adminService) UserCreate(ctx context.Context, req *connect.Request[pb.UserCreateRequest]) (*connect.Response[pb.UserCreateResponse], error) { +func (a *consoleService) UserCreate(ctx context.Context, req *connect.Request[apipb.UserCreateRequest]) (*connect.Response[apipb.UserCreateResponse], error) { id, err := a.srv.db.UserCreate(ctx, ActorFromContext(ctx), req.Msg.Email, req.Msg.Password, []byte(a.srv.cfg.Pepper)) if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.UserCreateResponse{Id: id.Bytes()}), nil + return connect.NewResponse(&apipb.UserCreateResponse{Id: id.Bytes()}), nil } -func (a *adminService) UserGet(ctx context.Context, req *connect.Request[pb.UserGetRequest]) (*connect.Response[pb.UserGetResponse], error) { +func (a *consoleService) UserGet(ctx context.Context, req *connect.Request[apipb.UserGetRequest]) (*connect.Response[apipb.UserGetResponse], error) { ref, err := userRef(req.Msg.User) if err != nil { return nil, mapErr(err) @@ -209,10 +209,10 @@ func (a *adminService) UserGet(ctx context.Context, req *connect.Request[pb.User if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.UserGetResponse{User: userToProto(*u)}), nil + return connect.NewResponse(&apipb.UserGetResponse{User: userToProto(*u)}), nil } -func (a *adminService) UserList(ctx context.Context, req *connect.Request[pb.UserListRequest]) (*connect.Response[pb.UserListResponse], error) { +func (a *consoleService) UserList(ctx context.Context, req *connect.Request[apipb.UserListRequest]) (*connect.Response[apipb.UserListResponse], error) { cursor, limit, err := pageParams[UserKind](req.Msg.Page) if err != nil { return nil, mapErr(err) @@ -227,7 +227,7 @@ func (a *adminService) UserList(ctx context.Context, req *connect.Request[pb.Use return nil, mapErr(err) } - out := make([]*pb.User, len(users)) + out := make([]*apipb.User, len(users)) for i := range users { out[i] = userToProto(users[i]) } @@ -237,13 +237,13 @@ func (a *adminService) UserList(ctx context.Context, req *connect.Request[pb.Use lastID = users[len(users)-1].ID } - return connect.NewResponse(&pb.UserListResponse{ + return connect.NewResponse(&apipb.UserListResponse{ Page: pageResponse(len(users), limit, lastID, total), Users: out, }), nil } -func (a *adminService) UserUpdate(ctx context.Context, req *connect.Request[pb.UserUpdateRequest]) (*connect.Response[pb.UserUpdateResponse], error) { +func (a *consoleService) UserUpdate(ctx context.Context, req *connect.Request[apipb.UserUpdateRequest]) (*connect.Response[apipb.UserUpdateResponse], error) { ref, err := userRef(req.Msg.User) if err != nil { return nil, mapErr(err) @@ -251,10 +251,10 @@ func (a *adminService) UserUpdate(ctx context.Context, req *connect.Request[pb.U if err := a.srv.db.UserUpdate(ctx, ActorFromContext(ctx), ref, req.Msg.Email, req.Msg.Password, []byte(a.srv.cfg.Pepper)); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.UserUpdateResponse{}), nil + return connect.NewResponse(&apipb.UserUpdateResponse{}), nil } -func (a *adminService) UserDelete(ctx context.Context, req *connect.Request[pb.UserDeleteRequest]) (*connect.Response[pb.UserDeleteResponse], error) { +func (a *consoleService) UserDelete(ctx context.Context, req *connect.Request[apipb.UserDeleteRequest]) (*connect.Response[apipb.UserDeleteResponse], error) { ref, err := userRef(req.Msg.User) if err != nil { return nil, mapErr(err) @@ -262,12 +262,12 @@ func (a *adminService) UserDelete(ctx context.Context, req *connect.Request[pb.U if err := a.srv.db.UserDelete(ctx, ActorFromContext(ctx), ref); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.UserDeleteResponse{}), nil + return connect.NewResponse(&apipb.UserDeleteResponse{}), nil } // --- Org handlers --- -func (a *adminService) OrgCreate(ctx context.Context, req *connect.Request[pb.OrgCreateRequest]) (*connect.Response[pb.OrgCreateResponse], error) { +func (a *consoleService) OrgCreate(ctx context.Context, req *connect.Request[apipb.OrgCreateRequest]) (*connect.Response[apipb.OrgCreateResponse], error) { slug, err := ValidateSlug(req.Msg.Slug) if err != nil { return nil, mapErr(err) @@ -280,10 +280,10 @@ func (a *adminService) OrgCreate(ctx context.Context, req *connect.Request[pb.Or if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgCreateResponse{Id: id.Bytes()}), nil + return connect.NewResponse(&apipb.OrgCreateResponse{Id: id.Bytes()}), nil } -func (a *adminService) OrgGet(ctx context.Context, req *connect.Request[pb.OrgGetRequest]) (*connect.Response[pb.OrgGetResponse], error) { +func (a *consoleService) OrgGet(ctx context.Context, req *connect.Request[apipb.OrgGetRequest]) (*connect.Response[apipb.OrgGetResponse], error) { ref, err := orgRef(req.Msg.Org) if err != nil { return nil, mapErr(err) @@ -292,10 +292,10 @@ func (a *adminService) OrgGet(ctx context.Context, req *connect.Request[pb.OrgGe if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgGetResponse{Org: orgToProto(*o)}), nil + return connect.NewResponse(&apipb.OrgGetResponse{Org: orgToProto(*o)}), nil } -func (a *adminService) OrgList(ctx context.Context, req *connect.Request[pb.OrgListRequest]) (*connect.Response[pb.OrgListResponse], error) { +func (a *consoleService) OrgList(ctx context.Context, req *connect.Request[apipb.OrgListRequest]) (*connect.Response[apipb.OrgListResponse], error) { cursor, limit, err := pageParams[OrgKind](req.Msg.Page) if err != nil { return nil, mapErr(err) @@ -310,7 +310,7 @@ func (a *adminService) OrgList(ctx context.Context, req *connect.Request[pb.OrgL return nil, mapErr(err) } - out := make([]*pb.Org, len(orgs)) + out := make([]*apipb.Org, len(orgs)) for i := range orgs { out[i] = orgToProto(orgs[i]) } @@ -320,13 +320,13 @@ func (a *adminService) OrgList(ctx context.Context, req *connect.Request[pb.OrgL lastID = orgs[len(orgs)-1].ID } - return connect.NewResponse(&pb.OrgListResponse{ + return connect.NewResponse(&apipb.OrgListResponse{ Page: pageResponse(len(orgs), limit, lastID, total), Organizations: out, }), nil } -func (a *adminService) OrgUpdate(ctx context.Context, req *connect.Request[pb.OrgUpdateRequest]) (*connect.Response[pb.OrgUpdateResponse], error) { +func (a *consoleService) OrgUpdate(ctx context.Context, req *connect.Request[apipb.OrgUpdateRequest]) (*connect.Response[apipb.OrgUpdateResponse], error) { var slug *string if req.Msg.Slug != nil { s, err := ValidateSlug(*req.Msg.Slug) @@ -342,10 +342,10 @@ func (a *adminService) OrgUpdate(ctx context.Context, req *connect.Request[pb.Or if err := a.srv.db.OrgUpdate(ctx, ActorFromContext(ctx), ref, req.Msg.Name, slug, req.Msg.Public); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgUpdateResponse{}), nil + return connect.NewResponse(&apipb.OrgUpdateResponse{}), nil } -func (a *adminService) OrgDelete(ctx context.Context, req *connect.Request[pb.OrgDeleteRequest]) (*connect.Response[pb.OrgDeleteResponse], error) { +func (a *consoleService) OrgDelete(ctx context.Context, req *connect.Request[apipb.OrgDeleteRequest]) (*connect.Response[apipb.OrgDeleteResponse], error) { ref, err := orgRef(req.Msg.Org) if err != nil { return nil, mapErr(err) @@ -353,15 +353,15 @@ func (a *adminService) OrgDelete(ctx context.Context, req *connect.Request[pb.Or if err := a.srv.db.OrgDelete(ctx, ActorFromContext(ctx), ref); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgDeleteResponse{}), nil + return connect.NewResponse(&apipb.OrgDeleteResponse{}), nil } // --- OrgMember handlers --- -func (a *adminService) OrgMemberAdd(ctx context.Context, req *connect.Request[pb.OrgMemberAddRequest]) (*connect.Response[pb.OrgMemberAddResponse], error) { +func (a *consoleService) OrgMemberAdd(ctx context.Context, req *connect.Request[apipb.OrgMemberAddRequest]) (*connect.Response[apipb.OrgMemberAddResponse], error) { role, ok := roleToString[req.Msg.Role] if !ok { - return nil, newAPIError(connect.CodeInvalidArgument, pb.ErrorReason_ERROR_REASON_INVALID_ROLE, nil) + return nil, newAPIError(connect.CodeInvalidArgument, apipb.ErrorReason_ERROR_REASON_INVALID_ROLE, nil) } org, err := orgRef(req.Msg.Org) if err != nil { @@ -374,10 +374,10 @@ func (a *adminService) OrgMemberAdd(ctx context.Context, req *connect.Request[pb if err := a.srv.db.OrgMemberAdd(ctx, ActorFromContext(ctx), org, user, role); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgMemberAddResponse{}), nil + return connect.NewResponse(&apipb.OrgMemberAddResponse{}), nil } -func (a *adminService) OrgMemberGet(ctx context.Context, req *connect.Request[pb.OrgMemberGetRequest]) (*connect.Response[pb.OrgMemberGetResponse], error) { +func (a *consoleService) OrgMemberGet(ctx context.Context, req *connect.Request[apipb.OrgMemberGetRequest]) (*connect.Response[apipb.OrgMemberGetResponse], error) { org, err := orgRef(req.Msg.Org) if err != nil { return nil, mapErr(err) @@ -390,10 +390,10 @@ func (a *adminService) OrgMemberGet(ctx context.Context, req *connect.Request[pb if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgMemberGetResponse{Member: memberToProto(*m)}), nil + return connect.NewResponse(&apipb.OrgMemberGetResponse{Member: memberToProto(*m)}), nil } -func (a *adminService) OrgMemberList(ctx context.Context, req *connect.Request[pb.OrgMemberListRequest]) (*connect.Response[pb.OrgMemberListResponse], error) { +func (a *consoleService) OrgMemberList(ctx context.Context, req *connect.Request[apipb.OrgMemberListRequest]) (*connect.Response[apipb.OrgMemberListResponse], error) { cursor, limit, err := pageParams[UserKind](req.Msg.Page) if err != nil { return nil, mapErr(err) @@ -412,7 +412,7 @@ func (a *adminService) OrgMemberList(ctx context.Context, req *connect.Request[p return nil, mapErr(err) } - out := make([]*pb.OrgMemberInfo, len(members)) + out := make([]*apipb.OrgMemberInfo, len(members)) for i := range members { out[i] = memberToProto(members[i]) } @@ -422,16 +422,16 @@ func (a *adminService) OrgMemberList(ctx context.Context, req *connect.Request[p lastID = members[len(members)-1].User.ID } - return connect.NewResponse(&pb.OrgMemberListResponse{ + return connect.NewResponse(&apipb.OrgMemberListResponse{ Page: pageResponse(len(members), limit, lastID, total), Members: out, }), nil } -func (a *adminService) OrgMemberUpdate(ctx context.Context, req *connect.Request[pb.OrgMemberUpdateRequest]) (*connect.Response[pb.OrgMemberUpdateResponse], error) { +func (a *consoleService) OrgMemberUpdate(ctx context.Context, req *connect.Request[apipb.OrgMemberUpdateRequest]) (*connect.Response[apipb.OrgMemberUpdateResponse], error) { role, ok := roleToString[req.Msg.Role] if !ok { - return nil, newAPIError(connect.CodeInvalidArgument, pb.ErrorReason_ERROR_REASON_INVALID_ROLE, nil) + return nil, newAPIError(connect.CodeInvalidArgument, apipb.ErrorReason_ERROR_REASON_INVALID_ROLE, nil) } org, err := orgRef(req.Msg.Org) if err != nil { @@ -444,10 +444,10 @@ func (a *adminService) OrgMemberUpdate(ctx context.Context, req *connect.Request if err := a.srv.db.OrgMemberUpdateRole(ctx, ActorFromContext(ctx), org, user, role); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgMemberUpdateResponse{}), nil + return connect.NewResponse(&apipb.OrgMemberUpdateResponse{}), nil } -func (a *adminService) OrgMemberRemove(ctx context.Context, req *connect.Request[pb.OrgMemberRemoveRequest]) (*connect.Response[pb.OrgMemberRemoveResponse], error) { +func (a *consoleService) OrgMemberRemove(ctx context.Context, req *connect.Request[apipb.OrgMemberRemoveRequest]) (*connect.Response[apipb.OrgMemberRemoveResponse], error) { org, err := orgRef(req.Msg.Org) if err != nil { return nil, mapErr(err) @@ -459,12 +459,12 @@ func (a *adminService) OrgMemberRemove(ctx context.Context, req *connect.Request if err := a.srv.db.OrgMemberRemove(ctx, ActorFromContext(ctx), org, user); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.OrgMemberRemoveResponse{}), nil + return connect.NewResponse(&apipb.OrgMemberRemoveResponse{}), nil } // --- Worker handlers --- -func (a *adminService) WorkerCreate(ctx context.Context, req *connect.Request[pb.WorkerCreateRequest]) (*connect.Response[pb.WorkerCreateResponse], error) { +func (a *consoleService) WorkerCreate(ctx context.Context, req *connect.Request[apipb.WorkerCreateRequest]) (*connect.Response[apipb.WorkerCreateResponse], error) { var org *OrgRef if req.Msg.Org != nil { r, err := orgRef(req.Msg.Org) @@ -477,10 +477,10 @@ func (a *adminService) WorkerCreate(ctx context.Context, req *connect.Request[pb if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.WorkerCreateResponse{Id: id.Bytes()}), nil + return connect.NewResponse(&apipb.WorkerCreateResponse{Id: id.Bytes()}), nil } -func (a *adminService) WorkerGet(ctx context.Context, req *connect.Request[pb.WorkerGetRequest]) (*connect.Response[pb.WorkerGetResponse], error) { +func (a *consoleService) WorkerGet(ctx context.Context, req *connect.Request[apipb.WorkerGetRequest]) (*connect.Response[apipb.WorkerGetResponse], error) { wid, err := IDFromBytes[WorkerKind](req.Msg.Id) if err != nil { return nil, mapErr(err) @@ -489,10 +489,10 @@ func (a *adminService) WorkerGet(ctx context.Context, req *connect.Request[pb.Wo if err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.WorkerGetResponse{Worker: workerToProto(*w)}), nil + return connect.NewResponse(&apipb.WorkerGetResponse{Worker: workerToProto(*w)}), nil } -func (a *adminService) WorkerList(ctx context.Context, req *connect.Request[pb.WorkerListRequest]) (*connect.Response[pb.WorkerListResponse], error) { +func (a *consoleService) WorkerList(ctx context.Context, req *connect.Request[apipb.WorkerListRequest]) (*connect.Response[apipb.WorkerListResponse], error) { cursor, limit, err := pageParams[WorkerKind](req.Msg.Page) if err != nil { return nil, mapErr(err) @@ -507,7 +507,7 @@ func (a *adminService) WorkerList(ctx context.Context, req *connect.Request[pb.W return nil, mapErr(err) } - out := make([]*pb.Worker, len(workers)) + out := make([]*apipb.Worker, len(workers)) for i := range workers { out[i] = workerToProto(workers[i]) } @@ -517,13 +517,13 @@ func (a *adminService) WorkerList(ctx context.Context, req *connect.Request[pb.W lastID = workers[len(workers)-1].ID } - return connect.NewResponse(&pb.WorkerListResponse{ + return connect.NewResponse(&apipb.WorkerListResponse{ Page: pageResponse(len(workers), limit, lastID, total), Workers: out, }), nil } -func (a *adminService) WorkerDelete(ctx context.Context, req *connect.Request[pb.WorkerDeleteRequest]) (*connect.Response[pb.WorkerDeleteResponse], error) { +func (a *consoleService) WorkerDelete(ctx context.Context, req *connect.Request[apipb.WorkerDeleteRequest]) (*connect.Response[apipb.WorkerDeleteResponse], error) { wid, err := IDFromBytes[WorkerKind](req.Msg.Id) if err != nil { return nil, mapErr(err) @@ -531,5 +531,5 @@ func (a *adminService) WorkerDelete(ctx context.Context, req *connect.Request[pb if err := a.srv.db.WorkerDelete(ctx, ActorFromContext(ctx), wid); err != nil { return nil, mapErr(err) } - return connect.NewResponse(&pb.WorkerDeleteResponse{}), nil + return connect.NewResponse(&apipb.WorkerDeleteResponse{}), nil } diff --git a/cmd/mirumd/server_grpc.go b/cmd/mirum-server/server_grpc.go index a41604d..a2f7cf0 100644 --- a/cmd/mirumd/server_grpc.go +++ b/cmd/mirum-server/server_grpc.go @@ -20,14 +20,14 @@ import ( "dimidiumlabs/mirum/internal/config" "dimidiumlabs/mirum/internal/protocol" - "dimidiumlabs/mirum/internal/protocol/pb" - "dimidiumlabs/mirum/internal/protocol/pb/pbconnect" + "dimidiumlabs/mirum/internal/protocol/wirepb" + "dimidiumlabs/mirum/internal/protocol/wirepb/wirepbconnect" ) func NewGrpcServer(ctx context.Context, srv *server) *http.Server { gsrv := &grpcService{srv: srv} - path, handler := pbconnect.NewMirumHandler(gsrv, + path, handler := wirepbconnect.NewWorkerHandler(gsrv, connect.WithInterceptors(validate.NewInterceptor()), ) @@ -80,11 +80,11 @@ func NewGrpcServer(ctx context.Context, srv *server) *http.Server { // grpcService is the ConnectRPC transport adapter over server. type grpcService struct { - pbconnect.UnimplementedMirumHandler + wirepbconnect.UnimplementedWorkerHandler srv *server } -func (g *grpcService) Poll(ctx context.Context, req *connect.Request[pb.PollRequest]) (*connect.Response[pb.Task], error) { +func (g *grpcService) Poll(ctx context.Context, req *connect.Request[wirepb.PollRequest]) (*connect.Response[wirepb.Task], error) { select { case task, ok := <-g.srv.queue: if !ok { @@ -97,11 +97,11 @@ func (g *grpcService) Poll(ctx context.Context, req *connect.Request[pb.PollRequ } } -func (g *grpcService) Complete(ctx context.Context, req *connect.Request[pb.TaskResult]) (*connect.Response[pb.CompleteResponse], error) { +func (g *grpcService) Complete(ctx context.Context, req *connect.Request[wirepb.TaskResult]) (*connect.Response[wirepb.CompleteResponse], error) { if err := g.srv.complete(ctx, req.Msg.TaskId, req.Msg.Success, req.Msg.Error); err != nil { return nil, err } - return connect.NewResponse(&pb.CompleteResponse{}), nil + return connect.NewResponse(&wirepb.CompleteResponse{}), nil } // workerLog logs worker metadata from the mTLS client certificate diff --git a/cmd/mirumd/server_web.go b/cmd/mirum-server/server_web.go index 13c5154..8e62e53 100644 --- a/cmd/mirumd/server_web.go +++ b/cmd/mirum-server/server_web.go @@ -21,9 +21,9 @@ import ( "github.com/go-chi/chi/v5/middleware" "github.com/go-chi/httprate" + "dimidiumlabs/mirum/cmd/mirum-server/apipb" "dimidiumlabs/mirum/internal/config" "dimidiumlabs/mirum/internal/forges" - "dimidiumlabs/mirum/internal/protocol/pb" ) // __Host- prefixed cookies can only be set with Secure, Path=/, and no @@ -34,7 +34,7 @@ const ( csrfCookie = "__Host-csrf" ) -func NewWebServer(ctx context.Context, srv *server, adminPath string, adminHandler http.Handler) *http.Server { +func NewWebServer(ctx context.Context, srv *server, consolePath string, consoleHandler http.Handler) *http.Server { h := &webHandler{ srv: srv, assets: newAssetResolver(), @@ -90,7 +90,7 @@ func NewWebServer(ctx context.Context, srv *server, adminPath string, adminHandl }) r.With(middleware.NoCache, httprate.LimitByIP(config.APIRateLimit, config.APIRateWindow)). - Mount("/api/v1", http.StripPrefix("/api/v1", adminHandler)) + Mount("/api/v1", http.StripPrefix("/api/v1", consoleHandler)) r.NotFound(func(w http.ResponseWriter, r *http.Request) { h.renderError(w, r, http.StatusNotFound) @@ -221,15 +221,15 @@ func (h *webHandler) webhook(w http.ResponseWriter, r *http.Request) { } func (h *webHandler) loginPage(w http.ResponseWriter, r *http.Request) { - h.renderLogin(w, r, http.StatusOK, pb.ErrorReason_ERROR_REASON_UNSPECIFIED) + h.renderLogin(w, r, http.StatusOK, apipb.ErrorReason_ERROR_REASON_UNSPECIFIED) } // renderLogin is the single entry point for every login-flow outcome that // lands back on the login page. Reason == UNSPECIFIED means no error banner. // No caller writes error text itself — the client maps reason → copy. -func (h *webHandler) renderLogin(w http.ResponseWriter, r *http.Request, status int, reason pb.ErrorReason) { +func (h *webHandler) renderLogin(w http.ResponseWriter, r *http.Request, status int, reason apipb.ErrorReason) { data := map[string]any{"csrf": csrfToken(w, r)} - if reason != pb.ErrorReason_ERROR_REASON_UNSPECIFIED { + if reason != apipb.ErrorReason_ERROR_REASON_UNSPECIFIED { data["errorReason"] = int32(reason) } h.assets.renderPage(w, "login", status, data) @@ -238,7 +238,7 @@ func (h *webHandler) renderLogin(w http.ResponseWriter, r *http.Request, status func (h *webHandler) login(w http.ResponseWriter, r *http.Request) { if !csrfOK(r) { clearCookie(w, csrfCookie) - h.renderLogin(w, r, http.StatusForbidden, pb.ErrorReason_ERROR_REASON_INVALID_CSRF) + h.renderLogin(w, r, http.StatusForbidden, apipb.ErrorReason_ERROR_REASON_INVALID_CSRF) return } @@ -247,14 +247,14 @@ func (h *webHandler) login(w http.ResponseWriter, r *http.Request) { userID, err := h.srv.db.UserVerifyPassword(r.Context(), SystemActor(), email, password, []byte(h.srv.cfg.Pepper)) if err != nil { - h.renderLogin(w, r, http.StatusUnauthorized, pb.ErrorReason_ERROR_REASON_INVALID_CREDENTIALS) + h.renderLogin(w, r, http.StatusUnauthorized, apipb.ErrorReason_ERROR_REASON_INVALID_CREDENTIALS) return } token, err := h.srv.db.UserSessionCreate(r.Context(), SystemActor(), userID) if err != nil { slog.Error("create session failed", "err", err) - h.renderLogin(w, r, http.StatusInternalServerError, pb.ErrorReason_ERROR_REASON_INTERNAL) + h.renderLogin(w, r, http.StatusInternalServerError, apipb.ErrorReason_ERROR_REASON_INTERNAL) return } diff --git a/cmd/mirumd/static.go b/cmd/mirum-server/static.go index bc304ba..bc304ba 100644 --- a/cmd/mirumd/static.go +++ b/cmd/mirum-server/static.go diff --git a/cmd/mirumd/static_dev.go b/cmd/mirum-server/static_dev.go index 2f6d559..2f6d559 100644 --- a/cmd/mirumd/static_dev.go +++ b/cmd/mirum-server/static_dev.go diff --git a/cmd/mirumd/static_prod.go b/cmd/mirum-server/static_prod.go index 891b04e..891b04e 100644 --- a/cmd/mirumd/static_prod.go +++ b/cmd/mirum-server/static_prod.go diff --git a/cmd/mirumd/web/api/client.ts b/cmd/mirum-server/web/api/client.ts index 5d0ebc3..669fd60 100644 --- a/cmd/mirumd/web/api/client.ts +++ b/cmd/mirum-server/web/api/client.ts @@ -3,7 +3,7 @@ import { createClient, type Interceptor } from "@connectrpc/connect" import { createConnectTransport } from "@connectrpc/connect-web" -import { Admin, ErrorReason } from "@/gen/admin_pb" +import { Console, ErrorReason } from "@/gen/api_pb" import { errorReason } from "@/lib/errors" const csrfInterceptor = (csrfToken: string): Interceptor => @@ -25,10 +25,10 @@ const authInterceptor: Interceptor = (next) => async (req) => { } } -export function createAdminClient(csrfToken: string) { +export function createConsoleClient(csrfToken: string) { const transport = createConnectTransport({ baseUrl: "/api/v1", interceptors: [csrfInterceptor(csrfToken), authInterceptor], }) - return createClient(Admin, transport) + return createClient(Console, transport) } diff --git a/cmd/mirumd/web/components.json b/cmd/mirum-server/web/components.json index a2f7d0b..a2f7d0b 100644 --- a/cmd/mirumd/web/components.json +++ b/cmd/mirum-server/web/components.json diff --git a/cmd/mirumd/web/components/pages/dashboard.tsx b/cmd/mirum-server/web/components/pages/dashboard.tsx index 51c2f83..dcfd8ba 100644 --- a/cmd/mirumd/web/components/pages/dashboard.tsx +++ b/cmd/mirum-server/web/components/pages/dashboard.tsx @@ -2,9 +2,9 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import { useEffect, useMemo, useState } from "react" -import { createAdminClient } from "@/api/client" +import { createConsoleClient } from "@/api/client" import { formatError } from "@/lib/errors" -import type { Org } from "@/gen/admin_pb" +import type { Org } from "@/gen/api_pb" import { Button } from "@/components/ui/button" import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" @@ -14,7 +14,7 @@ export type DashboardProps = { } export function Page({ user, csrf }: DashboardProps) { - const client = useMemo(() => createAdminClient(csrf), [csrf]) + const client = useMemo(() => createConsoleClient(csrf), [csrf]) const [orgs, setOrgs] = useState<Org[] | null>(null) const [error, setError] = useState<string | null>(null) diff --git a/cmd/mirumd/web/components/pages/error.tsx b/cmd/mirum-server/web/components/pages/error.tsx index 9184ed2..9184ed2 100644 --- a/cmd/mirumd/web/components/pages/error.tsx +++ b/cmd/mirum-server/web/components/pages/error.tsx diff --git a/cmd/mirumd/web/components/pages/login.tsx b/cmd/mirum-server/web/components/pages/login.tsx index 71d07db..4e85144 100644 --- a/cmd/mirumd/web/components/pages/login.tsx +++ b/cmd/mirum-server/web/components/pages/login.tsx @@ -7,7 +7,7 @@ import { Button } from "@/components/ui/button" import { Input } from "@/components/ui/input" import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Field, FieldDescription, FieldGroup, FieldLabel } from "@/components/ui/field" -import { ErrorReason } from "@/gen/admin_pb" +import { ErrorReason } from "@/gen/api_pb" import { textForReason } from "@/lib/errors" export type LoginFormProps = { diff --git a/cmd/mirumd/web/components/ui/alert.tsx b/cmd/mirum-server/web/components/ui/alert.tsx index 746962e..746962e 100644 --- a/cmd/mirumd/web/components/ui/alert.tsx +++ b/cmd/mirum-server/web/components/ui/alert.tsx diff --git a/cmd/mirumd/web/components/ui/button.tsx b/cmd/mirum-server/web/components/ui/button.tsx index ca6d679..ca6d679 100644 --- a/cmd/mirumd/web/components/ui/button.tsx +++ b/cmd/mirum-server/web/components/ui/button.tsx diff --git a/cmd/mirumd/web/components/ui/card.tsx b/cmd/mirum-server/web/components/ui/card.tsx index 9859e5f..9859e5f 100644 --- a/cmd/mirumd/web/components/ui/card.tsx +++ b/cmd/mirum-server/web/components/ui/card.tsx diff --git a/cmd/mirumd/web/components/ui/field.tsx b/cmd/mirum-server/web/components/ui/field.tsx index 9517f55..9517f55 100644 --- a/cmd/mirumd/web/components/ui/field.tsx +++ b/cmd/mirum-server/web/components/ui/field.tsx diff --git a/cmd/mirumd/web/components/ui/input.tsx b/cmd/mirum-server/web/components/ui/input.tsx index 2bd6cad..2bd6cad 100644 --- a/cmd/mirumd/web/components/ui/input.tsx +++ b/cmd/mirum-server/web/components/ui/input.tsx diff --git a/cmd/mirumd/web/components/ui/label.tsx b/cmd/mirum-server/web/components/ui/label.tsx index 48d4304..48d4304 100644 --- a/cmd/mirumd/web/components/ui/label.tsx +++ b/cmd/mirum-server/web/components/ui/label.tsx diff --git a/cmd/mirumd/web/components/ui/separator.tsx b/cmd/mirum-server/web/components/ui/separator.tsx index 3c2a347..3c2a347 100644 --- a/cmd/mirumd/web/components/ui/separator.tsx +++ b/cmd/mirum-server/web/components/ui/separator.tsx diff --git a/cmd/mirumd/web/entries/dashboard.tsx b/cmd/mirum-server/web/entries/dashboard.tsx index 0921015..0921015 100644 --- a/cmd/mirumd/web/entries/dashboard.tsx +++ b/cmd/mirum-server/web/entries/dashboard.tsx diff --git a/cmd/mirumd/web/entries/error.tsx b/cmd/mirum-server/web/entries/error.tsx index 10b509e..10b509e 100644 --- a/cmd/mirumd/web/entries/error.tsx +++ b/cmd/mirum-server/web/entries/error.tsx diff --git a/cmd/mirumd/web/entries/login.tsx b/cmd/mirum-server/web/entries/login.tsx index 36428f9..36428f9 100644 --- a/cmd/mirumd/web/entries/login.tsx +++ b/cmd/mirum-server/web/entries/login.tsx diff --git a/cmd/mirumd/web/index.css b/cmd/mirum-server/web/index.css index 1c06db4..1c06db4 100644 --- a/cmd/mirumd/web/index.css +++ b/cmd/mirum-server/web/index.css diff --git a/cmd/mirumd/web/lib/errors.ts b/cmd/mirum-server/web/lib/errors.ts index cf8a20b..2244d1b 100644 --- a/cmd/mirumd/web/lib/errors.ts +++ b/cmd/mirum-server/web/lib/errors.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: AGPL-3.0-or-later import { Code, ConnectError } from "@connectrpc/connect" -import { ErrorInfoSchema, ErrorReason } from "@/gen/admin_pb" +import { ErrorInfoSchema, ErrorReason } from "@/gen/api_pb" // errorReason extracts the ErrorInfo.reason attached by the server. // Returns null for transport failures or non-mirum responses. diff --git a/cmd/mirumd/web/lib/mount.tsx b/cmd/mirum-server/web/lib/mount.tsx index 2c026ab..2c026ab 100644 --- a/cmd/mirumd/web/lib/mount.tsx +++ b/cmd/mirum-server/web/lib/mount.tsx diff --git a/cmd/mirumd/web/lib/utils.ts b/cmd/mirum-server/web/lib/utils.ts index 5d8fa3d..5d8fa3d 100644 --- a/cmd/mirumd/web/lib/utils.ts +++ b/cmd/mirum-server/web/lib/utils.ts diff --git a/cmd/mirumd/web/package-lock.json b/cmd/mirum-server/web/package-lock.json index aeb900c..aeb900c 100644 --- a/cmd/mirumd/web/package-lock.json +++ b/cmd/mirum-server/web/package-lock.json diff --git a/cmd/mirumd/web/package.json b/cmd/mirum-server/web/package.json index ce805ca..ce805ca 100644 --- a/cmd/mirumd/web/package.json +++ b/cmd/mirum-server/web/package.json diff --git a/cmd/mirumd/web/shell.html b/cmd/mirum-server/web/shell.html index ce74041..ce74041 100644 --- a/cmd/mirumd/web/shell.html +++ b/cmd/mirum-server/web/shell.html diff --git a/cmd/mirumd/web/tsconfig.json b/cmd/mirum-server/web/tsconfig.json index 90b2f46..90b2f46 100644 --- a/cmd/mirumd/web/tsconfig.json +++ b/cmd/mirum-server/web/tsconfig.json diff --git a/cmd/mirumd/web/vite.config.ts b/cmd/mirum-server/web/vite.config.ts index 6de11ff..6de11ff 100644 --- a/cmd/mirumd/web/vite.config.ts +++ b/cmd/mirum-server/web/vite.config.ts diff --git a/cmd/mirumw/client.go b/cmd/mirum-worker/client.go index 7f3886d..1525904 100644 --- a/cmd/mirumw/client.go +++ b/cmd/mirum-worker/client.go @@ -17,14 +17,14 @@ import ( "dimidiumlabs/mirum/internal/executor" "dimidiumlabs/mirum/internal/protocol" - "dimidiumlabs/mirum/internal/protocol/pb" - "dimidiumlabs/mirum/internal/protocol/pb/pbconnect" + "dimidiumlabs/mirum/internal/protocol/wirepb" + "dimidiumlabs/mirum/internal/protocol/wirepb/wirepbconnect" ) type client struct { cfg *config http *http.Client - handle pbconnect.MirumClient + handle wirepbconnect.WorkerClient } func dial(ctx context.Context, cfg *config) (*client, error) { @@ -75,7 +75,7 @@ func dial(ctx context.Context, cfg *config) (*client, error) { Transport: &http.Transport{TLSClientConfig: tlsCfg, ForceAttemptHTTP2: true}, }, } - c.handle = pbconnect.NewMirumClient(c.http, "https://"+cfg.Server, connect.WithGRPC()) + c.handle = wirepbconnect.NewWorkerClient(c.http, "https://"+cfg.Server, connect.WithGRPC()) slog.Info("dialing", "server", cfg.Server) @@ -86,7 +86,7 @@ func (c *client) close() {} func (c *client) work(ctx context.Context) error { for ctx.Err() == nil { - resp, err := c.handle.Poll(ctx, connect.NewRequest(&pb.PollRequest{})) + resp, err := c.handle.Poll(ctx, connect.NewRequest(&wirepb.PollRequest{})) if err != nil { return fmt.Errorf("poll: %w", err) } @@ -100,7 +100,7 @@ func (c *client) work(ctx context.Context) error { execErr := executor.Run(task.CloneUrl, task.Branch) - result := &pb.TaskResult{TaskId: task.Id, Success: execErr == nil} + result := &wirepb.TaskResult{TaskId: task.Id, Success: execErr == nil} if execErr != nil { result.Error = execErr.Error() slog.Error("task failed", "id", task.Id, "err", execErr) diff --git a/cmd/mirumw/config.go b/cmd/mirum-worker/config.go index 50e82d3..366fdf4 100644 --- a/cmd/mirumw/config.go +++ b/cmd/mirum-worker/config.go @@ -11,7 +11,7 @@ import ( ) // Runtime type of this worker binary. Different worker types -// (mirumw-vm, mirumw-docker, etc.) will have different values. +// (mirum-worker-vm, mirum-worker-docker, etc.) will have different values. const workerRuntime = "host" type config struct { diff --git a/cmd/mirumw/main.go b/cmd/mirum-worker/main.go index e2132b5..e2132b5 100644 --- a/cmd/mirumw/main.go +++ b/cmd/mirum-worker/main.go diff --git a/cmd/mirum/main.go b/cmd/mirum/main.go new file mode 100644 --- /dev/null +++ b/cmd/mirum/main.go @@ -0,0 +1,33 @@ +// Copyright (c) 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command mirum is the developer-facing CLI for the Mirum CI system. +// +// In a future iteration it will host the full set of commands described in +// docs/whitepaper.md (`mirum task`, `mirum run`, `mirum list`, `mirum try`, +// `mirum ssh`, `mirum eval`). The current revision is a scaffold that +// compiles and ships through the existing build pipeline so that subsequent +// changes only have to add subcommand implementations. +package main + +import ( + "os" + + "dimidiumlabs/mirum/internal/protocol" + + "github.com/spf13/cobra" +) + +func main() { + root := &cobra.Command{ + Use: "mirum", + Short: "Mirum CI client", + Long: "Mirum CI client. Reads Mirumfile from the repository root.", + Version: protocol.VersionString(), + } + root.SetVersionTemplate("mirum {{.Version}}\n") + + if err := root.Execute(); err != nil { + os.Exit(1) + } +} diff --git a/docs/mirumfile.md b/docs/mirumfile.md new file mode 100644 --- /dev/null +++ b/docs/mirumfile.md @@ -0,0 +1,590 @@ +# Mirumfile + +> [!CAUTION] +> This document is the API reference for the `Mirumfile` format. The +> implementation is in progress and the runtime does not yet match this +> document. + +`Mirumfile` is a single Starlark file at the repository root. It defines +**tasks** (units of work, run on a host or in a VM) and **pipelines** +(graphs of tasks, scheduled across one or more VMs). The same file is +read by the local `mirum` CLI and by mirum-server in a cluster. + +## CLI + +``` +mirum task <name> [args...] Run a registered task. Positional args after + <name> become positional arguments to the + task function after tctx: + mirum task build linux amd64 + → build(tctx, "linux", "amd64") + +mirum run <pipeline> Run a registered pipeline. +``` + +## File format and discovery + +`Mirumfile` lives at the repository root. This is the only required file +for Mirum. + +The file is standard Starlark with the following predeclared globals: + +| Name | Kind | Purpose | +|-------------|---------------------|--------------------------------------| +| `task` | builtin function | Register a task | +| `pipeline` | builtin function | Register a pipeline | +| `fail` | standard Starlark | Abort with a message | +| `print` | standard Starlark | Write to the task log | +| `struct` | standard Starlark | Build anonymous records | + +Multi-file projects use standard Starlark `load()`: + +```python +load("//tasks/build.star", "build", "test") +load("//tasks/release.star", "package") +``` + +The mirum standard library is mounted under `@mirum//`: + +```python +load("@mirum//on.star", "git", "cron", "any_of") +load("@mirum//pkg.star", "install") +``` + +## Naming convention + +By convention, the first parameter of a function is named according to the +ctx kind it expects: + +- `tctx` for task functions and helpers that operate on a task ctx +- `pctx` for pipeline functions and helpers that operate on a pipeline ctx +- `event` for trigger predicates + +This convention is not enforced by the runner — it is just a readability +aid. A reader of `def lint(tctx):` immediately knows it expects a task +ctx. + +## Registration + +Tasks and pipelines are registered as **side effects** of top-level +calls, not as values bound to names. The function is defined first, then +registered: + +```python +def build(tctx): + """Build mirum-server""" + tctx.exec(["go", "build", "-o", "build/mirum-server", "./cmd/mirum-server"]) +task(build) + +def test(tctx): + """Run tests""" + tctx.need(build) + tctx.exec(["go", "test", "-race", "-count=1", "./..."]) +task(test) + +load("@mirum//on.star", "git") + +def ci(pctx): + """CI: build + test on Linux""" + pctx.run(test, image="mirum/ubuntu-24.04") +pipeline(ci, on=[git.push(branches=["main"])]) +``` + +The function being registered remains an ordinary callable. It can be +called directly (`build(tctx)`), passed as a value (`pctx.run(test, ...)`), +and referenced by `tctx.need`. Registration is metadata for the CLI and +the server — it does not wrap or replace the function. + +A function defined at the top level that is **not** registered is a +**helper**: invisible to `mirum list`, not invocable as `mirum task`, +no special calling convention. Helpers are just regular functions called +from registered ones. There is no `_` prefix convention. + +```python +def check_gofmt(tctx): # helper, not registered + r = tctx.exec(["gofmt", "-l", "."], check=False) + if r.stdout.strip(): + fail("gofmt: needs formatting:\n" + r.stdout) + +def lint(tctx): + """Run static checks""" + tctx.exec(["go", "vet", "./..."]) + check_gofmt(tctx) +task(lint) +``` + +### `task(fn, name=None)` + +Registers `fn` as a task. The default name is the Starlark function name; +the optional `name` keyword overrides it. Returns `None`. Re-registration +of the same name is a hard error at eval time. + +### `pipeline(fn, name=None, on=None)` + +Registers `fn` as a pipeline. Same naming rules as `task`. The `on` +parameter is a single predicate or a list of predicates that decide when +mirum-server triggers the pipeline (see [Triggers](#triggers)). Returns +`None`. + +## Triggers + +A trigger is a **predicate function**: it takes an event and returns a +bool. mirum-server runs each registered pipeline whose `on=` predicate +returns True for an incoming event. With a list, the pipeline runs if +**any** predicate matches (OR semantics). + +Predicates are ordinary Starlark functions, defined either by the user or +by the standard library. There is no separate trigger DSL. + +### Writing predicates by hand + +```python +def main_push(event): + return (event.kind == "git" + and event.type == "push" + and event.branch == "main") + +def go_changes(event): + if event.kind != "git": + return False + if event.type != "push" and event.type != "pull_request": + return False + return any([p.endswith(".go") or p == "go.mod" for p in event.paths]) + +def by_release_bot(event): + return (event.kind == "git" + and event.type == "push" + and event.author == "release-bot") + +def ci(pctx): + ... +pipeline(ci, on=[main_push, go_changes]) +``` + +### Standard library factories + +For common cases the `@mirum//on.star` stdlib provides predicate +factories — Starlark functions that return predicates. They are +themselves written in plain Starlark; users can fork the file or write +their own factories the same way. Factories for source-specific events +are grouped by source kind, exposed as structs: + +```python +load("@mirum//on.star", "git", "cron", "manual", "any_of", "all_of") + +def ci(pctx): + ... +pipeline(ci, on=[ + git.push(branches=["main"], paths=["**.go", "go.mod"]), + git.pull_request(branches=["main"]), +]) + +def release(pctx): + pctx.run(build_release, image="mirum/ubuntu-24.04") +pipeline(release, on=[git.tag(names=["v*"])]) + +def nightly(pctx): + ... +pipeline(nightly, on=[cron("0 6 * * *")]) +``` + +A factory is just a closure-returning function — there is nothing +special about it on the runtime side. A trimmed version of the stdlib's +`git.push`: + +```python +# @mirum//on.star +def _git_push(branches=None, paths=None): + def predicate(event): + if event.kind != "git" or event.type != "push": + return False + if branches and event.branch not in branches: + return False + if paths and not _any_glob(event.paths, paths): + return False + return True + return predicate + +git = struct( + push = _git_push, + tag = _git_tag, + pull_request = _git_pull_request, +) +``` + +When a non-git source is added — Perforce, Mercurial, Subversion — its +factories live in their own namespace alongside `git`: + +```python +load("@mirum//on.star", "git", "perforce") + +pipeline(ci, on=[ + git.push(branches=["main"]), + perforce.submit(branches=["//depot/main/..."]), +]) +``` + +User-written factories compose with stdlib ones identically. To require +that a build runs only when both `main` is pushed AND a release-bot is +the author, mix stdlib and hand-written predicates with `all_of`: + +```python +load("@mirum//on.star", "git", "all_of") + +def by_release_bot(event): + return (event.kind == "git" + and event.type == "push" + and event.author == "release-bot") + +def bot_release(pctx): + ... +pipeline(bot_release, on=[ + all_of(git.push(branches=["main"]), by_release_bot), +]) +``` + +### `EventCtx` + +The single argument passed to every predicate. Same value is also +available inside pipeline functions as `pctx.event`. EventCtx is +read-only. + +Only two fields are guaranteed on every event: + +| Field | Type | Meaning | +|-------------------|------|----------------------------------------| +| `event.kind` | str | Source family that produced the event: `"git"`, `"perforce"`, `"hg"`, `"cron"`, `"manual"`, `"s3"`, `"webhook"`, … | +| `event.timestamp` | int | Unix epoch seconds of when the event occurred | + +**Everything else depends on `event.kind`.** Each kind documents its +own fields. Source families that have multiple distinct event types +(e.g. `git` has push, tag, pull_request) also expose `event.type` as a +sub-discriminator. + +A predicate that wants to read kind-specific fields **must check +`event.kind` first** (and `event.type` if the kind has multiple types). +Accessing a field that does not exist for the current event raises an +error. Predicates that do not recognize a kind should return `False`, +not crash. New event kinds and types may be added at any time; existing +predicates that check for known kinds remain valid. + +**`kind == "manual"`** + +| Field | Type | Meaning | +|------------------|-------------|----------------------------------------| +| `event.user` | str | Identifier of the user who triggered | +| `event.reason` | str \| None | Optional reason supplied by the user | + +**`kind == "cron"`** + +| Field | Type | Meaning | +|------------------|------|---------------------------------------------| +| `event.schedule` | str | The cron expression that fired | + +**`kind == "git"`** + +Fields common to all git events: + +| Field | Type | Meaning | +|------------------|------|--------------------------------------------------| +| `event.type` | str | `"push"`, `"tag"`, or `"pull_request"` | +| `event.source` | str | Name of the source set | +| `event.commit` | str | Commit SHA | +| `event.author` | str | Author / committer / tagger / PR author | + +Additional fields by `event.type`: + +`type == "push"`: + +| Field | Type | Meaning | +|------------------|-----------|----------------------------------------| +| `event.branch` | str | Branch pushed to | +| `event.message` | str | Commit message | +| `event.paths` | list[str] | Paths changed by the push | + +`type == "tag"`: + +| Field | Type | Meaning | +|------------------|------|--------------------------------------------------| +| `event.tag` | str | Tag name | +| `event.message` | str | Tag annotation, if any | + +`type == "pull_request"`: + +| Field | Type | Meaning | +|------------------|-----------|----------------------------------------| +| `event.number` | int | PR number | +| `event.title` | str | PR title | +| `event.base` | str | Target branch | +| `event.head` | str | Source branch | +| `event.draft` | bool | Draft state | +| `event.labels` | list[str] | Labels currently applied | +| `event.paths` | list[str] | Paths changed in the PR | + +Additional event kinds (other VCS systems, S3 object events, generic +webhooks, file watchers, external systems) bring their own field sets +and are documented as they are added. Existing predicates are +unaffected: they match the kinds they know and return `False` for +everything else. + +## TaskCtx + +The argument passed to every task function. The only ctx that can +execute commands. + +### Execution + +``` +tctx.shell(script, **opts) -> Result +tctx.exec(argv, **opts) -> Result +``` + +`shell` runs `script` through a POSIX/bash interpreter with full support +for pipes, redirects, command substitution, here-docs, variable expansion, +and control flow. Same shell semantics on every platform; no `/bin/bash` +dependency. + +`exec` runs a single command without shell parsing. Use it when you have +an argv list and want zero shell interpretation. + +Common keyword options for both: + +| Option | Default | Meaning | +|------------|-------------|-----------------------------------------------------------| +| `env` | `{}` | Additional environment variables (added to `tctx.env`) | +| `cwd` | `None` | Working directory (relative to current `tctx.cwd`) | +| `stdin` | `None` | None / str / bytes / path | +| `stdout` | `None` | None (capture+stream) / path / `DEVNULL` | +| `stderr` | `None` | None (capture+stream) / path / `DEVNULL` / `STDOUT` | +| `append` | `False` | Open `stdout` / `stderr` paths in append mode | +| `timeout` | `None` | Seconds; kill on expiry | +| `check` | `True` | Non-zero exit raises `fail()` automatically | +| `capture` | `True` | Populate `Result.stdout` / `Result.stderr` | + +`Result` is a struct value with attributes: + +```python +result.stdout # str +result.stderr # str +result.code # int +result.ok # bool (code == 0) +``` + +`check=True` is the default — non-zero exits abort the task. Tasks that +want to inspect the exit code use `check=False`: + +```python +r = tctx.exec(["test", "-f", path], check=False) +if r.ok: + ... +``` + +### Immutable derive + +``` +tctx.with_env({"K": "V"}) -> tctx +tctx.with_cwd("subdir") -> tctx +``` + +Both return a new ctx with the modification applied. The original ctx is +unchanged. + +### Introspection + +``` +tctx.cwd # str — current working directory (absolute) +tctx.env # dict-like — current environment +tctx.os # "linux" | "darwin" | "freebsd" | "windows" | ... +tctx.arch # "amd64" | "arm64" | "riscv64" | ... +``` + +### `tctx.need(fn, *args, **kwargs) -> Result | None` + +Runs `fn(tctx, *args, **kwargs)` if it has not already been called with +the same arguments in this invocation; otherwise returns the cached +result. Use it for "make sure this happened" semantics; use a direct +call (`build(tctx)`) for "definitely run this now" semantics. + +Dedup key is `(fn, args, kwargs)`. Different arguments to the same +function are different invocations: + +```python +def build(tctx, goos="linux", goarch="amd64"): + ... + +def all_platforms(tctx): + for goos, goarch in [("linux", "amd64"), ("darwin", "arm64")]: + tctx.need(build, goos, goarch) # two distinct invocations +``` + +### `tctx.checkout(name="main", ref=None) -> str` + +Materializes a source set inside the task's environment and returns +the absolute path to it. The runtime is responsible for fetching the +right files; from the task's perspective the call is idempotent — +calling it again returns the same path without doing extra work. + +When `ref` is omitted, the runtime picks the natural ref for the +triggering event: + +| Event | Default ref | +|-----------------------------|---------------------------| +| `git push` | The pushed commit | +| `git tag` | The tagged commit | +| `git pull_request` | The PR head | +| `cron` | Default branch | +| `manual` / `mirum task` | Current working tree | + +To use a different ref explicitly, pass `ref=` (a SHA, branch, or tag +name). Pipelines that need to override the default usually do so by +forwarding event data through `args`: + +```python +def build_at(tctx, ref): + src = tctx.checkout(ref=ref) + ... +task(build_at) + +def replay(pctx): + pctx.run(build_at, image="...", args={"ref": pctx.event.commit}) +``` + +Multiple source sets are accessed by name: + +```python +def integration(tctx): + src = tctx.checkout() # default source set + fixtures = tctx.checkout("fixtures") # named source set + tctx.with_env({"FIXTURES": fixtures}).exec(["go", "test", "./tests/integration/..."]) +``` + +Source set names are defined in [Source sets](#source-sets), not in +pipeline or task code. A task that needs source files should call +`tctx.checkout()` explicitly — both as documentation of intent and +because some workers may stage source lazily on first call. + +### `tctx.upload(path, artifact="name")` and `tctx.download(name, dest=".")` + +`upload` registers a file as a named artifact under the current task. +`download` retrieves a previously uploaded artifact by name into a +destination directory. + +Locally, artifacts are tracked in `.mirum/local-artifacts.json` in the +repo root and persist between `mirum` invocations. In the cluster they +move through mirum-server. The task functions are unchanged either way. + +`download` of an artifact that was never uploaded fails with a clear +error. + +## PipelineCtx + +The argument passed to every pipeline function. **Cannot execute shell +commands.** Pipeline code may come from untrusted PRs; it is restricted +to orchestration. + +### `pctx.event` + +Read-only [`EventCtx`](#eventctx) for the event that triggered this +pipeline. The same value that was passed to the trigger predicates. + +When `mirum run` is invoked locally, the event is synthetic +(`{"kind": "manual"}` by default; overridable via CLI flags). + +### `pctx.run(task_fn, image=, args={}, setup=None, depends=[]) -> handle` + +Dispatches a task into a new VM. Returns a handle that can be passed as +`depends=[handle, ...]` to subsequent `pctx.run` calls. + +| Argument | Meaning | +|------------|--------------------------------------------------------------------------| +| `task_fn` | Task function to invoke (the function value, not the registered name) | +| `image` | OCI reference to the VM image | +| `args` | Keyword arguments forwarded to `task_fn(tctx, **args)` | +| `setup` | Optional setup function for cached snapshots | +| `depends` | Handles from prior `pctx.run` calls. This VM does not start until they finish, and inherits their `upload`'d artifacts | + +Source materialization is a task-side concern, not a pipeline-side one +— see [`tctx.checkout`](#tctxcheckoutnamemain---str). The pipeline does +not pass source refs to its tasks; each task asks for the source sets +it needs by name, and the runner resolves them based on the triggering +event. + +`depends=` expresses **VM topology**, not task dependency. Within a +single VM, task functions compose via `tctx.need(...)`. Across VMs, the +pipeline orchestrates via `pctx.run(..., depends=[...])`. These are two +distinct mechanisms and are not interchangeable. + +Example: cross-platform release that fans out builds and fans in publish. + +```python +load("@mirum//on.star", "git") + +PLATFORMS = [ + ("linux", "amd64"), ("linux", "arm64"), + ("darwin", "arm64"), + ("windows", "amd64"), + ("freebsd", "amd64"), +] + +IMAGES = { + "linux": "mirum/ubuntu-24.04", + "darwin": "mirum/macos-15", + "windows": "mirum/windows-2025", + "freebsd": "mirum/freebsd-14", +} + +def release(pctx): + builds = [] + for goos, goarch in PLATFORMS: + builds.append(pctx.run(build, + image=IMAGES[goos], + args={"goos": goos, "goarch": goarch})) + + pctx.run(publish, image=IMAGES["linux"], depends=builds) +pipeline(release, on=[git.tag(names=["v*"])]) +``` + +## Source sets + +Source URLs, refs, auth, and repo locations are **configuration**. +Pipeline and task code reference source sets only by **name**. + +In a cluster, source sets are configured per project in mirum-server's +WebUI. The server resolves names to URLs and credentials when staging a +VM, and credentials never reach the VM or the task code. + +Locally, source sets are defined in `.mirum/sources.json` in the repo +root: + +```json +{ + "main": ".", + "fixtures": "/home/user/work/test-fixtures" +} +``` + +Values are absolute paths to local checkouts. The `main` entry can be +omitted; it defaults to the directory containing `Mirumfile`. The file +is opt-in for git tracking — projects with shared layout may commit it, +projects with user-specific paths typically gitignore it. + +A `tctx.checkout("name")` for a name not in the configuration is a hard +error. + +## Safe shell interpolation + +Dynamic values are passed through the `env=` keyword and referenced as +quoted shell variables in the script body. This gives POSIX single-word +expansion semantics — the value becomes one literal argument and is +never re-parsed: + +```python +# Even if branch == "; rm -rf /", this is safe — it is one literal arg +tctx.shell('git log --format=%H "$BRANCH"', env={"BRANCH": branch}) +``` + +Never build shell scripts by string concatenation. Always pass dynamic +values through `env=`. + +`tctx.exec(argv, ...)` bypasses the shell entirely; use it when you +already have an argv list and want zero chance of shell interpretation. diff --git a/docs/whitepaper.md b/docs/whitepaper.md new file mode 100644 --- /dev/null +++ b/docs/whitepaper.md @@ -0,0 +1,594 @@ +# Mirum Whitepaper + +## Modules + +Four executable modules: + +``` +┌──────────────────────────────────────────────────────┐ +│ mirum-server │ +│ watch registry, task queue, log aggregation, WebUI │ +└─────────────────────────────────────────────────────┘ + ↑ gRPC (worker-initiated) ↑ gRPC + │ │ +┌────────┴──────────┐ ┌─────────┴─────────┐ +│ mirum-worker │ │ mirum-worker │ +│ Linux (KVM) │ │ macOS (Vz) │ +│ │ vsock │ │ │ vsock │ +│ ↓ │ │ ↓ │ +│ ┌────────────┐ │ │ ┌────────────┐ │ +│ │mirum-agent │ │ │ │mirum-agent │ │ +│ │ inside VM │ │ │ │ inside VM │ │ +│ └────────────┘ │ │ └────────────┘ │ +└───────────────────┘ └───────────────────┘ + +┌───────────────────┐ +│ mirum (CLI) │ +│ Starlark eval, │ +│ spawns worker │ +└───────────────────┘ +``` + +**mirum-server** — the orchestrator. +Contains a database, stores users, organizations, project and pipeline settings, +provides a WebUI and API, responds to webhooks, and distributes tasks to workers. +Collects logs and build results from workers. + +It's a control plane. + +**mirum-worker** — the task executor +Connects to the server via outbound gRPC, declares its capabilities, +and picks tasks from the queue that it can execute. + +Evaluates pipeline Starlark (coroutine model), starts a VM for each +task, runs the task function inside, and returns the results to the +server. If a pipeline yields on a task result it is not yet ready for, +the worker leaves the suspended coroutine in the queue and resumes it +when the awaited task completes. + +It's a data plane. _Only the worker has access to the user's secrets and code._ + +Workers come in different kinds: KVM worker for local VMs, +macOS worker with Vz.framework, windows worker with Hyper-V, +EC2 worker for cloud VMs, host worker for direct execution. +A new worker type joins the cluster and starts picking up tasks with no changes +to the server. + +**mirum-agent** — static binary pre-installed in every VM. + +A tiny bridge between the worker on the host and the tasks running in the VM. +Written in C99, no dependencies, posix-only. Implements a simple TLV for communicating +with the host (via virtio-vsock or tcp socket, depending on the hypervisor). + +Channels: control, stdin, stdout, stderr, file transfers, interactive shell sessions. + +Small and simple enough to be auditable. +This component lives close to the actual code, making it highly security-sensitive. + +**mirum (CLI)** — developer tool. +Contains a Starlark runtime for local eval and embeds a host worker for +in-process execution. Takes the server's role locally. + +Technically, it is a lightweight disposable server that runs a real worker locally +to replicate real-world conditions in a cluster as closely as possible. + +- `mirum task <name>`: run a single registered task on the host, no VM. +- `mirum run <pipeline>`: eval pipeline → spawn worker → dispatch tasks → display logs. +- `mirum list`: list registered tasks and pipelines. +- `mirum try <pipeline>`: send a local diff to the server. +- `mirum ssh <vm>`: shell into a failed VM. +- `mirum eval <pipeline>`: show the DAG without running anything. + +## Configuration Model + +Mirum operates on projects, a project consists of pipelines, +pipelines consist of tasks. A task is the minimum unit of execution — it always +runs in a single VM. Pipelines are DAGs (directed acyclic graphs) that invoke multiple +tasks and pass state between them. A project tracks a list of pipelines, their trigger +rules (watch, cron, manual, ...), and result notifications. + +A quick example, all in one starlark file: + +```python +# /Mirumfile — the only file required + +load("@mirum//on.star", "git") + +# A single pipeline describes a matrix of multiple operating systems and architectures. +# A single Linux host with KVM serves Linux, Windows, and BSD guests. +# A macOS host serves macOS, Linux, and BSD. +# Tasks adapt to the platform via `tctx.os` — they don't choose it. + +# pctx.run takes an optional setup function to indicate which steps +# are environment setup only, so the resulting image can be cached +def setup(tctx): + tctx.shell("apt-get update && apt-get install -y cargo") + +def build(tctx): + tctx.checkout() + tctx.shell("cargo build --release") + tctx.upload("target/release/myapp", artifact="bin") +task(build) + +def test(tctx): + # test knows nothing about build — only that it needs an artifact. + # that artifact could have been built right here, come from cache + # or a registry, or even uploaded from a developer's laptop + tctx.download("bin", dest=".") + tctx.shell("cargo test") +task(test) + +# A pipeline is a function that dispatches tasks. It is registered the +# same way tasks are, with a list of triggers. +def ci(pctx): + b = pctx.run(build, setup=setup, image="mirum/ubuntu-24.04") + pctx.run(test, setup=setup, depends=b, image="mirum/ubuntu-24.04") +pipeline(ci, on=[git.push(branches=["main"])]) +``` + +### Functions All the Way Down + +Mirum configuration is nested function composition in Starlark. There +are two kinds of registered things — **tasks** and **pipelines** — and +both are registered as side effects of top-level calls in the file: + +```python +def build(tctx): + ... +task(build) + +def ci(pctx): + ... +pipeline(ci, on=[git.push(branches=["main"])]) +``` + +A pipeline is a function that imperatively dispatches tasks and passes +dependencies between them. A task's result can be used to launch further +tasks, enabling dynamic task creation. There is no separate syntax for +"allow failure / retry / skip_if / matrix" — it is just `if`/`for` in +Starlark. + +``` +Mirumfile → the file the server looks for (one per repo, at root) + +triggers → event routing "when to run" + (predicates passed via pipeline(..., on=[...])) + +pipeline(pctx) → DAG of tasks "what to run, on which platforms" + +task(tctx) → scripts + artifacts "how to build" +``` + +Starlark supports imports, so splitting a large project works out of the +box. The only requirement is a `Mirumfile` at the repository root. + +The server reads `Mirumfile` (and any files it transitively `load()`s) +via the forge contents API, not via git clone. This allows: fetching +only configuration files without accessing source code, filtering +webhooks (push with no changes in `Mirumfile`'s closure → skip re-eval), +caching configuration by file SHAs. + +To reuse code both within and outside the project, Starlark `load` is used. + +``` +@mirum// Standard library (triggers, services, apt, bazel helpers) +@pkg// External packages (from deps.star, pinned by commit) +// Local files (relative to repo root) +``` + +### Coroutine Eval: Dynamic DAGs + +Pipeline functions execute as coroutines. `pctx.run()` without accessing +results is non-blocking — the server accumulates pending tasks. +Accessing a result (`.output()`) is a yield point: the server dispatches +all pending tasks, waits for the needed result, and resumes the pipeline +function. + +```python +def build(tctx): + ... +task(build) + +def discover_tests(tctx): + ... +task(discover_tests) + +def run_test(tctx, module): + ... +task(run_test) + +def ci(pctx): + # All pctx.run() calls before the first .output() accumulate and run in parallel + builds = {} + for os in ["linux", "mac"]: + builds[os] = pctx.run(build, + image="mirum/%s" % os, args={"os": os}) + + discovery = pctx.run(discover_tests, image="mirum/linux") + + # Yield: server dispatches builds + discovery in parallel, + # waits for discovery to complete, resumes with result + test_modules = discovery.output("modules") + + # Dynamic phase: use runtime data + for module in test_modules: + pctx.run(run_test, args={"module": module}, + depends=list(builds.values())) +pipeline(ci, on=[git.push(branches=["main"])]) +``` + +If a pipeline function never calls `.output()`, the entire DAG is built +in a single pass. A static DAG is a special case of the dynamic model. + +`mirum eval` executes the pipeline function locally without dispatching +tasks. For static DAGs it prints the full graph. For dynamic DAGs — +everything up to the first yield point, marked "depends on runtime data +beyond this point." + +### Example: Everything in One File + +```python +# /Mirumfile — complete CI + +load("@mirum//on.star", "git") + +# pctx.run takes an optional setup function to indicate which steps +# are environment setup only, so the resulting image can be cached +def setup(tctx): + tctx.shell("apt-get update && apt-get install -y cargo") + +def build(tctx): + tctx.checkout() + tctx.shell("cargo build --release") + tctx.upload("target/release/myapp", artifact="bin") +task(build) + +def test(tctx): + # test knows nothing about build — only that it needs an artifact. + # that artifact could have been built right here, come from cache + # or a registry, or even uploaded from a developer's laptop + tctx.download("bin", dest=".") + tctx.shell("cargo test") +task(test) + +def ci(pctx): + b = pctx.run(build, setup=setup, image="mirum/ubuntu-24.04") + pctx.run(test, setup=setup, depends=b, image="mirum/ubuntu-24.04") +pipeline(ci, on=[git.push(branches=["main"])]) +``` + +### Example: Cross-Platform Project + +```python +# tasks/setup.star +def cpp_toolchain(tctx): + if tctx.os == "linux": + tctx.shell("apt-get update && apt-get install -y cmake ninja-build") + elif tctx.os == "freebsd": + tctx.shell("pkg install -y cmake ninja") + elif tctx.os == "windows": + tctx.shell("choco install -y cmake ninja visualstudio2022-workload-vctools") + elif tctx.os == "macos": + tctx.shell("brew install cmake ninja") + +# tasks/build.star +def build(tctx): + tctx.checkout() + if tctx.os == "windows": + tctx.shell('cmake -G "Visual Studio 17 2022" -B build .') + elif tctx.os == "macos": + tctx.shell("cmake -B build -DCMAKE_OSX_DEPLOYMENT_TARGET=12.0 .") + else: + tctx.shell("cmake -B build .") + tctx.shell("cmake --build build --config Release") + tctx.upload("build/out/*", artifact="pkg") +task(build) + +def publish(tctx): + tctx.download("pkg", dest="release/") + tctx.shell('gh release create "$TAG" release/* --generate-notes') +task(publish) + +# /Mirumfile +load("@mirum//on.star", "git") +load("//tasks/setup.star", "cpp_toolchain") +load("//tasks/build.star", "build", "publish") + +IMAGES = { + "linux": "mirum/ubuntu-24.04", + "freebsd": "mirum/freebsd-14", + "windows": "mirum/windows-2025", + "macos": "mirum/macos-15", +} + +# Irregular matrix — just a list. No exclude needed. +PLATFORMS = [ + ("linux", "amd64"), + ("linux", "arm64"), + ("macos", "arm64"), + ("windows", "amd64"), + ("freebsd", "amd64"), +] + +def release(pctx): + # Build: loop over platforms, each task gets its own handle + builds = {} + for os, arch in PLATFORMS: + builds[(os, arch)] = pctx.run(build, + setup=cpp_toolchain, + image=IMAGES[os], + args={"os": os, "arch": arch}) + + # Publish: fan-in, waits for all builds + pctx.run(publish, depends=list(builds.values())) +pipeline(release, on=[git.tag(names=["v*"])]) +``` + +The pipeline decides WHERE (image, platforms). Setup decides WITH WHAT +(toolchain, cached snapshot). The task decides HOW (checkout, build, upload). +Tasks don't know what platform they're running on — `tctx.os` and `tctx.arch` +are injected by the pipeline. + +## Images + +The most tedious and time-consuming task is preparing images for various operating +systems. Some distributions distribute qcow2, some support cloud-init, and some only +offer an ISO installer. Some require a network connection for configuration, while +others work offline. + +Another problem is distribution. The reason containers are popular is the OCI registry. +A container is easy to upload to a server, and just as easy to download and deploy. +Nothing similar exists for VMs. + +An elegant solution was found in Tart by CirrusCI: use OCI as a black box for storing +the VM image. Load it into your existing infrastructure, easily update, and distribute. +A single distribution format for all platforms. Images are stored as compressed +raw disk chunks: + +``` +OCI Image Manifest: + config: + mediaType: "application/vnd.mirum.image.config.v1+json" + { mirum.version, os, arch, distro, distro_version, + agent_version, disk_size, chunk_size } + + layers: + - mediaType: "application/vnd.mirum.disk.raw.v1+zstd" + annotations: { "mirum.offset": "0", "mirum.length": "67108864" } + - ... + + # macOS additionally: + - mediaType: "application/vnd.mirum.aux.v1+zstd" + - mediaType: "application/vnd.mirum.hwmodel.v1+json" +``` + +Each chunk is independently zstd-compressed. The worker downloads and decompresses +in parallel. 64MB chunks for a 4GB disk ≈ 64 layers. On pull worker reassembles +raw disk from chunks → converts to hypervisor format +(qcow2, vhdx, Vz native) → caches → CoW clone per task. + +### Three Layers (VM Runtime) + +VM images are larger than container images, but there are fewer of them. +We can borrow the layer caching idea and apply it to image snapshots (like in qcow2+). + +``` +Layer 0: Base image (from OCI registry) + Golden (Mirum-maintained) or organization (external). + Worker downloads, converts to hypervisor format, caches locally. + +Layer 1: Setup (function from pctx.run(setup=...)) + Declared in the pipeline. Worker executes, takes a snapshot. + Cache is local, best-effort, evicted by LRU. + +Layer 2: Ephemeral overlay + CoW clone of setup cache (or base). Per-task. Destroyed. +``` + +### Setup as a Function + +The pipeline passes two functions to `pctx.run()`: +`setup` (optional) and the main task. The image is also specified in the pipeline: + +```python +def cpp_setup(tctx): + if tctx.os == "linux": + tctx.shell("apt-get update && apt-get install -y cmake ninja-build") + elif tctx.os == "freebsd": + tctx.shell("pkg install -y cmake ninja") + +def build(tctx): + tctx.checkout() + tctx.shell("cmake -B build . && cmake --build build") +task(build) + +def ci(pctx): + pctx.run(build, setup=cpp_setup, + image="mirum/ubuntu-24.04", + args={"os": "linux"}) +pipeline(ci, on=[git.push(branches=["main"])]) +``` + +The worker hashes `(image_digest, setup_function_hash, os, arch)`. +Cache hit → CoW clone, boot in milliseconds. Miss → boot base, run setup, +snapshot, cache. + +Setup is an ordinary Starlark function, composable via `load()`: + +```python +# @pkg//acme/setup.star +def cpp_toolchain(tctx): + if tctx.os == "linux": + tctx.shell("apt-get update && apt-get install -y cmake ninja-build") + elif tctx.os == "windows": + tctx.shell("choco install -y cmake ninja") +``` + +```python +load("@pkg//acme/setup.star", "cpp_toolchain") + +def ci(pctx): + for os in ["linux", "windows"]: + pctx.run(build, setup=cpp_toolchain, + image=IMAGES[os], args={"os": os}) +``` + +One `cpp_toolchain` across the entire organization — one hash — one snapshot per worker. + +Three levels, cleanly separated: + +- **Pipeline**: WHERE (image, platforms) +- **Setup**: WITH WHAT (toolchain, dependencies — cached snapshot) +- **Task**: HOW (checkout, build, test, upload) + +Organizations that need a fully pre-built image can publish it to any OCI registry +using external tooling and reference it directly: + +```python +# no need for setup with a preconfigured image +pctx.run(build, image="acme-registry.com/ci-base:latest") +``` + +### Golden Images + +Golden images are built by the Mirum team using Packer, not by users: + +| Platform | Packer builder | Install method | +| ------------------------- | ---------------------- | ----------------------------------- | +| Linux (Ubuntu, Fedora...) | `qemu` | cloud-init / preseed / kickstart | +| macOS | `tart` (Packer plugin) | VZMacOSInstaller + VNC boot_command | +| Windows | `qemu` | autounattend.xml (evaluation ISO) | +| NetBSD | `qemu` | sysinst auto | +| FreeBSD | `qemu` | bsdinstall scripted | +| OpenBSD | `qemu` | autoinstall response file | + +Windows: evaluation ISO is freely downloadable. The evaluation period (180 days) +is irrelevant for ephemeral CI VMs. Users activate with their own key if needed. + +macOS: `.ipsw` installed via Virtualization.framework. Setup Assistant automated +via VNC keystroke injection. Requires Apple hardware for building and running. + +## Extensibility + +Starlark simplifies building plugins and libraries. You already have `load`, +so you don't need to invent your own systems like reusable actions. + +**Transparent worker optimizations** — invisible to the task. +Configured in `worker.yaml`. The worker configures the VM environment before running +any scripts: apt mirror, cargo/npm cache mount, HTTP proxy. +`./ci.sh` with `apt-get install` inside simply runs faster. +Bash scripts speed up for free. + +```yaml +# worker.yaml +optimizations: + apt_mirror: "http://apt-cache.internal:3142" + http_proxy: "http://squid.internal:3128" + cargo_cache: "/mnt/shared/cargo" +``` + +**Starlark stdlib (@mirum//)** — for things that require an explicit decision. +This is a standard Starlark that, although it comes with an agent, +doesn't require any additional APIs (see the Bazel vs Buck configurations). +Users can read, fork, or write their own. + +Starlark sees capabilities via `tctx.worker.has("docker")`. +The stdlib adapts. No hidden magic — the code is readable. + +The dividing principle: if an optimization can be applied without changing +task behavior — it's a transparent worker optimization. If the task needs to know +(e.g. a postgres address) — it's Starlark stdlib with graceful degradation. + +Checks worker capabilities and adapts: + +```python +load("@mirum//services", "service") + +def test(tctx): + # docker on the worker? → sidecar container + # no docker? → install and run inside the VM + service(tctx, "postgres", image="postgres:16", port=5432) + tctx.shell("make test") +task(test) +``` + +Workers declare capabilities on registration: + +```yaml +# worker.yaml +capabilities: + kvm: true + gpu: false + docker: true +``` + +**Server plugins (traits)** — extend the platform. Implement trait interfaces. +Configured in `server.yaml`: + +| Trait | AGPL built-in | External plugin | +| ---------------- | ------------- | ---------------------- | +| AuthBackend | Token, basic | SAML, OIDC, LDAP | +| Source provider | Git, VSC, s3 | Mercurial, Perforse | +| SecretProvider | Env, systemd | Vault, AWS KMS | +| NotificationSink | — | Slack, email, webhooks | +| BillingHook | Noop | Usage metering | + +## Comparison + +We were inspired by many wonderful tools +- Buildbot: centralized master, property model, `try` for pre-commit testing, dynamic build steps; +- TeamCity: vsc roots, multi-tenant, role model, breadth of tool support; +- Concourse: the idea of universal input/output; +- SourceHut: SSH into VMs for debugging, BSD support; +- Cirrus CI: ephemeral VMs, bring-your-own cloud, Starlark, agent inside VM; +- GitHub Actions: how not to do it. + +### vs GitHub Actions + +Paid, closed, inseparable from Microsoft, only ubuntu/windows/macOS, +only x64/arm64, nodejs required in runtime, +yaml configs (and only for the current repository). +No cross-OS matrices out of the box (macOS runners are a paid add-on). +No local execution. No dynamic DAG. Caching is an action, not a primitive. + +### vs GitLab CI + +GitLab CI is part of GitLab. YAML. DAG via `needs:` — a hack on top of a stage-based model. +Runners are stateful machines or Docker. No VM isolation. `include:` / YAML anchors — fragile reuse. +Dynamic child pipelines — via YAML generation. + +### vs Jenkins + +Groovy DSL is powerful but allows arbitrary code (RCE when eval'ing PRs). +Plugin ecosystem is huge but fragile (Security Advisories every month). +Agents are stateful, workspace persists. Shared Libraries are Groovy classes, trusted/untrusted. + +### vs TeamCity + +Powerful and popular, with clever ideas (dedicated VCS root, for example). +But it's paid and expensive, closed-source, and difficult to use. +Kotlin DSL is typed with IDE support, but allows side effects (HTTP, filesystem). +Agents are stateful and require maintenance. Snapshot dependencies equal our source consistency. + +Templates (1:1) vs our `load()` (N:N) — Starlark is strictly more powerful. + +### vs Buildbot + +The most portable and flexible of all. However, it's outdated, difficult to configure, +and designed for hosted builds. Its architecture doesn't support SaaS. +Pure Python configurations on both the master and agents. No IaC out of the box. + +### vs Cirrus CI + +Great tool, but unfortunately still closed source and dependent on gcloud. +Starlark is only available as an advanced mode with yaml. +It lacks support for many BSDs (but FreeBSD is available!). + +## Licensing + +**AGPL-3.0** — all four modules, all built-in traits, all runtimes, standard library, +full CLI, basic Web UI, SQLite, single-tenant auth. + +**Commercial license** — For companies unwilling to use the AGPL, offer a commercial +license, certifications, and SLAs in SaaS. Don't hesitate to take money +from enterprises and spend it on open source. diff --git a/internal/config/config.go b/internal/config/config.go index c9d0521..3d1256c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,7 +1,7 @@ // Copyright (c) 2026 Nikolay Govorov // SPDX-License-Identifier: AGPL-3.0-or-later -// Package config holds hard-coded tunables shared across mirumd and mirumw: +// Package config holds hard-coded tunables shared across mirum-server and mirum-worker: // timeouts, sizes, intervals, and limits that are not (yet) exposed through // the YAML user config. Grouping them here keeps magic numbers out of call // sites and gives a single place to audit defaults. diff --git a/internal/protocol/platform.go b/internal/protocol/platform.go index 638a664..d48da44 100644 --- a/internal/protocol/platform.go +++ b/internal/protocol/platform.go @@ -11,7 +11,7 @@ import ( "strings" mirum "dimidiumlabs/mirum" - "dimidiumlabs/mirum/internal/protocol/pb" + "dimidiumlabs/mirum/internal/protocol/wirepb" ) var ErrInvalidVersion = errors.New("invalid version string") @@ -22,53 +22,53 @@ var ( Patch uint32 ) -var osMap = map[string]pb.Os{ - "linux": pb.Os_OS_LINUX, - "darwin": pb.Os_OS_DARWIN, - "windows": pb.Os_OS_WINDOWS, - "freebsd": pb.Os_OS_FREEBSD, - "openbsd": pb.Os_OS_OPENBSD, - "netbsd": pb.Os_OS_NETBSD, - "dragonfly": pb.Os_OS_DRAGONFLY, - "illumos": pb.Os_OS_ILLUMOS, - "solaris": pb.Os_OS_SOLARIS, - "aix": pb.Os_OS_AIX, - "plan9": pb.Os_OS_PLAN9, - "android": pb.Os_OS_ANDROID, - "ios": pb.Os_OS_IOS, - "js": pb.Os_OS_JS, - "wasip1": pb.Os_OS_WASIP1, +var osMap = map[string]wirepb.Os{ + "linux": wirepb.Os_OS_LINUX, + "darwin": wirepb.Os_OS_DARWIN, + "windows": wirepb.Os_OS_WINDOWS, + "freebsd": wirepb.Os_OS_FREEBSD, + "openbsd": wirepb.Os_OS_OPENBSD, + "netbsd": wirepb.Os_OS_NETBSD, + "dragonfly": wirepb.Os_OS_DRAGONFLY, + "illumos": wirepb.Os_OS_ILLUMOS, + "solaris": wirepb.Os_OS_SOLARIS, + "aix": wirepb.Os_OS_AIX, + "plan9": wirepb.Os_OS_PLAN9, + "android": wirepb.Os_OS_ANDROID, + "ios": wirepb.Os_OS_IOS, + "js": wirepb.Os_OS_JS, + "wasip1": wirepb.Os_OS_WASIP1, } -var archMap = map[string]pb.Arch{ - "amd64": pb.Arch_ARCH_AMD64, - "arm64": pb.Arch_ARCH_ARM64, - "386": pb.Arch_ARCH_386, - "arm": pb.Arch_ARCH_ARM, - "riscv64": pb.Arch_ARCH_RISCV64, - "ppc64le": pb.Arch_ARCH_PPC64LE, - "ppc64": pb.Arch_ARCH_PPC64, - "s390x": pb.Arch_ARCH_S390X, - "mips64le": pb.Arch_ARCH_MIPS64LE, - "mips64": pb.Arch_ARCH_MIPS64, - "mipsle": pb.Arch_ARCH_MIPSLE, - "mips": pb.Arch_ARCH_MIPS, - "loong64": pb.Arch_ARCH_LOONG64, - "wasm": pb.Arch_ARCH_WASM, +var archMap = map[string]wirepb.Arch{ + "amd64": wirepb.Arch_ARCH_AMD64, + "arm64": wirepb.Arch_ARCH_ARM64, + "386": wirepb.Arch_ARCH_386, + "arm": wirepb.Arch_ARCH_ARM, + "riscv64": wirepb.Arch_ARCH_RISCV64, + "ppc64le": wirepb.Arch_ARCH_PPC64LE, + "ppc64": wirepb.Arch_ARCH_PPC64, + "s390x": wirepb.Arch_ARCH_S390X, + "mips64le": wirepb.Arch_ARCH_MIPS64LE, + "mips64": wirepb.Arch_ARCH_MIPS64, + "mipsle": wirepb.Arch_ARCH_MIPSLE, + "mips": wirepb.Arch_ARCH_MIPS, + "loong64": wirepb.Arch_ARCH_LOONG64, + "wasm": wirepb.Arch_ARCH_WASM, } -func DetectOs() pb.Os { +func DetectOs() wirepb.Os { if v, ok := osMap[runtime.GOOS]; ok { return v } - return pb.Os_OS_UNSPECIFIED + return wirepb.Os_OS_UNSPECIFIED } -func DetectArch() pb.Arch { +func DetectArch() wirepb.Arch { if v, ok := archMap[runtime.GOARCH]; ok { return v } - return pb.Arch_ARCH_UNSPECIFIED + return wirepb.Arch_ARCH_UNSPECIFIED } func init() { @@ -99,6 +99,6 @@ func VersionString() string { return fmt.Sprintf("%d.%d.%d", Major, Minor, Patch) } -func VersionProto() *pb.Version { - return &pb.Version{Major: Major, Minor: Minor, Patch: Patch} +func VersionProto() *wirepb.Version { + return &wirepb.Version{Major: Major, Minor: Minor, Patch: Patch} } diff --git a/internal/protocol/proto/buf.gen.yaml b/internal/protocol/proto/buf.gen.yaml new file mode 100644 --- /dev/null +++ b/internal/protocol/proto/buf.gen.yaml @@ -0,0 +1,11 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +version: v2 +plugins: + - local: protoc-gen-go + out: ../wirepb + opt: paths=source_relative + - local: protoc-gen-connect-go + out: ../wirepb + opt: paths=source_relative diff --git a/proto/mirum.proto b/internal/protocol/proto/wire.proto index dd85e26..73668e7 100644 --- a/proto/mirum.proto +++ b/internal/protocol/proto/wire.proto @@ -3,9 +3,9 @@ syntax = "proto3"; -package mirum; +package mirum.wire; -option go_package = "dimidiumlabs/mirum/internal/protocol/pb"; +option go_package = "dimidiumlabs/mirum/internal/protocol/wirepb"; // Describes the contract between the worker and the server. // GRPC is the only contract between them, so to implement your own worker, @@ -17,7 +17,7 @@ option go_package = "dimidiumlabs/mirum/internal/protocol/pb"; // // Worker metadata (name, version, os, arch) and clock skew detection // are embedded in the certificate (URI SAN and NotBefore). -service Mirum { +service Worker { // When a worker has free resources, it requests a task from the server. // The call will block if the server currently has no tasks. rpc Poll(PollRequest) returns (Task); diff --git a/nfpm.yaml b/nfpm.yaml index f19f0a3..0b070cb 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -18,55 +18,60 @@ contents: - src: ./README.md dst: /usr/share/doc/mirum/README.md - - src: build/mirumd - dst: /usr/local/bin/mirumd + - src: build/mirum-server + dst: /usr/local/bin/mirum-server file_info: mode: 0755 - - src: build/mirumw - dst: /usr/local/bin/mirumw + - src: build/mirum-worker + dst: /usr/local/bin/mirum-worker file_info: mode: 0755 - - src: pkg/mirumd.yaml - dst: /etc/mirum/mirumd.yaml + - src: build/mirum + dst: /usr/local/bin/mirum + file_info: + mode: 0755 + + - src: pkg/server/config.yaml + dst: /etc/mirum/server/config.yaml type: config|noreplace file_info: mode: 0640 owner: root - group: mirumd + group: mirum-server - - src: pkg/mirumw-default.yaml - dst: /etc/mirum/mirumw-default.yaml + - src: pkg/worker/default.yaml + dst: /etc/mirum/worker/default.yaml type: config|noreplace file_info: mode: 0640 owner: root - group: mirumw + group: mirum-worker - - src: pkg/mirumd.service - dst: /usr/lib/systemd/system/mirumd.service + - src: pkg/mirum-server.service + dst: /usr/lib/systemd/system/mirum-server.service file_info: mode: 0644 - - src: pkg/mirumw@.service - dst: /usr/lib/systemd/system/mirumw@.service + - src: pkg/mirum-worker@.service + dst: /usr/lib/systemd/system/mirum-worker@.service file_info: mode: 0644 - - dst: /var/lib/mirumd + - dst: /var/lib/mirum-server type: dir file_info: mode: 0750 - owner: mirumd - group: mirumd + owner: mirum-server + group: mirum-server - - dst: /var/lib/mirumw + - dst: /var/lib/mirum-worker type: dir file_info: mode: 0750 - owner: mirumw - group: mirumw + owner: mirum-worker + group: mirum-worker scripts: diff --git a/pkg/mirumd.service b/pkg/mirum-server.service index ab46526..69e440f 100644 --- a/pkg/mirumd.service +++ b/pkg/mirum-server.service @@ -8,7 +8,7 @@ After=time-sync.target network-online.target remote-fs.target nss-lookup.target Wants=time-sync.target # Socket activation (optional): -# Create mirumd.socket with named file descriptors "web" and "grpc": +# Create mirum-server.socket with named file descriptors "web" and "grpc": # # [Socket] # ListenStream=0.0.0.0:3000 @@ -22,14 +22,14 @@ Wants=time-sync.target [Service] Type=notify -User=mirumd -Group=mirumd +User=mirum-server +Group=mirum-server Restart=always RestartSec=30 WatchdogSec=30 NotifyAccess=main -ExecPaths=/usr/local/bin/mirumd /usr/lib -ExecStart=/usr/local/bin/mirumd daemon --config=/etc/mirum/mirumd.yaml +ExecPaths=/usr/local/bin/mirum-server /usr/lib +ExecStart=/usr/local/bin/mirum-server daemon --config=/etc/mirum/server/config.yaml LimitCORE=infinity LimitNOFILE=500000 AmbientCapabilities=CAP_NET_BIND_SERVICE diff --git a/pkg/mirumw@.service b/pkg/mirum-worker@.service index cf7ddd5..dbbc5e0 100644 --- a/pkg/mirumw@.service +++ b/pkg/mirum-worker@.service @@ -9,14 +9,14 @@ Wants=time-sync.target [Service] Type=notify -User=mirumw -Group=mirumw +User=mirum-worker +Group=mirum-worker Restart=always RestartSec=30 WatchdogSec=30 NotifyAccess=main -ExecPaths=/usr/local/bin/mirumw -ExecStart=/usr/local/bin/mirumw --config=/etc/mirum/mirumw-%i.yaml +ExecPaths=/usr/local/bin/mirum-worker +ExecStart=/usr/local/bin/mirum-worker --config=/etc/mirum/worker/%i.yaml LimitCORE=infinity LimitNOFILE=500000 AmbientCapabilities= diff --git a/pkg/scripts/postinstall.sh b/pkg/scripts/postinstall.sh index 2c80442..3479c75 100644 --- a/pkg/scripts/postinstall.sh +++ b/pkg/scripts/postinstall.sh @@ -4,10 +4,10 @@ set -e -if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ] && [ -f /usr/lib/systemd/system/mirumd.service ]; then +if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ] && [ -f /usr/lib/systemd/system/mirum-server.service ]; then /bin/systemctl daemon-reload # Don't enable by default, don't know in advance whether it's a daemon or a worker - # /bin/systemctl enable mirumd - # /bin/systemctl enable mirumw + # /bin/systemctl enable mirum-server + # /bin/systemctl enable mirum-worker fi diff --git a/pkg/scripts/preinstall.sh b/pkg/scripts/preinstall.sh index d15fee2..15f9f40 100644 --- a/pkg/scripts/preinstall.sh +++ b/pkg/scripts/preinstall.sh @@ -4,7 +4,7 @@ set -e -for svc in mirumd mirumw; do +for svc in mirum-server mirum-worker; do if ! getent group $svc >/dev/null; then groupadd --system $svc fi diff --git a/pkg/scripts/preremove.sh b/pkg/scripts/preremove.sh index fdb71a8..1c92a81 100644 --- a/pkg/scripts/preremove.sh +++ b/pkg/scripts/preremove.sh @@ -5,9 +5,9 @@ set -e if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ]; then - /bin/systemctl stop mirumd.service || true - /bin/systemctl disable mirumd.service || true + /bin/systemctl stop mirum-server.service || true + /bin/systemctl disable mirum-server.service || true - /bin/systemctl stop 'mirumw@*' || true - /bin/systemctl disable mirumw@.service || true + /bin/systemctl stop 'mirum-worker@*' || true + /bin/systemctl disable mirum-worker@.service || true fi diff --git a/pkg/mirumd.yaml b/pkg/server/config.yaml index 4828d9b..6ea9881 100644 --- a/pkg/mirumd.yaml +++ b/pkg/server/config.yaml @@ -2,10 +2,10 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Ignored when the corresponding systemd socket activation fd is present. -# See mirumd.socket for details (FileDescriptorName=grpc / web). +# See mirum-server.socket for details (FileDescriptorName=grpc / web). grpc_addr: :2026 web_addr: :3000 -admin_socket: /run/mirumd/admin.sock +admin_socket: /run/mirum-server/admin.sock database_uri: "" webhook_secret: "" token: "" diff --git a/pkg/mirumw-default.yaml b/pkg/worker/default.yaml index 3fc2076..f982ba1 100644 --- a/pkg/mirumw-default.yaml +++ b/pkg/worker/default.yaml @@ -5,13 +5,13 @@ server: localhost:2026 # Ed25519 private key for worker authentication (PEM-encoded PKCS8). # Generate with: -# openssl genpkey -algorithm Ed25519 -out /etc/mirum/mirumw-default.key -# chmod 600 /etc/mirum/mirumw-default.key +# openssl genpkey -algorithm Ed25519 -out /etc/mirum/worker/default.key +# chmod 600 /etc/mirum/worker/default.key # # Register the public key on the daemon: -# PUBKEY=$(openssl pkey -in /etc/mirum/mirumw-default.key -pubout -outform der | base64 -w0) -# mirumd --socket /run/mirumd/admin.sock worker add --pubkey "$PUBKEY" -key_file: /etc/mirum/mirumw-default.key +# PUBKEY=$(openssl pkey -in /etc/mirum/worker/default.key -pubout -outform der | base64 -w0) +# mirum-server --socket /run/mirum-server/admin.sock worker create --pubkey "$PUBKEY" +key_file: /etc/mirum/worker/default.key # Custom CA certificate for self-signed/dev TLS. # Leave empty to use system trust store. |
