From 1c073c677a19c3cf6dafc6d3db4afbbb2000c9d3 Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Mon, 13 Apr 2026 04:45:07 +0100 Subject: More idiomatic folder structure, adds editorconfigs and codeowners --- .editorconfig | 21 +++ CODEOWNERS | 5 + REUSE.toml | 4 +- Taskfile.yml | 8 +- cmd/mirum-server/licenses.go | 2 +- cmd/mirum-server/web/.prettierignore | 2 + internal/licensegen/main.go | 224 ------------------------- internal/licensegen/scan_go.go | 102 ------------ internal/licensegen/scan_npm.go | 238 --------------------------- nfpm.yaml | 14 +- packaging/apt-ftparchive.conf | 29 ++++ packaging/dl/404.html | 17 ++ packaging/dl/index.html | 17 ++ packaging/dl/robots.txt | 2 + packaging/logo.svg | 25 +++ packaging/mirum-server.service | 64 +++++++ packaging/mirum-worker@.service | 48 ++++++ packaging/scripts/postinstall.sh | 13 ++ packaging/scripts/preinstall.sh | 14 ++ packaging/scripts/preremove.sh | 13 ++ packaging/server/config.yaml | 27 +++ packaging/worker/default.yaml | 19 +++ pkg/apt-ftparchive.conf | 29 ---- pkg/dl/404.html | 17 -- pkg/dl/index.html | 17 -- pkg/dl/robots.txt | 2 - pkg/logo.svg | 25 --- pkg/mirum-server.service | 64 ------- pkg/mirum-worker@.service | 48 ------ pkg/scripts/postinstall.sh | 13 -- pkg/scripts/preinstall.sh | 14 -- pkg/scripts/preremove.sh | 13 -- pkg/server/config.yaml | 27 --- pkg/worker/default.yaml | 19 --- tools/licensegen/main.go | 224 +++++++++++++++++++++++++ tools/licensegen/scan_go.go | 102 ++++++++++++ tools/licensegen/scan_npm.go | 238 +++++++++++++++++++++++++++ 37 files changed, 894 insertions(+), 866 deletions(-) create mode 100644 .editorconfig create mode 100644 CODEOWNERS delete mode 100644 internal/licensegen/main.go delete mode 100644 internal/licensegen/scan_go.go delete mode 100644 internal/licensegen/scan_npm.go create mode 100644 packaging/apt-ftparchive.conf create mode 100644 packaging/dl/404.html create mode 100644 packaging/dl/index.html create mode 100644 packaging/dl/robots.txt create mode 100644 packaging/logo.svg create mode 100644 packaging/mirum-server.service create mode 100644 packaging/mirum-worker@.service create mode 100644 packaging/scripts/postinstall.sh create mode 100644 packaging/scripts/preinstall.sh create mode 100644 packaging/scripts/preremove.sh create mode 100644 packaging/server/config.yaml create mode 100644 packaging/worker/default.yaml delete mode 100644 pkg/apt-ftparchive.conf delete mode 100644 pkg/dl/404.html delete mode 100644 pkg/dl/index.html delete mode 100644 pkg/dl/robots.txt delete mode 100644 pkg/logo.svg delete mode 100644 pkg/mirum-server.service delete mode 100644 pkg/mirum-worker@.service delete mode 100644 pkg/scripts/postinstall.sh delete mode 100644 pkg/scripts/preinstall.sh delete mode 100644 pkg/scripts/preremove.sh delete mode 100644 pkg/server/config.yaml delete mode 100644 pkg/worker/default.yaml create mode 100644 tools/licensegen/main.go create mode 100644 tools/licensegen/scan_go.go create mode 100644 tools/licensegen/scan_npm.go diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..61d0094 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,21 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.go] +indent_style = tab + +[{Makefile,*.mk}] +indent_style = tab + +[*.md] +trim_trailing_whitespace = false diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..f496bf2 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,5 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +* @mrdimidium +cmd/mirum-server/web/ @PaulineNemchak diff --git a/REUSE.toml b/REUSE.toml index 8cd1ea7..791cdce 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -11,8 +11,8 @@ path = [ "cmd/mirum-server/web/*.json", "go.mod", "go.sum", - "pkg/dl/*", - "pkg/logo.svg", + "packaging/dl/*", + "packaging/logo.svg", "buf.*", ] SPDX-FileCopyrightText = "2026 Nikolay Govorov " diff --git a/Taskfile.yml b/Taskfile.yml index e7646ea..d81ad36 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -88,14 +88,14 @@ tasks: sources: - go.sum - cmd/mirum-server/web/package-lock.json - - internal/licensegen/**/*.go + - tools/licensegen/**/*.go - LICENSES.go - LICENSES/*.txt generates: - build/licenses.json cmds: - mkdir -p {{.BUILD_DIR}} - - go run -tags licensegen ./internal/licensegen -out {{.BUILD_DIR}}/licenses.json + - go run -tags licensegen ./tools/licensegen -out {{.BUILD_DIR}}/licenses.json build: desc: "Build binaries (override GOOS/GOARCH for cross-compilation)" @@ -240,7 +240,7 @@ tasks: # Upload static files gpg --export --armor "{{.GPG_KEY_ID}}" > "$GNUPGHOME/public.gpg" rclone copyto "$GNUPGHOME/public.gpg" "s3:{{.S3_BUCKET}}/public.gpg" - rclone copy pkg/dl/ "s3:{{.S3_BUCKET}}/" + rclone copy packaging/dl/ "s3:{{.S3_BUCKET}}/" # APT repository CHANNEL={{.CHANNEL}} @@ -256,7 +256,7 @@ tasks: cp {{.DIST_DIR}}/*.deb apt-repo/pool/$CHANNEL/ cd apt-repo - apt-ftparchive generate "$(pwd)/../pkg/apt-ftparchive.conf" + apt-ftparchive generate "$(pwd)/../packaging/apt-ftparchive.conf" apt-ftparchive \ -o APT::FTPArchive::Release::Codename=$CHANNEL \ -o APT::FTPArchive::Release::Components=main \ diff --git a/cmd/mirum-server/licenses.go b/cmd/mirum-server/licenses.go index d6630ce..ca01f82 100644 --- a/cmd/mirum-server/licenses.go +++ b/cmd/mirum-server/licenses.go @@ -14,7 +14,7 @@ import ( // primarySPDX is the SPDX identifier of mirum itself. const primarySPDX = "AGPL-3.0-or-later" -// licensesPageData mirrors internal/licensegen's Manifest with a primary +// licensesPageData mirrors tools/licensegen's Manifest with a primary // license header prepended. The frontend consumes it verbatim. type licensesPageData struct { Primary struct { diff --git a/cmd/mirum-server/web/.prettierignore b/cmd/mirum-server/web/.prettierignore index d04bea6..b32d4d2 100644 --- a/cmd/mirum-server/web/.prettierignore +++ b/cmd/mirum-server/web/.prettierignore @@ -3,4 +3,6 @@ /node_modules /gen + +# it's a go template, not just a html shell.html diff --git a/internal/licensegen/main.go b/internal/licensegen/main.go deleted file mode 100644 index cafb861..0000000 --- a/internal/licensegen/main.go +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright (c) 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -// Command licensegen writes build/licenses.json — the third-party dependency -// manifest embedded into mirum binaries. Output is pre-grouped: each -// ecosystem contains SPDX groups; each group contains text variants (packages -// sharing identical LICENSE text collapse into one variant); each variant -// lists its deps. The frontend renders without further transformation. -package main - -import ( - "cmp" - "encoding/json" - "flag" - "fmt" - "log" - "os" - "path/filepath" - "slices" - "strconv" - "strings" - "time" - - "github.com/github/go-spdx/v2/spdxexp" -) - -// allowedSPDX are SPDX ids approved for runtime deps. mirum sells a -// commercial license, so GPL-family ids are excluded even though the -// upstream distribution is AGPL-3.0-or-later. -var allowedSPDX = []string{ - "0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", - "CC0-1.0", "ISC", "MIT", "OFL-1.1", - "Unicode-3.0", "Unlicense", "Zlib", -} - -// collapsedScopes are npm scopes whose sub-packages come from a single -// upstream monorepo and should render as one "@scope" row. All sub-packages -// of a collapsed scope must declare the same SPDX — mismatch aborts. -var collapsedScopes = []string{"@radix-ui"} - -type Dep struct { - Name string `json:"name"` - Version string `json:"version,omitempty"` - SPDX string `json:"spdx"` - URL string `json:"url,omitempty"` - Count int `json:"count,omitempty"` // >0 means a collapsed scope entry covering N sub-packages - - atoms []string // internal: atomic SPDX ids for cross-listing - text string // internal: verbatim LICENSE text -} - -type Variant struct { - Text string `json:"text"` - Deps []Dep `json:"deps"` -} - -type Group struct { - SPDX string `json:"spdx"` - Total int `json:"total"` - Variants []Variant `json:"variants"` -} - -type Ecosystem struct { - Total int `json:"total"` - Groups []Group `json:"groups"` -} - -type Manifest struct { - GeneratedAt string `json:"generated_at"` - Go Ecosystem `json:"go"` - NPM Ecosystem `json:"npm"` -} - -func main() { - log.SetFlags(0) - log.SetPrefix("licensegen: ") - - out := flag.String("out", "", "output path for licenses.json") - repo := flag.String("repo", "", "repo root (defaults to walking up from cwd)") - flag.Parse() - - if *out == "" { - log.Fatal("missing required -out flag") - } - root, err := resolveRepoRoot(*repo) - if err != nil { - log.Fatal(err) - } - - goDeps, err := scanGo(root) - if err != nil { - log.Fatalf("scan go: %v", err) - } - npmDeps, err := scanNPM(root) - if err != nil { - log.Fatalf("scan npm: %v", err) - } - - m := Manifest{ - GeneratedAt: sourceDateEpoch().UTC().Format(time.RFC3339), - Go: group(goDeps), - NPM: group(npmDeps), - } - - body, err := json.MarshalIndent(m, "", " ") - if err != nil { - log.Fatal(err) - } - if err := os.WriteFile(*out, append(body, '\n'), 0o644); err != nil { - log.Fatalf("write %s: %v", *out, err) - } - log.Printf("wrote %s (go=%d npm=%d)", *out, m.Go.Total, m.NPM.Total) -} - -// group assembles deps into SPDX atoms × text variants. A dep with a compound -// expression ("A AND B") is cross-listed under every atom. -// Unexported Dep fields (atoms, text) are dropped by encoding/json. -func group(deps []Dep) Ecosystem { - // atom → text → *Variant - byAtom := map[string]map[string]*Variant{} - for _, d := range deps { - for _, atom := range d.atoms { - byText := byAtom[atom] - if byText == nil { - byText = map[string]*Variant{} - byAtom[atom] = byText - } - v := byText[d.text] - if v == nil { - v = &Variant{Text: d.text} - byText[d.text] = v - } - v.Deps = append(v.Deps, d) - } - } - - groups := make([]Group, 0, len(byAtom)) - for atom, byText := range byAtom { - variants := make([]Variant, 0, len(byText)) - total := 0 - for _, v := range byText { - slices.SortFunc(v.Deps, func(a, b Dep) int { return strings.Compare(a.Name, b.Name) }) - variants = append(variants, *v) - total += len(v.Deps) - } - slices.SortFunc(variants, func(a, b Variant) int { - return cmp.Or( - cmp.Compare(len(b.Deps), len(a.Deps)), // desc - cmp.Compare(a.Deps[0].Name, b.Deps[0].Name), - ) - }) - groups = append(groups, Group{SPDX: atom, Total: total, Variants: variants}) - } - slices.SortFunc(groups, func(a, b Group) int { - return cmp.Or(cmp.Compare(b.Total, a.Total), cmp.Compare(a.SPDX, b.SPDX)) - }) - return Ecosystem{Total: len(deps), Groups: groups} -} - -// validateSPDX checks expr against allowedSPDX and returns its atomic ids. -func validateSPDX(expr string) ([]string, error) { - ok, err := spdxexp.Satisfies(expr, allowedSPDX) - if err != nil { - return nil, err - } - if !ok { - return nil, fmt.Errorf("SPDX %q not allowed", expr) - } - return spdxexp.ExtractLicenses(expr) -} - -// readLicenseFile returns the contents and path of the first LICENSE-like -// file in dir. "LICENSE", "LICENCE", "COPYING" prefixes with any extension. -func readLicenseFile(dir string) (string, string, error) { - entries, err := os.ReadDir(dir) - if err != nil { - return "", "", err - } - for _, e := range entries { - n := strings.ToLower(e.Name()) - if strings.HasPrefix(n, "license") || strings.HasPrefix(n, "licence") || strings.HasPrefix(n, "copying") { - p := filepath.Join(dir, e.Name()) - if b, err := os.ReadFile(p); err == nil { - return normalize(string(b)), p, nil - } - } - } - return "", "", os.ErrNotExist -} - -// normalize strips BOM + trims whitespace + LF line endings, so near-identical -// texts (differing only by trailing blank lines or CRLF) dedupe. -func normalize(s string) string { - s = strings.TrimPrefix(s, "\ufeff") - s = strings.ReplaceAll(s, "\r\n", "\n") - return strings.TrimSpace(s) + "\n" -} - -func sourceDateEpoch() time.Time { - if v, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); v > 0 { - return time.Unix(v, 0) - } - return time.Now() -} - -func resolveRepoRoot(explicit string) (string, error) { - if explicit != "" { - return filepath.Abs(explicit) - } - dir, err := os.Getwd() - if err != nil { - return "", err - } - for { - if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { - return dir, nil - } - p := filepath.Dir(dir) - if p == dir { - return "", os.ErrNotExist - } - dir = p - } -} diff --git a/internal/licensegen/scan_go.go b/internal/licensegen/scan_go.go deleted file mode 100644 index 67b7e4b..0000000 --- a/internal/licensegen/scan_go.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -package main - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "os" - "os/exec" - - "github.com/google/licensecheck" -) - -// goEntrypoints are the main packages whose linker inputs form the runtime -// graph. Every production binary we ship lives under cmd/. -var goEntrypoints = []string{ - "./cmd/mirum-server", - "./cmd/mirum-worker", - "./cmd/mirum", -} - -// scanGo reads the prod dependency graph via `go list -deps -json` and -// classifies each module's LICENSE file with google/licensecheck at a 75% -// coverage threshold. Below that we refuse to guess. -func scanGo(root string) ([]Dep, error) { - args := append([]string{"list", "-tags=licensegen", "-deps", "-json"}, goEntrypoints...) - cmd := exec.Command("go", args...) - cmd.Dir = root - cmd.Stderr = os.Stderr - stdout, err := cmd.StdoutPipe() - if err != nil { - return nil, err - } - if err := cmd.Start(); err != nil { - return nil, err - } - - type mod struct { - Path, Version, Dir string - Main bool - Replace *mod - } - type pkg struct { - Standard bool - Module *mod - } - - mods := map[string]*mod{} - dec := json.NewDecoder(stdout) - for { - var p pkg - if err := dec.Decode(&p); err != nil { - if errors.Is(err, io.EOF) { - break - } - return nil, err - } - if p.Standard || p.Module == nil || p.Module.Main { - continue - } - m := p.Module - if m.Replace != nil { - m = m.Replace - } - if m.Dir == "" { - return nil, fmt.Errorf("%s: empty Dir (run `go mod download`)", m.Path) - } - mods[m.Path+"@"+m.Version] = m - } - if err := cmd.Wait(); err != nil { - return nil, err - } - - deps := make([]Dep, 0, len(mods)) - for _, m := range mods { - text, path, err := readLicenseFile(m.Dir) - if err != nil { - return nil, fmt.Errorf("%s@%s: no LICENSE file", m.Path, m.Version) - } - cov := licensecheck.Scan([]byte(text)) - if cov.Percent < 75 || len(cov.Match) == 0 { - return nil, fmt.Errorf("%s@%s: cannot classify %s (%.0f%%)", m.Path, m.Version, path, cov.Percent) - } - spdx := cov.Match[0].ID - atoms, err := validateSPDX(spdx) - if err != nil { - return nil, fmt.Errorf("%s@%s: %w", m.Path, m.Version, err) - } - deps = append(deps, Dep{ - Name: m.Path, - Version: m.Version, - SPDX: spdx, - URL: "https://pkg.go.dev/" + m.Path + "@" + m.Version, - atoms: atoms, - text: text, - }) - } - return deps, nil -} diff --git a/internal/licensegen/scan_npm.go b/internal/licensegen/scan_npm.go deleted file mode 100644 index 0cb16f1..0000000 --- a/internal/licensegen/scan_npm.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright (c) 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -package main - -import ( - "encoding/json" - "fmt" - "os" - "path/filepath" - "slices" - "strings" -) - -const webDir = "cmd/mirum-server/web" - -// scanNPM reads package-lock.json (v3+), filters to runtime packages, reads -// each package's LICENSE verbatim (or synthesizes a copyright notice when -// none ships — canonical SPDX text is never substituted), and collapses the -// scopes listed in collapsedScopes into single "@scope" rows. -func scanNPM(root string) ([]Dep, error) { - var lock struct { - LockfileVersion int `json:"lockfileVersion"` - Packages map[string]struct { - Version string `json:"version"` - License any `json:"license"` - Dev, DevOptional, Link, Peer bool - } `json:"packages"` - } - raw, err := os.ReadFile(filepath.Join(root, webDir, "package-lock.json")) - if err != nil { - return nil, err - } - if err := json.Unmarshal(raw, &lock); err != nil { - return nil, err - } - if lock.LockfileVersion < 3 { - return nil, fmt.Errorf("lockfileVersion %d unsupported, regenerate with npm v7+", lock.LockfileVersion) - } - - // npm hoists identical name@version under multiple paths; skip duplicates - // before the expensive LICENSE read. - seen := map[string]bool{} - var deps []Dep - for path, pkg := range lock.Packages { - if path == "" || pkg.Link || pkg.Dev || pkg.DevOptional { - continue - } - name := npmName(path) - key := name + "@" + pkg.Version - if seen[key] { - continue - } - seen[key] = true - - expr, err := npmSPDX(pkg.License) - if err != nil { - return nil, fmt.Errorf("%s: %w", key, err) - } - - atoms, err := validateSPDX(expr) - if err != nil { - return nil, fmt.Errorf("%s declares %q: %w", key, expr, err) - } - - pkgDir := filepath.Join(root, webDir, path) - deps = append(deps, Dep{ - Name: name, - Version: pkg.Version, - SPDX: expr, - URL: "https://www.npmjs.com/package/" + name + "/v/" + pkg.Version, - atoms: atoms, - text: npmText(pkgDir, name), - }) - } - - scopeOf := func(name string) string { - if !strings.HasPrefix(name, "@") { - return "" - } - scope, _, ok := strings.Cut(name, "/") - if !ok { - return "" - } - return scope - } - - buckets := map[string][]Dep{} - var out []Dep - for _, d := range deps { - if s := scopeOf(d.Name); slices.Contains(collapsedScopes, s) { - buckets[s] = append(buckets[s], d) - continue - } - out = append(out, d) - } - - for scope, items := range buckets { - spdx := items[0].SPDX - version := items[0].Version - for _, d := range items[1:] { - if d.SPDX != spdx { - return nil, fmt.Errorf("scope %s: mixed SPDX %q vs %q (%s)", scope, spdx, d.SPDX, d.Name) - } - if d.Version != version { - version = "" - } - } - - // Prefer umbrella LICENSE, then any sub-package's own. - text, _, err := readLicenseFile(filepath.Join(root, webDir, "node_modules", strings.TrimPrefix(scope, "@"))) - if err != nil { - text = items[0].text - for _, d := range items { - if !strings.HasPrefix(d.text, "Copyright (c) contributors to ") { - text = d.text - break - } - } - } - - out = append(out, Dep{ - Name: strings.TrimPrefix(scope, "@"), - Version: version, - SPDX: spdx, - URL: "https://www.npmjs.com/~" + strings.TrimPrefix(scope, "@"), - Count: len(items), - atoms: items[0].atoms, - text: text, - }) - } - return out, nil -} - -// npmName extracts the package name from an npm lockfile key like -// "node_modules/foo" or "node_modules/foo/node_modules/@scope/bar". -func npmName(path string) string { - i := strings.LastIndex(path, "node_modules/") - if i < 0 { - return "" - } - n := path[i+len("node_modules/"):] - if strings.HasPrefix(n, "@") { - return n // scoped "@scope/name" is one name - } - head, _, _ := strings.Cut(n, "/") - return head -} - -// npmSPDX normalises package.json's `license` field. Modern packages use a -// string; we accept legacy array-of-objects too. Outer parens are stripped -// so "(MIT OR Apache-2.0)" displays as "MIT OR Apache-2.0". -func npmSPDX(v any) (string, error) { - var s string - switch x := v.(type) { - case string: - s = strings.TrimSpace(x) - case []any: - var ids []string - for _, it := range x { - if m, ok := it.(map[string]any); ok { - if t, ok := m["type"].(string); ok && t != "" { - ids = append(ids, t) - } - } - } - s = strings.Join(ids, " OR ") - } - if s == "" { - return "", fmt.Errorf("no license field") - } - for strings.HasPrefix(s, "(") && strings.HasSuffix(s, ")") { - s = strings.TrimSpace(s[1 : len(s)-1]) - } - return s, nil -} - -// npmText returns the LICENSE text shipped with a package, or a copyright -// notice derived from package.json when no LICENSE file exists. Canonical -// SPDX text is never substituted — doing so would claim the author wrote -// something they didn't ship. -func npmText(pkgDir, name string) string { - if text, _, err := readLicenseFile(pkgDir); err == nil { - return text - } - if c := copyrightFromPackageJSON(pkgDir); c != "" { - return c + "\n" - } - return "Copyright (c) contributors to " + name + "\n" -} - -// copyrightFromPackageJSON builds a "Copyright (c) ..." line from the -// package's author/contributors fields. Returns empty if neither is present. -func copyrightFromPackageJSON(pkgDir string) string { - data, err := os.ReadFile(filepath.Join(pkgDir, "package.json")) - if err != nil { - return "" - } - var pj struct { - Author any `json:"author"` - Contributors []any `json:"contributors"` - } - if err := json.Unmarshal(data, &pj); err != nil { - return "" - } - var names []string - for _, v := range append([]any{pj.Author}, pj.Contributors...) { - if s := personName(v); s != "" { - names = append(names, s) - } - } - if len(names) == 0 { - return "" - } - return "Copyright (c) " + strings.Join(names, ", ") -} - -// personName renders an npm author/contributor entry (string or {name,email}) -// as "Name " or just "Name". -func personName(v any) string { - switch x := v.(type) { - case string: - return strings.TrimSpace(x) - case map[string]any: - name, _ := x["name"].(string) - email, _ := x["email"].(string) - name = strings.TrimSpace(name) - email = strings.TrimSpace(email) - if name == "" { - return "" - } - if email == "" { - return name - } - return name + " <" + email + ">" - } - return "" -} diff --git a/nfpm.yaml b/nfpm.yaml index fa12a4a..c776bf7 100644 --- a/nfpm.yaml +++ b/nfpm.yaml @@ -33,7 +33,7 @@ contents: file_info: mode: 0755 - - src: pkg/server/config.yaml + - src: packaging/server/config.yaml dst: /etc/mirum/server/config.yaml type: config|noreplace file_info: @@ -41,7 +41,7 @@ contents: owner: root group: mirum-server - - src: pkg/worker/default.yaml + - src: packaging/worker/default.yaml dst: /etc/mirum/worker/default.yaml type: config|noreplace file_info: @@ -49,12 +49,12 @@ contents: owner: root group: mirum-worker - - src: pkg/mirum-server.service + - src: packaging/mirum-server.service dst: /usr/lib/systemd/system/mirum-server.service file_info: mode: 0644 - - src: pkg/mirum-worker@.service + - src: packaging/mirum-worker@.service dst: /usr/lib/systemd/system/mirum-worker@.service file_info: mode: 0644 @@ -75,9 +75,9 @@ contents: scripts: - preinstall: pkg/scripts/preinstall.sh - postinstall: pkg/scripts/postinstall.sh - preremove: pkg/scripts/preremove.sh + preinstall: packaging/scripts/preinstall.sh + postinstall: packaging/scripts/postinstall.sh + preremove: packaging/scripts/preremove.sh deb: signature: diff --git a/packaging/apt-ftparchive.conf b/packaging/apt-ftparchive.conf new file mode 100644 index 0000000..d581965 --- /dev/null +++ b/packaging/apt-ftparchive.conf @@ -0,0 +1,29 @@ +// Copyright (c) 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +Dir { + ArchiveDir "."; + CacheDir "/tmp/apt-cache"; +}; + +Default { + Packages::Compress ". gzip"; + Packages::Extensions ".deb"; +}; + +TreeDefault { + Packages "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"; + BinCacheDB "packages-$(DIST)-$(ARCH).db"; +}; + +Tree "dists/stable" { + Sections "main"; + Architectures "amd64 arm64 riscv64 ppc64le"; + Directory "pool/stable"; +}; + +Tree "dists/nightly" { + Sections "main"; + Architectures "amd64 arm64 riscv64 ppc64le"; + Directory "pool/nightly"; +}; diff --git a/packaging/dl/404.html b/packaging/dl/404.html new file mode 100644 index 0000000..3ae5a33 --- /dev/null +++ b/packaging/dl/404.html @@ -0,0 +1,17 @@ + + + + + + + + 404 - Mirum + + +

404

+

Looking for the mirum docs? Visit mirum site.

+ + diff --git a/packaging/dl/index.html b/packaging/dl/index.html new file mode 100644 index 0000000..60c970d --- /dev/null +++ b/packaging/dl/index.html @@ -0,0 +1,17 @@ + + + + + + + + + Mirum - Package Repository + + +

Visit mirum site.

+ + diff --git a/packaging/dl/robots.txt b/packaging/dl/robots.txt new file mode 100644 index 0000000..c6742d8 --- /dev/null +++ b/packaging/dl/robots.txt @@ -0,0 +1,2 @@ +User-Agent: * +Disallow: / diff --git a/packaging/logo.svg b/packaging/logo.svg new file mode 100644 index 0000000..0a2bfc2 --- /dev/null +++ b/packaging/logo.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packaging/mirum-server.service b/packaging/mirum-server.service new file mode 100644 index 0000000..69e440f --- /dev/null +++ b/packaging/mirum-server.service @@ -0,0 +1,64 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +[Unit] +Description=Mirum daemon (modern CI platform) +Requires=network-online.target +After=time-sync.target network-online.target remote-fs.target nss-lookup.target postgresql.service +Wants=time-sync.target + +# Socket activation (optional): +# Create mirum-server.socket with named file descriptors "web" and "grpc": +# +# [Socket] +# ListenStream=0.0.0.0:3000 +# FileDescriptorName=web +# +# [Socket] +# ListenStream=0.0.0.0:2026 +# FileDescriptorName=grpc +# +# Without socket activation the daemon binds www_addr and grpc_addr from config. + +[Service] +Type=notify +User=mirum-server +Group=mirum-server +Restart=always +RestartSec=30 +WatchdogSec=30 +NotifyAccess=main +ExecPaths=/usr/local/bin/mirum-server /usr/lib +ExecStart=/usr/local/bin/mirum-server daemon --config=/etc/mirum/server/config.yaml +LimitCORE=infinity +LimitNOFILE=500000 +AmbientCapabilities=CAP_NET_BIND_SERVICE + +# %p is resolved to the systemd unit name +LogsDirectory=%p +StateDirectory=%p +CacheDirectory=%p +RuntimeDirectory=%p + +UMask=0077 +LockPersonality=yes +NoNewPrivileges=yes +PrivateDevices=yes +PrivateTmp=true +ProcSubset=pid +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectProc=invisible +ProtectSystem=strict +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictSUIDSGID=yes + +[Install] +# service should not start from the rescue shell (rescue.target). +WantedBy=multi-user.target diff --git a/packaging/mirum-worker@.service b/packaging/mirum-worker@.service new file mode 100644 index 0000000..dbbc5e0 --- /dev/null +++ b/packaging/mirum-worker@.service @@ -0,0 +1,48 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +[Unit] +Description=Mirum worker %i (modern CI platform) +Requires=network-online.target +After=time-sync.target network-online.target remote-fs.target nss-lookup.target +Wants=time-sync.target + +[Service] +Type=notify +User=mirum-worker +Group=mirum-worker +Restart=always +RestartSec=30 +WatchdogSec=30 +NotifyAccess=main +ExecPaths=/usr/local/bin/mirum-worker +ExecStart=/usr/local/bin/mirum-worker --config=/etc/mirum/worker/%i.yaml +LimitCORE=infinity +LimitNOFILE=500000 +AmbientCapabilities= + +# %p is resolved to the systemd unit name +LogsDirectory=%p +StateDirectory=%p +CacheDirectory=%p +RuntimeDirectory=%p + +UMask=0077 +LockPersonality=yes +NoNewPrivileges=yes +PrivateTmp=true +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelLogs=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectSystem=strict +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX +RestrictNamespaces=yes +RestrictSUIDSGID=yes + +[Install] +# service should not start from the rescue shell (rescue.target). +WantedBy=multi-user.target diff --git a/packaging/scripts/postinstall.sh b/packaging/scripts/postinstall.sh new file mode 100644 index 0000000..3479c75 --- /dev/null +++ b/packaging/scripts/postinstall.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ] && [ -f /usr/lib/systemd/system/mirum-server.service ]; then + /bin/systemctl daemon-reload + + # Don't enable by default, don't know in advance whether it's a daemon or a worker + # /bin/systemctl enable mirum-server + # /bin/systemctl enable mirum-worker +fi diff --git a/packaging/scripts/preinstall.sh b/packaging/scripts/preinstall.sh new file mode 100644 index 0000000..15f9f40 --- /dev/null +++ b/packaging/scripts/preinstall.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +for svc in mirum-server mirum-worker; do + if ! getent group $svc >/dev/null; then + groupadd --system $svc + fi + if ! getent passwd $svc >/dev/null; then + useradd --system --gid $svc --no-create-home --shell /usr/sbin/nologin $svc + fi +done diff --git a/packaging/scripts/preremove.sh b/packaging/scripts/preremove.sh new file mode 100644 index 0000000..1c92a81 --- /dev/null +++ b/packaging/scripts/preremove.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +set -e + +if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ]; then + /bin/systemctl stop mirum-server.service || true + /bin/systemctl disable mirum-server.service || true + + /bin/systemctl stop 'mirum-worker@*' || true + /bin/systemctl disable mirum-worker@.service || true +fi diff --git a/packaging/server/config.yaml b/packaging/server/config.yaml new file mode 100644 index 0000000..6ea9881 --- /dev/null +++ b/packaging/server/config.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +# Ignored when the corresponding systemd socket activation fd is present. +# See mirum-server.socket for details (FileDescriptorName=grpc / web). +grpc_addr: :2026 +web_addr: :3000 +admin_socket: /run/mirum-server/admin.sock +database_uri: "" +webhook_secret: "" +token: "" +pepper: "" + +grpc_tls: + cert: "" + key: "" + +# Optional — omit or leave empty to disable TLS on the web listener. +# web_tls: +# cert: "" +# key: "" + +# CIDR list of trusted reverse proxies for X-Forwarded-For resolution. +# Empty = trust RemoteAddr only (safe default). +trusted_proxies: + - 127.0.0.0/8 + - ::1/128 diff --git a/packaging/worker/default.yaml b/packaging/worker/default.yaml new file mode 100644 index 0000000..a0b8e39 --- /dev/null +++ b/packaging/worker/default.yaml @@ -0,0 +1,19 @@ +# Copyright (c) 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +server: localhost:2026 + +# Ed25519 private key for worker authentication (PEM-encoded PKCS8). +# Generate with: +# openssl genpkey -algorithm Ed25519 -out /etc/mirum/worker/default.key +# chmod 0640 /etc/mirum/worker/default.key +# chown root:mirum-worker /etc/mirum/worker/default.key +# +# Register the public key on the daemon: +# PUBKEY=$(openssl pkey -in /etc/mirum/worker/default.key -pubout -outform der | base64 -w0) +# mirum-server --socket /run/mirum-server/admin.sock worker create --pubkey "$PUBKEY" +key_file: /etc/mirum/worker/default.key + +# Custom CA certificate for self-signed/dev TLS. +# Leave empty to use system trust store. +tls_ca: "" diff --git a/pkg/apt-ftparchive.conf b/pkg/apt-ftparchive.conf deleted file mode 100644 index d581965..0000000 --- a/pkg/apt-ftparchive.conf +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -Dir { - ArchiveDir "."; - CacheDir "/tmp/apt-cache"; -}; - -Default { - Packages::Compress ". gzip"; - Packages::Extensions ".deb"; -}; - -TreeDefault { - Packages "$(DIST)/$(SECTION)/binary-$(ARCH)/Packages"; - BinCacheDB "packages-$(DIST)-$(ARCH).db"; -}; - -Tree "dists/stable" { - Sections "main"; - Architectures "amd64 arm64 riscv64 ppc64le"; - Directory "pool/stable"; -}; - -Tree "dists/nightly" { - Sections "main"; - Architectures "amd64 arm64 riscv64 ppc64le"; - Directory "pool/nightly"; -}; diff --git a/pkg/dl/404.html b/pkg/dl/404.html deleted file mode 100644 index 3ae5a33..0000000 --- a/pkg/dl/404.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - 404 - Mirum - - -

404

-

Looking for the mirum docs? Visit mirum site.

- - diff --git a/pkg/dl/index.html b/pkg/dl/index.html deleted file mode 100644 index 60c970d..0000000 --- a/pkg/dl/index.html +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - Mirum - Package Repository - - -

Visit mirum site.

- - diff --git a/pkg/dl/robots.txt b/pkg/dl/robots.txt deleted file mode 100644 index c6742d8..0000000 --- a/pkg/dl/robots.txt +++ /dev/null @@ -1,2 +0,0 @@ -User-Agent: * -Disallow: / diff --git a/pkg/logo.svg b/pkg/logo.svg deleted file mode 100644 index 0a2bfc2..0000000 --- a/pkg/logo.svg +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/pkg/mirum-server.service b/pkg/mirum-server.service deleted file mode 100644 index 69e440f..0000000 --- a/pkg/mirum-server.service +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -[Unit] -Description=Mirum daemon (modern CI platform) -Requires=network-online.target -After=time-sync.target network-online.target remote-fs.target nss-lookup.target postgresql.service -Wants=time-sync.target - -# Socket activation (optional): -# Create mirum-server.socket with named file descriptors "web" and "grpc": -# -# [Socket] -# ListenStream=0.0.0.0:3000 -# FileDescriptorName=web -# -# [Socket] -# ListenStream=0.0.0.0:2026 -# FileDescriptorName=grpc -# -# Without socket activation the daemon binds www_addr and grpc_addr from config. - -[Service] -Type=notify -User=mirum-server -Group=mirum-server -Restart=always -RestartSec=30 -WatchdogSec=30 -NotifyAccess=main -ExecPaths=/usr/local/bin/mirum-server /usr/lib -ExecStart=/usr/local/bin/mirum-server daemon --config=/etc/mirum/server/config.yaml -LimitCORE=infinity -LimitNOFILE=500000 -AmbientCapabilities=CAP_NET_BIND_SERVICE - -# %p is resolved to the systemd unit name -LogsDirectory=%p -StateDirectory=%p -CacheDirectory=%p -RuntimeDirectory=%p - -UMask=0077 -LockPersonality=yes -NoNewPrivileges=yes -PrivateDevices=yes -PrivateTmp=true -ProcSubset=pid -ProtectClock=yes -ProtectControlGroups=yes -ProtectHome=yes -ProtectHostname=yes -ProtectKernelLogs=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -ProtectProc=invisible -ProtectSystem=strict -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX -RestrictNamespaces=yes -RestrictSUIDSGID=yes - -[Install] -# service should not start from the rescue shell (rescue.target). -WantedBy=multi-user.target diff --git a/pkg/mirum-worker@.service b/pkg/mirum-worker@.service deleted file mode 100644 index dbbc5e0..0000000 --- a/pkg/mirum-worker@.service +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -[Unit] -Description=Mirum worker %i (modern CI platform) -Requires=network-online.target -After=time-sync.target network-online.target remote-fs.target nss-lookup.target -Wants=time-sync.target - -[Service] -Type=notify -User=mirum-worker -Group=mirum-worker -Restart=always -RestartSec=30 -WatchdogSec=30 -NotifyAccess=main -ExecPaths=/usr/local/bin/mirum-worker -ExecStart=/usr/local/bin/mirum-worker --config=/etc/mirum/worker/%i.yaml -LimitCORE=infinity -LimitNOFILE=500000 -AmbientCapabilities= - -# %p is resolved to the systemd unit name -LogsDirectory=%p -StateDirectory=%p -CacheDirectory=%p -RuntimeDirectory=%p - -UMask=0077 -LockPersonality=yes -NoNewPrivileges=yes -PrivateTmp=true -ProtectClock=yes -ProtectControlGroups=yes -ProtectHome=yes -ProtectHostname=yes -ProtectKernelLogs=yes -ProtectKernelModules=yes -ProtectKernelTunables=yes -ProtectSystem=strict -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX -RestrictNamespaces=yes -RestrictSUIDSGID=yes - -[Install] -# service should not start from the rescue shell (rescue.target). -WantedBy=multi-user.target diff --git a/pkg/scripts/postinstall.sh b/pkg/scripts/postinstall.sh deleted file mode 100644 index 3479c75..0000000 --- a/pkg/scripts/postinstall.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -set -e - -if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ] && [ -f /usr/lib/systemd/system/mirum-server.service ]; then - /bin/systemctl daemon-reload - - # Don't enable by default, don't know in advance whether it's a daemon or a worker - # /bin/systemctl enable mirum-server - # /bin/systemctl enable mirum-worker -fi diff --git a/pkg/scripts/preinstall.sh b/pkg/scripts/preinstall.sh deleted file mode 100644 index 15f9f40..0000000 --- a/pkg/scripts/preinstall.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -set -e - -for svc in mirum-server mirum-worker; do - if ! getent group $svc >/dev/null; then - groupadd --system $svc - fi - if ! getent passwd $svc >/dev/null; then - useradd --system --gid $svc --no-create-home --shell /usr/sbin/nologin $svc - fi -done diff --git a/pkg/scripts/preremove.sh b/pkg/scripts/preremove.sh deleted file mode 100644 index 1c92a81..0000000 --- a/pkg/scripts/preremove.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -set -e - -if [ -x "/bin/systemctl" ] && [ -d /run/systemd/system ]; then - /bin/systemctl stop mirum-server.service || true - /bin/systemctl disable mirum-server.service || true - - /bin/systemctl stop 'mirum-worker@*' || true - /bin/systemctl disable mirum-worker@.service || true -fi diff --git a/pkg/server/config.yaml b/pkg/server/config.yaml deleted file mode 100644 index 6ea9881..0000000 --- a/pkg/server/config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -# Ignored when the corresponding systemd socket activation fd is present. -# See mirum-server.socket for details (FileDescriptorName=grpc / web). -grpc_addr: :2026 -web_addr: :3000 -admin_socket: /run/mirum-server/admin.sock -database_uri: "" -webhook_secret: "" -token: "" -pepper: "" - -grpc_tls: - cert: "" - key: "" - -# Optional — omit or leave empty to disable TLS on the web listener. -# web_tls: -# cert: "" -# key: "" - -# CIDR list of trusted reverse proxies for X-Forwarded-For resolution. -# Empty = trust RemoteAddr only (safe default). -trusted_proxies: - - 127.0.0.0/8 - - ::1/128 diff --git a/pkg/worker/default.yaml b/pkg/worker/default.yaml deleted file mode 100644 index a0b8e39..0000000 --- a/pkg/worker/default.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Copyright (c) 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -server: localhost:2026 - -# Ed25519 private key for worker authentication (PEM-encoded PKCS8). -# Generate with: -# openssl genpkey -algorithm Ed25519 -out /etc/mirum/worker/default.key -# chmod 0640 /etc/mirum/worker/default.key -# chown root:mirum-worker /etc/mirum/worker/default.key -# -# Register the public key on the daemon: -# PUBKEY=$(openssl pkey -in /etc/mirum/worker/default.key -pubout -outform der | base64 -w0) -# mirum-server --socket /run/mirum-server/admin.sock worker create --pubkey "$PUBKEY" -key_file: /etc/mirum/worker/default.key - -# Custom CA certificate for self-signed/dev TLS. -# Leave empty to use system trust store. -tls_ca: "" diff --git a/tools/licensegen/main.go b/tools/licensegen/main.go new file mode 100644 index 0000000..cafb861 --- /dev/null +++ b/tools/licensegen/main.go @@ -0,0 +1,224 @@ +// Copyright (c) 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +// Command licensegen writes build/licenses.json — the third-party dependency +// manifest embedded into mirum binaries. Output is pre-grouped: each +// ecosystem contains SPDX groups; each group contains text variants (packages +// sharing identical LICENSE text collapse into one variant); each variant +// lists its deps. The frontend renders without further transformation. +package main + +import ( + "cmp" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "time" + + "github.com/github/go-spdx/v2/spdxexp" +) + +// allowedSPDX are SPDX ids approved for runtime deps. mirum sells a +// commercial license, so GPL-family ids are excluded even though the +// upstream distribution is AGPL-3.0-or-later. +var allowedSPDX = []string{ + "0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", + "CC0-1.0", "ISC", "MIT", "OFL-1.1", + "Unicode-3.0", "Unlicense", "Zlib", +} + +// collapsedScopes are npm scopes whose sub-packages come from a single +// upstream monorepo and should render as one "@scope" row. All sub-packages +// of a collapsed scope must declare the same SPDX — mismatch aborts. +var collapsedScopes = []string{"@radix-ui"} + +type Dep struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + SPDX string `json:"spdx"` + URL string `json:"url,omitempty"` + Count int `json:"count,omitempty"` // >0 means a collapsed scope entry covering N sub-packages + + atoms []string // internal: atomic SPDX ids for cross-listing + text string // internal: verbatim LICENSE text +} + +type Variant struct { + Text string `json:"text"` + Deps []Dep `json:"deps"` +} + +type Group struct { + SPDX string `json:"spdx"` + Total int `json:"total"` + Variants []Variant `json:"variants"` +} + +type Ecosystem struct { + Total int `json:"total"` + Groups []Group `json:"groups"` +} + +type Manifest struct { + GeneratedAt string `json:"generated_at"` + Go Ecosystem `json:"go"` + NPM Ecosystem `json:"npm"` +} + +func main() { + log.SetFlags(0) + log.SetPrefix("licensegen: ") + + out := flag.String("out", "", "output path for licenses.json") + repo := flag.String("repo", "", "repo root (defaults to walking up from cwd)") + flag.Parse() + + if *out == "" { + log.Fatal("missing required -out flag") + } + root, err := resolveRepoRoot(*repo) + if err != nil { + log.Fatal(err) + } + + goDeps, err := scanGo(root) + if err != nil { + log.Fatalf("scan go: %v", err) + } + npmDeps, err := scanNPM(root) + if err != nil { + log.Fatalf("scan npm: %v", err) + } + + m := Manifest{ + GeneratedAt: sourceDateEpoch().UTC().Format(time.RFC3339), + Go: group(goDeps), + NPM: group(npmDeps), + } + + body, err := json.MarshalIndent(m, "", " ") + if err != nil { + log.Fatal(err) + } + if err := os.WriteFile(*out, append(body, '\n'), 0o644); err != nil { + log.Fatalf("write %s: %v", *out, err) + } + log.Printf("wrote %s (go=%d npm=%d)", *out, m.Go.Total, m.NPM.Total) +} + +// group assembles deps into SPDX atoms × text variants. A dep with a compound +// expression ("A AND B") is cross-listed under every atom. +// Unexported Dep fields (atoms, text) are dropped by encoding/json. +func group(deps []Dep) Ecosystem { + // atom → text → *Variant + byAtom := map[string]map[string]*Variant{} + for _, d := range deps { + for _, atom := range d.atoms { + byText := byAtom[atom] + if byText == nil { + byText = map[string]*Variant{} + byAtom[atom] = byText + } + v := byText[d.text] + if v == nil { + v = &Variant{Text: d.text} + byText[d.text] = v + } + v.Deps = append(v.Deps, d) + } + } + + groups := make([]Group, 0, len(byAtom)) + for atom, byText := range byAtom { + variants := make([]Variant, 0, len(byText)) + total := 0 + for _, v := range byText { + slices.SortFunc(v.Deps, func(a, b Dep) int { return strings.Compare(a.Name, b.Name) }) + variants = append(variants, *v) + total += len(v.Deps) + } + slices.SortFunc(variants, func(a, b Variant) int { + return cmp.Or( + cmp.Compare(len(b.Deps), len(a.Deps)), // desc + cmp.Compare(a.Deps[0].Name, b.Deps[0].Name), + ) + }) + groups = append(groups, Group{SPDX: atom, Total: total, Variants: variants}) + } + slices.SortFunc(groups, func(a, b Group) int { + return cmp.Or(cmp.Compare(b.Total, a.Total), cmp.Compare(a.SPDX, b.SPDX)) + }) + return Ecosystem{Total: len(deps), Groups: groups} +} + +// validateSPDX checks expr against allowedSPDX and returns its atomic ids. +func validateSPDX(expr string) ([]string, error) { + ok, err := spdxexp.Satisfies(expr, allowedSPDX) + if err != nil { + return nil, err + } + if !ok { + return nil, fmt.Errorf("SPDX %q not allowed", expr) + } + return spdxexp.ExtractLicenses(expr) +} + +// readLicenseFile returns the contents and path of the first LICENSE-like +// file in dir. "LICENSE", "LICENCE", "COPYING" prefixes with any extension. +func readLicenseFile(dir string) (string, string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return "", "", err + } + for _, e := range entries { + n := strings.ToLower(e.Name()) + if strings.HasPrefix(n, "license") || strings.HasPrefix(n, "licence") || strings.HasPrefix(n, "copying") { + p := filepath.Join(dir, e.Name()) + if b, err := os.ReadFile(p); err == nil { + return normalize(string(b)), p, nil + } + } + } + return "", "", os.ErrNotExist +} + +// normalize strips BOM + trims whitespace + LF line endings, so near-identical +// texts (differing only by trailing blank lines or CRLF) dedupe. +func normalize(s string) string { + s = strings.TrimPrefix(s, "\ufeff") + s = strings.ReplaceAll(s, "\r\n", "\n") + return strings.TrimSpace(s) + "\n" +} + +func sourceDateEpoch() time.Time { + if v, _ := strconv.ParseInt(os.Getenv("SOURCE_DATE_EPOCH"), 10, 64); v > 0 { + return time.Unix(v, 0) + } + return time.Now() +} + +func resolveRepoRoot(explicit string) (string, error) { + if explicit != "" { + return filepath.Abs(explicit) + } + dir, err := os.Getwd() + if err != nil { + return "", err + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + p := filepath.Dir(dir) + if p == dir { + return "", os.ErrNotExist + } + dir = p + } +} diff --git a/tools/licensegen/scan_go.go b/tools/licensegen/scan_go.go new file mode 100644 index 0000000..67b7e4b --- /dev/null +++ b/tools/licensegen/scan_go.go @@ -0,0 +1,102 @@ +// Copyright (c) 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + + "github.com/google/licensecheck" +) + +// goEntrypoints are the main packages whose linker inputs form the runtime +// graph. Every production binary we ship lives under cmd/. +var goEntrypoints = []string{ + "./cmd/mirum-server", + "./cmd/mirum-worker", + "./cmd/mirum", +} + +// scanGo reads the prod dependency graph via `go list -deps -json` and +// classifies each module's LICENSE file with google/licensecheck at a 75% +// coverage threshold. Below that we refuse to guess. +func scanGo(root string) ([]Dep, error) { + args := append([]string{"list", "-tags=licensegen", "-deps", "-json"}, goEntrypoints...) + cmd := exec.Command("go", args...) + cmd.Dir = root + cmd.Stderr = os.Stderr + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + + type mod struct { + Path, Version, Dir string + Main bool + Replace *mod + } + type pkg struct { + Standard bool + Module *mod + } + + mods := map[string]*mod{} + dec := json.NewDecoder(stdout) + for { + var p pkg + if err := dec.Decode(&p); err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, err + } + if p.Standard || p.Module == nil || p.Module.Main { + continue + } + m := p.Module + if m.Replace != nil { + m = m.Replace + } + if m.Dir == "" { + return nil, fmt.Errorf("%s: empty Dir (run `go mod download`)", m.Path) + } + mods[m.Path+"@"+m.Version] = m + } + if err := cmd.Wait(); err != nil { + return nil, err + } + + deps := make([]Dep, 0, len(mods)) + for _, m := range mods { + text, path, err := readLicenseFile(m.Dir) + if err != nil { + return nil, fmt.Errorf("%s@%s: no LICENSE file", m.Path, m.Version) + } + cov := licensecheck.Scan([]byte(text)) + if cov.Percent < 75 || len(cov.Match) == 0 { + return nil, fmt.Errorf("%s@%s: cannot classify %s (%.0f%%)", m.Path, m.Version, path, cov.Percent) + } + spdx := cov.Match[0].ID + atoms, err := validateSPDX(spdx) + if err != nil { + return nil, fmt.Errorf("%s@%s: %w", m.Path, m.Version, err) + } + deps = append(deps, Dep{ + Name: m.Path, + Version: m.Version, + SPDX: spdx, + URL: "https://pkg.go.dev/" + m.Path + "@" + m.Version, + atoms: atoms, + text: text, + }) + } + return deps, nil +} diff --git a/tools/licensegen/scan_npm.go b/tools/licensegen/scan_npm.go new file mode 100644 index 0000000..0cb16f1 --- /dev/null +++ b/tools/licensegen/scan_npm.go @@ -0,0 +1,238 @@ +// Copyright (c) 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +package main + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strings" +) + +const webDir = "cmd/mirum-server/web" + +// scanNPM reads package-lock.json (v3+), filters to runtime packages, reads +// each package's LICENSE verbatim (or synthesizes a copyright notice when +// none ships — canonical SPDX text is never substituted), and collapses the +// scopes listed in collapsedScopes into single "@scope" rows. +func scanNPM(root string) ([]Dep, error) { + var lock struct { + LockfileVersion int `json:"lockfileVersion"` + Packages map[string]struct { + Version string `json:"version"` + License any `json:"license"` + Dev, DevOptional, Link, Peer bool + } `json:"packages"` + } + raw, err := os.ReadFile(filepath.Join(root, webDir, "package-lock.json")) + if err != nil { + return nil, err + } + if err := json.Unmarshal(raw, &lock); err != nil { + return nil, err + } + if lock.LockfileVersion < 3 { + return nil, fmt.Errorf("lockfileVersion %d unsupported, regenerate with npm v7+", lock.LockfileVersion) + } + + // npm hoists identical name@version under multiple paths; skip duplicates + // before the expensive LICENSE read. + seen := map[string]bool{} + var deps []Dep + for path, pkg := range lock.Packages { + if path == "" || pkg.Link || pkg.Dev || pkg.DevOptional { + continue + } + name := npmName(path) + key := name + "@" + pkg.Version + if seen[key] { + continue + } + seen[key] = true + + expr, err := npmSPDX(pkg.License) + if err != nil { + return nil, fmt.Errorf("%s: %w", key, err) + } + + atoms, err := validateSPDX(expr) + if err != nil { + return nil, fmt.Errorf("%s declares %q: %w", key, expr, err) + } + + pkgDir := filepath.Join(root, webDir, path) + deps = append(deps, Dep{ + Name: name, + Version: pkg.Version, + SPDX: expr, + URL: "https://www.npmjs.com/package/" + name + "/v/" + pkg.Version, + atoms: atoms, + text: npmText(pkgDir, name), + }) + } + + scopeOf := func(name string) string { + if !strings.HasPrefix(name, "@") { + return "" + } + scope, _, ok := strings.Cut(name, "/") + if !ok { + return "" + } + return scope + } + + buckets := map[string][]Dep{} + var out []Dep + for _, d := range deps { + if s := scopeOf(d.Name); slices.Contains(collapsedScopes, s) { + buckets[s] = append(buckets[s], d) + continue + } + out = append(out, d) + } + + for scope, items := range buckets { + spdx := items[0].SPDX + version := items[0].Version + for _, d := range items[1:] { + if d.SPDX != spdx { + return nil, fmt.Errorf("scope %s: mixed SPDX %q vs %q (%s)", scope, spdx, d.SPDX, d.Name) + } + if d.Version != version { + version = "" + } + } + + // Prefer umbrella LICENSE, then any sub-package's own. + text, _, err := readLicenseFile(filepath.Join(root, webDir, "node_modules", strings.TrimPrefix(scope, "@"))) + if err != nil { + text = items[0].text + for _, d := range items { + if !strings.HasPrefix(d.text, "Copyright (c) contributors to ") { + text = d.text + break + } + } + } + + out = append(out, Dep{ + Name: strings.TrimPrefix(scope, "@"), + Version: version, + SPDX: spdx, + URL: "https://www.npmjs.com/~" + strings.TrimPrefix(scope, "@"), + Count: len(items), + atoms: items[0].atoms, + text: text, + }) + } + return out, nil +} + +// npmName extracts the package name from an npm lockfile key like +// "node_modules/foo" or "node_modules/foo/node_modules/@scope/bar". +func npmName(path string) string { + i := strings.LastIndex(path, "node_modules/") + if i < 0 { + return "" + } + n := path[i+len("node_modules/"):] + if strings.HasPrefix(n, "@") { + return n // scoped "@scope/name" is one name + } + head, _, _ := strings.Cut(n, "/") + return head +} + +// npmSPDX normalises package.json's `license` field. Modern packages use a +// string; we accept legacy array-of-objects too. Outer parens are stripped +// so "(MIT OR Apache-2.0)" displays as "MIT OR Apache-2.0". +func npmSPDX(v any) (string, error) { + var s string + switch x := v.(type) { + case string: + s = strings.TrimSpace(x) + case []any: + var ids []string + for _, it := range x { + if m, ok := it.(map[string]any); ok { + if t, ok := m["type"].(string); ok && t != "" { + ids = append(ids, t) + } + } + } + s = strings.Join(ids, " OR ") + } + if s == "" { + return "", fmt.Errorf("no license field") + } + for strings.HasPrefix(s, "(") && strings.HasSuffix(s, ")") { + s = strings.TrimSpace(s[1 : len(s)-1]) + } + return s, nil +} + +// npmText returns the LICENSE text shipped with a package, or a copyright +// notice derived from package.json when no LICENSE file exists. Canonical +// SPDX text is never substituted — doing so would claim the author wrote +// something they didn't ship. +func npmText(pkgDir, name string) string { + if text, _, err := readLicenseFile(pkgDir); err == nil { + return text + } + if c := copyrightFromPackageJSON(pkgDir); c != "" { + return c + "\n" + } + return "Copyright (c) contributors to " + name + "\n" +} + +// copyrightFromPackageJSON builds a "Copyright (c) ..." line from the +// package's author/contributors fields. Returns empty if neither is present. +func copyrightFromPackageJSON(pkgDir string) string { + data, err := os.ReadFile(filepath.Join(pkgDir, "package.json")) + if err != nil { + return "" + } + var pj struct { + Author any `json:"author"` + Contributors []any `json:"contributors"` + } + if err := json.Unmarshal(data, &pj); err != nil { + return "" + } + var names []string + for _, v := range append([]any{pj.Author}, pj.Contributors...) { + if s := personName(v); s != "" { + names = append(names, s) + } + } + if len(names) == 0 { + return "" + } + return "Copyright (c) " + strings.Join(names, ", ") +} + +// personName renders an npm author/contributor entry (string or {name,email}) +// as "Name " or just "Name". +func personName(v any) string { + switch x := v.(type) { + case string: + return strings.TrimSpace(x) + case map[string]any: + name, _ := x["name"].(string) + email, _ := x["email"].(string) + name = strings.TrimSpace(name) + email = strings.TrimSpace(email) + if name == "" { + return "" + } + if email == "" { + return name + } + return name + " <" + email + ">" + } + return "" +} -- Gilti