From a7b9c0ac6d5f23e7a948032df8ae7e820d73b8b3 Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Sat, 7 Feb 2026 21:32:08 +0000 Subject: Setup licenses page --- .cargo/config.toml | 5 + .github/workflows/build.yml | 4 +- Cargo.lock | 256 ++++++------ Cargo.toml | 32 +- Makefile | 9 + crates/zorian/Cargo.toml | 6 +- crates/zorian/build.rs | 13 + crates/zorian/src/assets/base.css | 19 + crates/zorian/src/controller_web.rs | 106 ++++- deny.toml | 1 + xtask/Cargo.toml | 16 + xtask/src/lib.rs | 4 + xtask/src/licenses.rs | 615 ++++++++++++++++++++++++++++ xtask/src/main.rs | 19 + 14 files changed, 963 insertions(+), 142 deletions(-) create mode 100644 .cargo/config.toml create mode 100644 crates/zorian/build.rs create mode 100644 xtask/Cargo.toml create mode 100644 xtask/src/lib.rs create mode 100644 xtask/src/licenses.rs create mode 100644 xtask/src/main.rs diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..6736699 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,5 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +[alias] +xtask = "run --release --package xtask --" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3e42e24..4d1c4b3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -89,8 +89,8 @@ jobs: GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} NFPM_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov + - name: Install tools + run: make setup - name: Generate coverage (lcov) run: cargo llvm-cov --all-features --workspace --lcov --output-path coverage.lcov diff --git a/Cargo.lock b/Cargo.lock index 61e2261..219520b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -126,7 +126,6 @@ dependencies = [ "matchit", "memchr", "mime", - "multer", "percent-encoding", "pin-project-lite", "serde_core", @@ -160,27 +159,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-extra" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fef252edff26ddba56bbcdf2ee3307b8129acb86f5749b68990c168a6fcc9c76" -dependencies = [ - "axum", - "axum-core", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "mime", - "pin-project-lite", - "tower-layer", - "tower-service", - "tracing", -] - [[package]] name = "axum-macros" version = "0.5.0" @@ -307,6 +285,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "camino" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" +dependencies = [ + "serde_core", +] + [[package]] name = "cc" version = "1.2.55" @@ -319,6 +306,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cfg-expr" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78cef5b5a1a6827c7322ae2a636368a573006b27cfa76c7ebd53e834daeaab6a" +dependencies = [ + "smallvec", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -433,6 +429,25 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -521,15 +536,6 @@ dependencies = [ "serde", ] -[[package]] -name = "encoding_rs" -version = "0.8.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" -dependencies = [ - "cfg-if", -] - [[package]] name = "equivalent" version = "1.0.2" @@ -580,6 +586,12 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + [[package]] name = "flate2" version = "1.1.9" @@ -663,21 +675,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "futures" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.31" @@ -757,7 +754,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ - "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -883,16 +879,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "hdrhistogram" -version = "7.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" -dependencies = [ - "byteorder", - "num-traits", -] - [[package]] name = "heck" version = "0.5.0" @@ -965,12 +951,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9171a2ea8a68358193d15dd5d70c1c10a2afc3e7e4c5bc92bc9f025cebd7359c" - [[package]] name = "httparse" version = "1.10.1" @@ -1053,12 +1033,9 @@ dependencies = [ "percent-encoding", "pin-project-lite", "socket2", - "system-configuration", "tokio", - "tower-layer", "tower-service", "tracing", - "windows-registry", ] [[package]] @@ -1248,6 +1225,20 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "krates" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c432bb7d7a968a338f6c7cbd8782f38ffd87a793ff06c3a20257f45e1d003f6d" +dependencies = [ + "camino", + "cfg-expr", + "petgraph", + "semver", + "serde", + "serde_json", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -1410,23 +1401,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "multer" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" -dependencies = [ - "bytes", - "encoding_rs", - "futures-util", - "http", - "httparse", - "memchr", - "mime", - "spin", - "version_check", -] - [[package]] name = "native-tls" version = "0.2.14" @@ -1705,6 +1679,17 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap", +] + [[package]] name = "pin-project" version = "1.1.10" @@ -1964,6 +1949,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1982,6 +1987,18 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -2276,6 +2293,10 @@ name = "semver" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +dependencies = [ + "serde", + "serde_core", +] [[package]] name = "serde" @@ -2446,6 +2467,18 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spdx" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afd765179d12cdd06192e02b1d8c6b0d46bd26de9bf16ce04c881428ca2cc7ed" +dependencies = [ + "regex", + "smallvec", + "unicode-normalization", + "zstd", +] + [[package]] name = "spin" version = "0.9.8" @@ -2741,27 +2774,6 @@ dependencies = [ "syn", ] -[[package]] -name = "system-configuration" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" -dependencies = [ - "bitflags", - "core-foundation 0.9.4", - "system-configuration-sys", -] - -[[package]] -name = "system-configuration-sys" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "tempfile" version = "3.24.0" @@ -2858,7 +2870,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -3010,7 +3021,6 @@ checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" dependencies = [ "futures-core", "futures-util", - "hdrhistogram", "indexmap", "pin-project-lite", "slab", @@ -3029,7 +3039,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ "async-compression", - "base64", "bitflags", "bytes", "futures-core", @@ -3037,12 +3046,7 @@ dependencies = [ "http", "http-body", "http-body-util", - "http-range-header", - "httpdate", "iri-string", - "mime", - "mime_guess", - "percent-encoding", "pin-project-lite", "tokio", "tokio-util", @@ -3489,17 +3493,6 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" -[[package]] -name = "windows-registry" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" -dependencies = [ - "windows-link", - "windows-result", - "windows-strings", -] - [[package]] name = "windows-result" version = "0.4.1" @@ -3758,6 +3751,18 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xtask" +version = "0.0.0" +dependencies = [ + "krates", + "rayon", + "serde", + "serde_json", + "spdx", + "tracing", +] + [[package]] name = "yoke" version = "0.8.1" @@ -3869,17 +3874,15 @@ checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445" [[package]] name = "zorian" -version = "0.0.0" +version = "0.1.0" dependencies = [ "async-trait", "axum", - "axum-extra", "axum-server", "bytes", "bytesize", "chrono", "crc32fast", - "futures", "hex", "http-body-util", "hyper", @@ -3913,6 +3916,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "xtask", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 179c094..daef133 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later [workspace] -members = ["crates/repos", "crates/zorian"] +members = ["crates/repos", "crates/zorian", "xtask"] resolver = "3" [workspace.package] @@ -20,19 +20,17 @@ repos = { path = "crates/repos" } # external async-trait = "0.1" -axum = { version = "0.8", features = ["http2", "macros", "multipart"] } -axum-extra = "0.12" +axum = { version = "0.8", features = ["http2", "macros"] } axum-server = { version = "0.8", features = ["tls-rustls-no-provider"] } bytes = "1.11" bytesize = { version = "2.3", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } crc32fast = "1.5" -futures = "0.3" hex = "0.4" http-body-util = "0.1" hyper = "1.8" hyper-tls = "0.6" -hyper-util = { version = "0.1", features = ["full"] } +hyper-util = { version = "0.1", features = ["client-legacy", "http1", "http2", "tokio"] } maud = { version = "0.27", features = ["axum"] } mime_guess = "2.0" rust-embed = "8.11" @@ -54,12 +52,28 @@ sqlx = { version = "0.8", features = [ ] } strum = { version = "0.27", features = ["derive"] } thiserror = "2.0" -tokio = { version = "1.49", features = ["full"] } +tokio = { version = "1.49", features = [ + "rt-multi-thread", + "macros", + "time", + "signal", + "fs", + "io-util", +] } tokio-util = "0.7" toml = "0.9" tonic = "0.14" -tower = { version = "0.5", features = ["full"] } -tower-http = { version = "0.6", features = ["full"] } +tower = { version = "0.5", features = ["util"] } +tower-http = { version = "0.6", features = [ + "trace", + "compression-full", + "limit", + "timeout", + "request-id", + "set-header", + "follow-redirect", + "util", +] } tower_governor = { version = "0.8", features = ["axum", "tracing"] } url = { version = "2.5", features = ["serde"] } uuid = { version = "1.19", features = ["v4", "v5"] } @@ -91,7 +105,5 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "json"] } sd-notify = "0.4" # dev -cargo-deny = "0.19" -cargo-llvm-cov = "0.8" proptest = "1.6" tempfile = "3.24" diff --git a/Makefile b/Makefile index a7ca28f..87b77ab 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,17 @@ # SPDX-FileCopyrightText: 2026 Nikolay Govorov # SPDX-License-Identifier: AGPL-3.0-or-later +CARGO_DENY_VERSION = ^0.19 +CARGO_LLVM_COV_VERSION = ^0.8 + all: fmt clippy test deps licenses +.PHONY: setup +setup: + cargo install \ + cargo-deny@$(CARGO_DENY_VERSION) \ + cargo-llvm-cov@$(CARGO_LLVM_COV_VERSION) + .PHONY: licenses licenses: reuse lint diff --git a/crates/zorian/Cargo.toml b/crates/zorian/Cargo.toml index fbdb676..d3ac7b9 100644 --- a/crates/zorian/Cargo.toml +++ b/crates/zorian/Cargo.toml @@ -18,13 +18,11 @@ repos.workspace = true async-trait.workspace = true axum.workspace = true -axum-extra.workspace = true axum-server.workspace = true bytes.workspace = true bytesize.workspace = true chrono.workspace = true crc32fast.workspace = true -futures.workspace = true hex.workspace = true http-body-util.workspace = true hyper.workspace = true @@ -61,5 +59,9 @@ opentelemetry_sdk.workspace = true [target.'cfg(target_os = "linux")'.dependencies] sd-notify.workspace = true +[build-dependencies] +xtask = { path = "../../xtask" } +serde_json.workspace = true + [dev-dependencies] tempfile.workspace = true diff --git a/crates/zorian/build.rs b/crates/zorian/build.rs new file mode 100644 index 0000000..36e1cd6 --- /dev/null +++ b/crates/zorian/build.rs @@ -0,0 +1,13 @@ +// SPDX-FileCopyrightText: 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +fn main() { + println!("cargo::rerun-if-changed=../../Cargo.lock"); + + let json = + xtask::licenses::generate_json("Cargo.toml").expect("failed to generate licenses JSON"); + + let out_dir = std::env::var("OUT_DIR").unwrap(); + std::fs::write(format!("{out_dir}/licenses.json"), json) + .expect("failed to write licenses.json"); +} diff --git a/crates/zorian/src/assets/base.css b/crates/zorian/src/assets/base.css index ebdd940..d2ddb6a 100644 --- a/crates/zorian/src/assets/base.css +++ b/crates/zorian/src/assets/base.css @@ -85,3 +85,22 @@ h3 a { h3 a:hover { text-decoration: underline; } + +.licenses-list { + list-style-type: none; + margin: 0; + padding: 0; +} + +.license-used-by { + margin-top: -10px; +} + +.license-text { + max-height: 280px; + overflow-y: scroll; + white-space: pre-wrap; + background-color: antiquewhite; + border-left: 4px solid black; + padding: 0.8em 1em; +} diff --git a/crates/zorian/src/controller_web.rs b/crates/zorian/src/controller_web.rs index 4ab4429..11391c8 100644 --- a/crates/zorian/src/controller_web.rs +++ b/crates/zorian/src/controller_web.rs @@ -3,7 +3,7 @@ use std::future::Future; use std::pin::Pin; -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; use std::task::{Context, Poll}; use axum::body::Body; @@ -12,6 +12,7 @@ use axum::{extract, routing}; use chrono::{DateTime, Utc}; use maud::{Markup, html}; use rust_embed::Embed; +use serde::Deserialize; use sqlx::types::chrono; use tower::{Layer, Service}; use tracing::error; @@ -24,6 +25,46 @@ struct Assets; const CSP: &str = "default-src 'self'; base-uri 'none'; img-src 'self'; font-src 'self'; style-src 'self'; script-src 'self'; object-src 'none'; frame-ancestors 'none'"; +#[derive(Deserialize)] +struct LicenseList { + overview: Vec, + licenses: Vec, +} + +#[derive(Deserialize)] +struct LicenseOverview { + id: String, + name: String, + count: usize, +} + +#[derive(Deserialize)] +struct LicenseEntry { + id: String, + name: String, + text: String, + first_of_kind: bool, + used_by: Vec, +} + +#[derive(Deserialize)] +struct LicenseUsedBy { + #[serde(rename = "crate")] + crate_: LicenseCrate, +} + +#[derive(Deserialize)] +struct LicenseCrate { + name: String, + version: String, + repository: Option, +} + +static LICENSES: LazyLock = LazyLock::new(|| { + let json = include_str!(concat!(env!("OUT_DIR"), "/licenses.json")); + serde_json::from_str(json).expect("failed to parse licenses.json") +}); + /// Handles html pages rendering and static files pub struct WebController { zig: Option>, @@ -40,6 +81,7 @@ impl WebController { pub fn router(self: Arc) -> axum::Router { let pages = axum::Router::new() .route("/", axum::routing::get(Self::index)) + .route("/licenses", axum::routing::get(Self::licenses)) .layer(tower_http::set_header::SetResponseHeaderLayer::overriding( header::CACHE_CONTROL, HeaderValue::from_static("no-cache"), @@ -133,7 +175,8 @@ impl WebController { p { "Zorian is open source software licensed under " a href="https://www.gnu.org/licenses/agpl-3.0.html" { "AGPL-3.0" } ". " - "Source code is available on " a href="https://github.com/mrdimidium/zorian" { "GitHub" } + "Source code is available on " a href="https://github.com/mrdimidium/zorian" { "GitHub" } ". " + "A list of dependency licenses " a href="/licenses" { "is available" } "." } h2 { "Usage" } @@ -269,6 +312,65 @@ impl WebController { } } } + + async fn licenses() -> Markup { + let data = &*LICENSES; + + html! { + (maud::DOCTYPE) + + html lang="en" { + head { + meta charset="UTF-8"; + meta name="viewport" content="width=device-width,initial-scale=1"; + + link rel="stylesheet" href="assets/base.css"; + title { "Third Party Licenses — Earth PKG" } + } + + body { + main { + h1 { "Third Party Licenses" } + p { "This page lists the licenses of the projects used in Zorian." } + + h2 { "Overview of licenses:" } + ul { + @for ov in &data.overview { + li { a href=(format!("#{}", ov.id)) { (&ov.name) } " (" (ov.count) ")" } + } + } + + h2 { "All license text:" } + + @for (i, lic) in data.licenses.iter().enumerate() { + div.license { + @if lic.first_of_kind { + h3 id=(&lic.id) { (&lic.name) } + } + h4 id=(format!("{}-{}", lic.id, i)) { (&lic.name) } + + h5 { "Used by:" } + ul .license-used-by { + @for usage in &lic.used_by { + li { + a href=( + usage.crate_.repository.as_deref() + .unwrap_or(&format!("https://crates.io/crates/{}", usage.crate_.name)) + ) { + (&usage.crate_.name) " " (&usage.crate_.version) + } + } + } + } + + pre .license-text { (&lic.text) } + } + } + } + } + } + } + } } #[derive(Clone)] diff --git a/deny.toml b/deny.toml index ba55068..0f7b2a5 100644 --- a/deny.toml +++ b/deny.toml @@ -24,6 +24,7 @@ exceptions = [ # The project itself is distributed under the AGPL, but avoid it in dependencies { crate = "zorian", allow = ["AGPL-3.0-or-later"] }, { crate = "repos", allow = ["AGPL-3.0-or-later"] }, + { crate = "xtask", allow = ["AGPL-3.0-or-later"] }, ] unused-allowed-license = "allow" diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 0000000..a8047fb --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,16 @@ +# SPDX-FileCopyrightText: 2026 Nikolay Govorov +# SPDX-License-Identifier: AGPL-3.0-or-later + +[package] +name = "xtask" +edition.workspace = true +publish.workspace = true +license.workspace = true + +[dependencies] +krates = "0.20" +rayon = "1.10" +tracing.workspace = true +serde.workspace = true +serde_json.workspace = true +spdx = { version = "0.13", features = ["detection-inline-cache", "text"] } diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs new file mode 100644 index 0000000..1b3440f --- /dev/null +++ b/xtask/src/lib.rs @@ -0,0 +1,4 @@ +// SPDX-FileCopyrightText: 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +pub mod licenses; diff --git a/xtask/src/licenses.rs b/xtask/src/licenses.rs new file mode 100644 index 0000000..162a090 --- /dev/null +++ b/xtask/src/licenses.rs @@ -0,0 +1,615 @@ +// SPDX-FileCopyrightText: 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// The approach to gathering licenses — resolving the dependency graph via +// krates, scanning crate sources for license texts with spdx detection, +// and deduplicating results — is based on cargo-about +// (https://github.com/EmbarkStudios/cargo-about) by Embark Studios, +// licensed under MIT OR Apache-2.0. + +use std::collections::BTreeMap; +use std::{cmp, fmt}; + +use krates::cm; +use krates::{Utf8Path, Utf8PathBuf}; +use serde::Serialize; +use spdx::detection as sd; +use spdx::{Expression, LicenseReq, Licensee}; + +const CONFIDENCE_THRESHOLD: f32 = 0.8; + +const IGNORE_PRIVATE: bool = true; +const IGNORE_DEV_DEPENDENCIES: bool = true; +const IGNORE_BUILD_DEPENDENCIES: bool = true; +const IGNORE_TRANSITIVE_DEPENDENCIES: bool = false; + +struct Crate(cm::Package); + +impl Crate { + fn get_license_expression(&self) -> LicenseInfo { + if let Some(license_field) = &self.0.license { + match Crate::parse_license_expression(license_field) { + Ok(validated) => LicenseInfo::Expr(validated), + Err(err) => { + tracing::error!("unable to parse license expression for '{self}': {err}"); + LicenseInfo::Unknown + } + } + } else { + tracing::warn!("crate '{self}' doesn't have a license field"); + LicenseInfo::Unknown + } + } + + fn parse_license_expression(license: &str) -> Result { + Expression::parse_mode( + license, + spdx::ParseMode { + allow_deprecated: true, + allow_imprecise_license_names: true, + allow_slash_as_or_operator: false, + allow_postfix_plus_on_gpl: true, + allow_unknown: false, + }, + ) + } +} + +impl Ord for Crate { + fn cmp(&self, o: &Self) -> cmp::Ordering { + match self.0.name.cmp(&o.0.name) { + cmp::Ordering::Equal => self.0.version.cmp(&o.0.version), + o => o, + } + } +} + +impl PartialOrd for Crate { + fn partial_cmp(&self, o: &Self) -> Option { + Some(self.cmp(o)) + } +} + +impl Eq for Crate {} + +impl PartialEq for Crate { + fn eq(&self, o: &Self) -> bool { + self.cmp(o) == cmp::Ordering::Equal + } +} + +impl From for Crate { + fn from(mut pkg: cm::Package) -> Self { + // Fix the license field as cargo used to allow the invalid / separator + if let Some(lf) = &mut pkg.license { + *lf = lf.replace('/', " OR "); + } + + Self(pkg) + } +} + +impl krates::KrateDetails for Crate { + fn name(&self) -> &str { + &self.0.name + } + + fn version(&self) -> &krates::semver::Version { + &self.0.version + } +} + +impl fmt::Display for Crate { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} {}", self.0.name, self.0.version) + } +} + +impl std::ops::Deref for Crate { + type Target = cm::Package; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +type Krates = krates::Krates; + +fn get_all_crates(cargo_toml: &Utf8Path) -> Result { + let mut mdc = krates::Cmd::new(); + mdc.manifest_path(cargo_toml); + + let mut builder = krates::Builder::new(); + + if IGNORE_BUILD_DEPENDENCIES { + builder.ignore_kind(krates::DepKind::Build, krates::Scope::All); + } + + if IGNORE_DEV_DEPENDENCIES { + builder.ignore_kind(krates::DepKind::Dev, krates::Scope::All); + } + + if IGNORE_TRANSITIVE_DEPENDENCIES { + builder.ignore_kind(krates::DepKind::Normal, krates::Scope::NonWorkspace); + builder.ignore_kind(krates::DepKind::Dev, krates::Scope::NonWorkspace); + builder.ignore_kind(krates::DepKind::Build, krates::Scope::NonWorkspace); + } + + builder.include_targets(std::iter::empty::<(&str, Vec)>()); + + let graph = builder.build(mdc, |filtered: cm::Package| { + tracing::debug!("filtered {} {}", filtered.name, filtered.version); + })?; + + Ok(graph) +} + +type LicenseStore = sd::Store; + +fn load_license_store() -> Result> { + Ok(sd::Store::load_inline()?) +} + +#[derive(Debug)] +#[allow(clippy::large_enum_variant)] +enum LicenseInfo { + Expr(Expression), + Unknown, +} + +enum LicenseFileKind { + /// The license file is the canonical text of the license + Text(String), + /// The file just has a license header + Header, +} + +struct LicenseFile { + license_expr: Expression, + confidence: f32, + kind: LicenseFileKind, +} + +impl Ord for LicenseFile { + fn cmp(&self, o: &Self) -> cmp::Ordering { + match self.license_expr.as_ref().cmp(o.license_expr.as_ref()) { + cmp::Ordering::Equal => o + .confidence + .partial_cmp(&self.confidence) + .expect("NaN encountered comparing license confidences"), + ord => ord, + } + } +} + +impl PartialOrd for LicenseFile { + fn partial_cmp(&self, o: &Self) -> Option { + Some(self.cmp(o)) + } +} + +impl PartialEq for LicenseFile { + fn eq(&self, o: &Self) -> bool { + self.cmp(o) == cmp::Ordering::Equal + } +} + +impl Eq for LicenseFile {} + +struct KrateLicense<'krate> { + krate: &'krate Crate, + lic_info: LicenseInfo, + license_files: Vec, +} + +fn walk_files(dir: &Utf8Path) -> Vec { + let mut files = Vec::new(); + let mut stack = vec![dir.to_path_buf()]; + + while let Some(current) = stack.pop() { + let entries = match std::fs::read_dir(¤t) { + Ok(entries) => entries, + Err(e) => { + tracing::warn!("failed to read directory '{current}': {e}"); + continue; + } + }; + + for entry in entries.filter_map(|e| e.ok()) { + let ft = match entry.file_type() { + Ok(ft) => ft, + Err(_) => continue, + }; + + let path = match Utf8PathBuf::from_path_buf(entry.path()) { + Ok(pb) => pb, + Err(e) => { + tracing::warn!("skipping path {}, not a valid utf-8 path", e.display()); + continue; + } + }; + + if ft.is_dir() { + stack.push(path); + } else if ft.is_file() { + files.push(path); + } + } + } + + files +} + +fn scan_files( + root_dir: &Utf8Path, + scanner: &sd::scan::Scanner<'_>, + threshold: f32, +) -> Vec { + walk_files(root_dir) + .into_iter() + .filter_map(|path| { + let contents = read_file(&path)?; + check_is_license_file(path, contents, scanner, threshold) + }) + .collect() +} + +fn read_file(path: &Utf8Path) -> Option { + match std::fs::read_to_string(path) { + Err(ref e) if e.kind() == std::io::ErrorKind::InvalidData => { + tracing::debug!("binary file '{path}' detected"); + None + } + Err(e) => { + tracing::error!("failed to read '{path}': {e}"); + None + } + Ok(c) => Some(c), + } +} + +fn check_is_license_file( + path: Utf8PathBuf, + contents: String, + scanner: &sd::scan::Scanner<'_>, + threshold: f32, +) -> Option { + match scan_text(&contents, scanner, threshold) { + ScanResult::Header(ided) => { + let license_expr = match Expression::parse(ided.id.name) { + Ok(expr) => expr, + Err(err) => { + tracing::error!( + "failed to parse license '{}' at {path:?} into a valid expression: {err}", + ided.id.name + ); + return None; + } + }; + + Some(LicenseFile { + license_expr, + confidence: ided.confidence, + kind: LicenseFileKind::Header, + }) + } + ScanResult::Text(ided) => { + let license_expr = match Expression::parse(ided.id.name) { + Ok(expr) => expr, + Err(err) => { + tracing::error!( + "failed to parse license '{}' at {path:?} into a valid expression: {err}", + ided.id.name + ); + return None; + } + }; + + Some(LicenseFile { + license_expr, + confidence: ided.confidence, + kind: LicenseFileKind::Text(contents), + }) + } + ScanResult::UnknownId(id_str) => { + tracing::error!("found unknown SPDX identifier '{id_str}' scanning '{path}'"); + None + } + ScanResult::LowLicenseChance(ided) => { + tracing::debug!( + "found '{}' scanning '{path}' but it only has a confidence score of {}", + ided.id.name, + ided.confidence, + ); + None + } + ScanResult::NoLicense => None, + } +} + +struct Identified { + confidence: f32, + id: spdx::LicenseId, +} + +enum ScanResult { + Header(Identified), + Text(Identified), + UnknownId(String), + LowLicenseChance(Identified), + NoLicense, +} + +fn scan_text(contents: &str, strat: &sd::scan::Scanner<'_>, threshold: f32) -> ScanResult { + let text = spdx::detection::TextData::new(contents); + let lic_match = strat.scan(&text); + + let Some(identified) = lic_match.license else { + return ScanResult::NoLicense; + }; + + let lic_id = match spdx::license_id(identified.name) { + Some(id) => Identified { + confidence: lic_match.score, + id, + }, + None => return ScanResult::UnknownId(identified.name.to_owned()), + }; + + use spdx::detection::LicenseType; + + if lic_match.score >= threshold { + match identified.kind { + LicenseType::Header => ScanResult::Header(lic_id), + LicenseType::Original => ScanResult::Text(lic_id), + LicenseType::Alternate => { + panic!("Alternate license detected") + } + } + } else { + ScanResult::LowLicenseChance(lic_id) + } +} + +fn gather_licenses<'k>(krates: &'k Krates, store: &LicenseStore) -> Vec> { + use rayon::prelude::*; + + let min_threshold = (CONFIDENCE_THRESHOLD - 0.5).max(0.1); + + let scanner = sd::scan::Scanner::new(store) + .confidence_threshold(min_threshold) + .optimize(false) + .max_passes(1); + + let mut licensed_krates: Vec<_> = krates + .krates() + .par_bridge() + .filter_map(|krate| { + // Skip private/workspace crates + if IGNORE_PRIVATE + && let Some(publish) = &krate.publish + && publish.is_empty() + { + tracing::debug!("ignoring private crate '{krate}'"); + return None; + } + + let lic_info = krate.get_license_expression(); + let root_path = krate.manifest_path.parent().unwrap(); + + let mut license_files = scan_files(root_path, &scanner, CONFIDENCE_THRESHOLD); + + // Condense each license down to the best candidate if + // multiple are found + license_files.sort(); + let mut last_expr = None; + license_files.retain(|lf| { + let dominated = last_expr.as_ref() == Some(&lf.license_expr); + last_expr = Some(lf.license_expr.clone()); + !dominated + }); + + Some(KrateLicense { + krate, + lic_info, + license_files, + }) + }) + .collect(); + + licensed_krates.sort_by(|a, b| a.krate.cmp(b.krate)); + licensed_krates +} + +/// For an OR expression like "MIT OR Apache-2.0", pick the minimal set of +/// licenses to satisfy the expression. Actual license policy validation +/// is handled by `cargo deny`. +fn pick_licenses(expr: &Expression) -> Vec { + let accepted: Vec = expr + .requirements() + .filter_map(|r| { + r.req + .license + .id() + .map(|id| Licensee::parse(id.name).unwrap()) + }) + .collect(); + + expr.minimized_requirements(&accepted).unwrap_or_default() +} + +/// For crates without a `license` field, synthesize requirements from +/// scanned license files. +fn synthesize_from_files(files: &[LicenseFile]) -> Vec { + let mut reqs = Vec::new(); + for lf in files { + for req in lf.license_expr.requirements() { + if !reqs.contains(&req.req) { + reqs.push(req.req.clone()); + } + } + } + reqs +} + +fn effective_licenses(kl: &KrateLicense<'_>) -> Vec { + match &kl.lic_info { + LicenseInfo::Expr(expr) => pick_licenses(expr), + LicenseInfo::Unknown => { + if kl.license_files.is_empty() { + tracing::warn!( + "unable to determine license for '{}': no `license` specified, and no license files were found", + kl.krate + ); + Vec::new() + } else { + synthesize_from_files(&kl.license_files) + } + } + } +} + +#[derive(Clone, Serialize)] +struct UsedBy { + #[serde(rename = "crate")] + krate: UsedByCrate, +} + +#[derive(Clone, Serialize)] +struct UsedByCrate { + name: String, + version: String, + repository: Option, +} + +#[derive(Clone, Serialize)] +struct License { + name: String, + id: String, + first_of_kind: bool, + text: String, + used_by: Vec, +} + +#[derive(Serialize)] +struct LicenseSet { + count: usize, + name: String, + id: String, +} + +#[derive(Serialize)] +struct LicenseList { + overview: Vec, + licenses: Vec, +} + +fn generate(nfos: &[KrateLicense<'_>]) -> LicenseList { + let mut licenses_map: BTreeMap> = BTreeMap::new(); + + for krate_license in nfos { + let reqs = effective_licenses(krate_license); + + for license_req in &reqs { + let spdx::LicenseItem::Spdx { id, .. } = license_req.license else { + tracing::warn!( + "{license_req} has no license file for crate '{}'", + krate_license.krate + ); + continue; + }; + + // Try to find actual license text from scanned files + let license_text = krate_license + .license_files + .iter() + .find_map(|lf| { + if !lf + .license_expr + .evaluate(|ereq| ereq.license.id() == Some(id)) + { + return None; + } + + match &lf.kind { + LicenseFileKind::Text(text) => Some(text.clone()), + LicenseFileKind::Header => None, + } + }) + .unwrap_or_else(|| { + tracing::debug!( + "unable to find text for license '{license_req}' for crate '{}', falling back to canonical text", + krate_license.krate + ); + id.text().to_owned() + }); + + let used_by = UsedBy { + krate: UsedByCrate { + name: krate_license.krate.name.clone(), + version: krate_license.krate.version.to_string(), + repository: krate_license.krate.repository.clone(), + }, + }; + + let entry = licenses_map.entry(id.full_name.to_owned()).or_default(); + + let lic = entry + .entry(license_text.clone()) + .or_insert_with(|| License { + name: id.full_name.to_owned(), + id: id.name.to_owned(), + text: license_text, + used_by: Vec::new(), + first_of_kind: false, + }); + lic.used_by.push(used_by); + } + } + + let mut licenses: Vec<_> = licenses_map + .into_iter() + .flat_map(|(_, v)| v.into_values()) + .collect(); + + for lic in &mut licenses { + lic.used_by + .sort_by(|a, b| a.krate.name.len().cmp(&b.krate.name.len())); + } + + licenses.sort_by(|a, b| a.id.cmp(&b.id)); + + let mut overview_map: BTreeMap<&str, LicenseSet> = BTreeMap::new(); + + for lic in &mut licenses { + let ls = overview_map.entry(&lic.id).or_insert_with(|| { + lic.first_of_kind = true; + LicenseSet { + count: 0, + name: lic.name.clone(), + id: lic.id.clone(), + } + }); + ls.count += lic.used_by.len(); + } + + let mut overview: Vec<_> = overview_map.into_values().collect(); + overview.sort_by(|a, b| a.name.cmp(&b.name)); + + LicenseList { overview, licenses } +} + +/// Gathers all dependency licenses and returns a JSON string. +pub fn generate_json(manifest_path: &str) -> Result> { + let manifest_path = Utf8PathBuf::from(manifest_path); + if !manifest_path.exists() { + return Err(format!("manifest path '{manifest_path}' does not exist").into()); + } + + let krates = get_all_crates(&manifest_path)?; + let store = load_license_store()?; + let summary = gather_licenses(&krates, &store); + let list = generate(&summary); + + Ok(serde_json::to_string_pretty(&list)?) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 0000000..786eafc --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +fn main() -> Result<(), Box> { + let args: Vec = std::env::args().skip(1).collect(); + + match args.first().map(|s| s.as_str()) { + Some("licenses") => { + let json = xtask::licenses::generate_json("crates/zorian/Cargo.toml")?; + println!("{json}"); + Ok(()) + } + Some(cmd) => Err(format!("unknown command: {cmd}").into()), + None => Err( + "usage: cargo xtask \n\ncommands:\n licenses Generate dependency license JSON" + .into(), + ), + } +} -- Gilti