diff options
| author | Nikolay Govorov <me@govorov.online> | 2026-08-30 00:38:30 +0100 |
|---|---|---|
| committer | Nikolay Govorov <me@govorov.online> | 2026-08-30 02:01:45 +0100 |
| commit | 9b6230016bf5ad159ba5af69bad6bdbdec8412c3 (patch) | |
| tree | a194a7dcc079279ca5354851f8dbfd816e1bd3af | |
| parent | f34b71115c9ce3c13d4f97aa2bdf90d87c88accc (diff) | |
| download | tar tar.gz tar.bz2 tar.lz tar.xz tar.zst zip | |
Unify ci checks, move licenses generator to platform
Diffstat
| -rw-r--r-- | .cargo/config.toml | 5 | +0 −5 |
| -rw-r--r-- | .github/workflows/build.yml | 19 | +16 −3 |
| -rw-r--r-- | Cargo.lock | 129 | +0 −129 |
| -rw-r--r-- | Cargo.toml | 2 | +1 −1 |
| -rw-r--r-- | crates/recluse/Cargo.toml | 4 | +0 −4 |
| -rw-r--r-- | crates/recluse/build.rs | 27 | +22 −5 |
| -rw-r--r-- | deny.toml | 1 | +0 −1 |
| -rw-r--r-- | mise.lock | 18 | +18 −0 |
| -rw-r--r-- | mise.toml | 4 | +3 −1 |
| -rw-r--r-- | pkg/scripts/preinstall.sh | 9 | +4 −5 |
| -rw-r--r-- | xtask/Cargo.toml | 16 | +0 −16 |
| -rw-r--r-- | xtask/build.rs | 9 | +0 −9 |
| -rw-r--r-- | xtask/src/lib.rs | 4 | +0 −4 |
| -rw-r--r-- | xtask/src/licenses.rs | 614 | +0 −614 |
| -rw-r--r-- | xtask/src/main.rs | 19 | +0 −19 |
15 files changed, 64 insertions, 816 deletions
diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 --- a/.cargo/config.toml +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -[alias] -xtask = "run --release --package xtask --" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8ba779..07dadb3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,6 +10,13 @@ on: pull_request: branches: [main] +permissions: + contents: read + +concurrency: + group: recluse-build-${{ github.ref }} + cancel-in-progress: true + env: CARGO_TERM_COLOR: always RELEASE_TAG: nightly @@ -30,9 +37,15 @@ jobs: run: mise bootstrap --locked --yes --update - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - - name: Check formatting + - name: formatting run: cargo fmt --all --check + - name: shellcheck + run: shellcheck pkg/scripts/*.sh tests/smoke/*.sh + + - name: clippy + run: cargo clippy --workspace --all-targets --all-features --release --locked -- -D warnings + build: name: Build (${{ matrix.arch }}) runs-on: ${{ matrix.runner }} @@ -57,9 +70,9 @@ jobs: run: mise bootstrap --locked --yes --update - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2.8.2 - - run: cargo clippy --all-targets --all-features --release -- -D warnings - - run: cargo build --release + - run: cargo clippy --workspace --locked --all-targets --all-features --release -- -D warnings - run: cargo test --all-features --release --locked + - run: cargo build --package recluse --bin recluse --release --locked - name: Build packages run: | diff --git a/Cargo.lock b/Cargo.lock index 2dda4d8..94f3647 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -294,15 +294,6 @@ dependencies = [ ] [[package]] -name = "camino" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629a66d692cb9ff1a1c664e41771b3dcaf961985a9774c0eb0bd1b51cf60a48" -dependencies = [ - "serde_core", -] - -[[package]] name = "cc" version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -315,15 +306,6 @@ dependencies = [ ] [[package]] -name = "cfg-expr" -version = "0.20.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c6b04e07d8080154ed4ac03546d9a2b303cc2fe1901ba0b35b301516e289368" -dependencies = [ - "smallvec", -] - -[[package]] name = "cfg-if" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -428,25 +410,6 @@ dependencies = [ ] [[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" -dependencies = [ - "crossbeam-utils", -] - -[[package]] name = "crossbeam-queue" version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -585,12 +548,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] -name = "fixedbitset" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" - -[[package]] name = "flate2" version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1244,20 +1201,6 @@ dependencies = [ ] [[package]] -name = "krates" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c432bb7d7a968a338f6c7cbd8782f38ffd87a793ff06c3a20257f45e1d003f6d" -dependencies = [ - "camino", - "cfg-expr", - "petgraph", - "semver", - "serde", - "serde_json", -] - -[[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1689,17 +1632,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] -name = "petgraph" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" -dependencies = [ - "fixedbitset", - "hashbrown 0.15.5", - "indexmap", -] - -[[package]] name = "pin-project" version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1981,26 +1913,6 @@ dependencies = [ ] [[package]] -name = "rayon" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] name = "recluse" version = "0.1.0" dependencies = [ @@ -2043,7 +1955,6 @@ dependencies = [ "tracing-subscriber", "url", "uuid", - "xtask", ] [[package]] @@ -2065,18 +1976,6 @@ dependencies = [ ] [[package]] -name = "regex" -version = "1.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] name = "regex-automata" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2349,10 +2248,6 @@ name = "semver" version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" -dependencies = [ - "serde", - "serde_core", -] [[package]] name = "serde" @@ -2533,18 +2428,6 @@ dependencies = [ ] [[package]] -name = "spdx" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8da593e30beb790fc9424502eb898320b44e5eb30367dbda1c1edde8e2f32d7" -dependencies = [ - "regex", - "smallvec", - "unicode-normalization", - "zstd", -] - -[[package]] name = "spin" version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3855,18 +3738,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] -name = "xtask" -version = "0.0.0" -dependencies = [ - "krates", - "rayon", - "serde", - "serde_json", - "spdx", - "tracing", -] - -[[package]] name = "yoke" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/Cargo.toml b/Cargo.toml index 475ca1f..06e3c86 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later [workspace] -members = ["crates/*", "tests/smoke", "xtask"] +members = ["crates/*", "tests/smoke"] resolver = "3" [workspace.package] diff --git a/crates/recluse/Cargo.toml b/crates/recluse/Cargo.toml index cdf8c42..c8e90db 100644 --- a/crates/recluse/Cargo.toml +++ b/crates/recluse/Cargo.toml @@ -58,9 +58,5 @@ opentelemetry_sdk.workspace = true [target.'cfg(target_os = "linux")'.dependencies] sd-notify.workspace = true -[build-dependencies] -xtask = { path = "../../xtask" } -serde_json.workspace = true - [dev-dependencies] tempfile.workspace = true diff --git a/crates/recluse/build.rs b/crates/recluse/build.rs index d5cdc6d..0f4e2d3 100644 --- a/crates/recluse/build.rs +++ b/crates/recluse/build.rs @@ -3,11 +3,28 @@ fn main() { println!("cargo::rerun-if-changed=../../Cargo.lock"); - - let json = - xtask::licenses::generate_json("Cargo.toml").expect("failed to generate licenses JSON"); + println!("cargo::rerun-if-changed=../../mise.toml"); + println!("cargo::rerun-if-changed=../../mise.lock"); let out_dir = std::env::var("OUT_DIR").unwrap(); - std::fs::write(format!("{out_dir}/licenses.json"), json) - .expect("failed to write licenses.json"); + let target = std::env::var("TARGET").unwrap(); + let output = format!("{out_dir}/licenses.json"); + + let status = std::process::Command::new("mise") + .current_dir(env!("CARGO_MANIFEST_DIR")) + .args([ + "run", + "licenses-json", + "--", + "--manifest-path", + "Cargo.toml", + "--output", + &output, + "--target", + &target, + ]) + .status() + .expect("failed to run licenses-json task"); + + assert!(status.success(), "failed to generate licenses.json"); } diff --git a/deny.toml b/deny.toml index cc39732..b0b87c8 100644 --- a/deny.toml +++ b/deny.toml @@ -22,7 +22,6 @@ allow = [ ] exceptions = [ # The project itself is distributed under the AGPL, but avoid it in dependencies - { crate = "xtask", allow = ["AGPL-3.0-or-later"] }, { crate = "base", allow = ["AGPL-3.0-or-later"] }, { crate = "repos", allow = ["AGPL-3.0-or-later"] }, { crate = "smoke", allow = ["AGPL-3.0-or-later"] }, diff --git a/mise.lock b/mise.lock index 4a2058b..0aded9e 100644 --- a/mise.lock +++ b/mise.lock @@ -14,6 +14,10 @@ checksum = "sha256:967b5cc996c29d8baa52bbb4595ef1f53af35255af8e2036ddbc6468d7b52 url = "https://github.com/taiki-e/cargo-llvm-cov/releases/download/v0.8.7/cargo-llvm-cov-x86_64-unknown-linux-musl.tar.gz" url_api = "https://api.github.com/repos/taiki-e/cargo-llvm-cov/releases/assets/418861752" +[[tools."cargo:cargo-about"]] +version = "0.8.4" +backend = "cargo:cargo-about" + [[tools.gh]] version = "2.96.0" backend = "aqua:cli/cli" @@ -53,3 +57,17 @@ backend = "core:rust" [tools.rust.options] components = "clippy,llvm-tools-preview,rustfmt" profile = "minimal" + +[[tools.shellcheck]] +version = "0.11.0" +backend = "aqua:koalaman/shellcheck" + +[tools.shellcheck."platforms.linux-arm64"] +checksum = "sha256:12b331c1d2db6b9eb13cfca64306b1b157a86eb69db83023e261eaa7e7c14588" +url = "https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.aarch64.tar.xz" +url_api = "https://api.github.com/repos/koalaman/shellcheck/releases/assets/279056934" + +[tools.shellcheck."platforms.linux-x64"] +checksum = "sha256:8c3be12b05d5c177a04c29e3c78ce89ac86f1595681cab149b65b97c4e227198" +url = "https://github.com/koalaman/shellcheck/releases/download/v0.11.0/shellcheck-v0.11.0.linux.x86_64.tar.xz" +url_api = "https://api.github.com/repos/koalaman/shellcheck/releases/assets/279056942" diff --git a/mise.toml b/mise.toml index f616adc..20e8d95 100644 --- a/mise.toml +++ b/mise.toml @@ -8,9 +8,11 @@ experimental = true [tools] "aqua:taiki-e/cargo-llvm-cov" = "0.8.7" +"cargo:cargo-about" = "0.8.4" gh = "2.96.0" jq = "1.8.2" rust = { version = "1.97.1", profile = "minimal", components = [ "clippy", "llvm-tools-preview", "rustfmt" ] } +shellcheck = "0.11.0" [bootstrap.packages] # APT @@ -34,7 +36,7 @@ rust = { version = "1.97.1", profile = "minimal", components = [ "clippy", "llvm [task_config] dir = "{{cwd}}" includes = [ - "git::https://github.com/dimidiumlabs/platform.git//tasks?ref=8bc35fe8be889c50db2d1fb4425cc1b4097dc6b8", + "git::https://github.com/dimidiumlabs/platform.git//tasks?ref=159ce265b95d655342c378d9aaa7dc14dbf1dfa4", ] [tasks.fmt] diff --git a/pkg/scripts/preinstall.sh b/pkg/scripts/preinstall.sh index 95c374d..3330200 100644 --- a/pkg/scripts/preinstall.sh +++ b/pkg/scripts/preinstall.sh @@ -4,14 +4,13 @@ set -e -PROGRAM=recluse RECLUSE_USER=${RECLUSE_USER:-recluse} RECLUSE_GROUP=${RECLUSE_GROUP:-${RECLUSE_USER}} -if ! getent group $RECLUSE_GROUP >/dev/null; then - groupadd --system $RECLUSE_GROUP +if ! getent group "$RECLUSE_GROUP" >/dev/null; then + groupadd --system "$RECLUSE_GROUP" fi -if ! getent passwd $RECLUSE_USER >/dev/null; then - useradd --system --gid $RECLUSE_GROUP --no-create-home --shell /usr/sbin/nologin $RECLUSE_USER +if ! getent passwd "$RECLUSE_USER" >/dev/null; then + useradd --system --gid "$RECLUSE_GROUP" --no-create-home --shell /usr/sbin/nologin "$RECLUSE_USER" fi diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml deleted file mode 100644 --- a/xtask/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-FileCopyrightText: 2026 Nikolay Govorov -# SPDX-License-Identifier: AGPL-3.0-or-later - -[package] -name = "xtask" -edition.workspace = true -publish.workspace = true -license.workspace = true - -[dependencies] -krates = "0.20" -rayon = "1.10" -tracing.workspace = true -serde.workspace = true -serde_json.workspace = true -spdx = { version = "0.13", features = ["detection-inline-cache", "text"] } diff --git a/xtask/build.rs b/xtask/build.rs deleted file mode 100644 --- a/xtask/build.rs +++ /dev/null @@ -1,9 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -fn main() { - println!( - "cargo:rustc-env=TARGET={}", - std::env::var("TARGET").unwrap() - ); -} diff --git a/xtask/src/lib.rs b/xtask/src/lib.rs deleted file mode 100644 --- a/xtask/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -pub mod licenses; diff --git a/xtask/src/licenses.rs b/xtask/src/licenses.rs deleted file mode 100644 --- a/xtask/src/licenses.rs +++ /dev/null @@ -1,614 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// The approach to gathering licenses — resolving the dependency graph via -// krates, scanning crate sources for license texts with spdx detection, -// and deduplicating results — is based on cargo-about -// (https://github.com/EmbarkStudios/cargo-about) by Embark Studios, -// licensed under MIT OR Apache-2.0. - -use std::collections::BTreeMap; -use std::{cmp, fmt}; - -use krates::cm; -use krates::{Utf8Path, Utf8PathBuf}; -use serde::Serialize; -use spdx::detection as sd; -use spdx::{Expression, LicenseReq, Licensee}; - -const CONFIDENCE_THRESHOLD: f32 = 0.8; - -const IGNORE_PRIVATE: bool = true; -const IGNORE_DEV_DEPENDENCIES: bool = true; -const IGNORE_BUILD_DEPENDENCIES: bool = true; -const IGNORE_TRANSITIVE_DEPENDENCIES: bool = false; - -struct Crate(cm::Package); - -impl Crate { - fn get_license_expression(&self) -> LicenseInfo { - if let Some(license_field) = &self.0.license { - match Crate::parse_license_expression(license_field) { - Ok(validated) => LicenseInfo::Expr(validated), - Err(err) => { - tracing::error!("unable to parse license expression for '{self}': {err}"); - LicenseInfo::Unknown - } - } - } else { - tracing::warn!("crate '{self}' doesn't have a license field"); - LicenseInfo::Unknown - } - } - - fn parse_license_expression(license: &str) -> Result<Expression, spdx::ParseError> { - Expression::parse_mode( - license, - spdx::ParseMode { - allow_deprecated: true, - allow_imprecise_license_names: true, - allow_slash_as_or_operator: false, - allow_postfix_plus_on_gpl: true, - allow_unknown: false, - }, - ) - } -} - -impl Ord for Crate { - fn cmp(&self, o: &Self) -> cmp::Ordering { - match self.0.name.cmp(&o.0.name) { - cmp::Ordering::Equal => self.0.version.cmp(&o.0.version), - o => o, - } - } -} - -impl PartialOrd for Crate { - fn partial_cmp(&self, o: &Self) -> Option<cmp::Ordering> { - Some(self.cmp(o)) - } -} - -impl Eq for Crate {} - -impl PartialEq for Crate { - fn eq(&self, o: &Self) -> bool { - self.cmp(o) == cmp::Ordering::Equal - } -} - -impl From<cm::Package> for Crate { - fn from(mut pkg: cm::Package) -> Self { - // Fix the license field as cargo used to allow the invalid / separator - if let Some(lf) = &mut pkg.license { - *lf = lf.replace('/', " OR "); - } - - Self(pkg) - } -} - -impl krates::KrateDetails for Crate { - fn name(&self) -> &str { - &self.0.name - } - - fn version(&self) -> &krates::semver::Version { - &self.0.version - } -} - -impl fmt::Display for Crate { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{} {}", self.0.name, self.0.version) - } -} - -impl std::ops::Deref for Crate { - type Target = cm::Package; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -type Krates = krates::Krates<Crate>; - -fn get_all_crates(cargo_toml: &Utf8Path) -> Result<Krates, krates::Error> { - let mut mdc = krates::Cmd::new(); - mdc.manifest_path(cargo_toml); - - let mut builder = krates::Builder::new(); - - if IGNORE_BUILD_DEPENDENCIES { - builder.ignore_kind(krates::DepKind::Build, krates::Scope::All); - } - - if IGNORE_DEV_DEPENDENCIES { - builder.ignore_kind(krates::DepKind::Dev, krates::Scope::All); - } - - if IGNORE_TRANSITIVE_DEPENDENCIES { - builder.ignore_kind(krates::DepKind::Normal, krates::Scope::NonWorkspace); - builder.ignore_kind(krates::DepKind::Dev, krates::Scope::NonWorkspace); - builder.ignore_kind(krates::DepKind::Build, krates::Scope::NonWorkspace); - } - - builder.include_targets(std::iter::once((env!("TARGET"), vec![]))); - - let graph = builder.build(mdc, |filtered: cm::Package| { - tracing::debug!("filtered {} {}", filtered.name, filtered.version); - })?; - - Ok(graph) -} - -type LicenseStore = sd::Store; - -fn load_license_store() -> Result<LicenseStore, Box<dyn std::error::Error>> { - Ok(sd::Store::load_inline()?) -} - -#[derive(Debug)] -#[allow(clippy::large_enum_variant)] -enum LicenseInfo { - Expr(Expression), - Unknown, -} - -enum LicenseFileKind { - /// The license file is the canonical text of the license - Text(String), - /// The file just has a license header - Header, -} - -struct LicenseFile { - license_expr: Expression, - confidence: f32, - kind: LicenseFileKind, -} - -impl Ord for LicenseFile { - fn cmp(&self, o: &Self) -> cmp::Ordering { - match self.license_expr.as_ref().cmp(o.license_expr.as_ref()) { - cmp::Ordering::Equal => o - .confidence - .partial_cmp(&self.confidence) - .expect("NaN encountered comparing license confidences"), - ord => ord, - } - } -} - -impl PartialOrd for LicenseFile { - fn partial_cmp(&self, o: &Self) -> Option<cmp::Ordering> { - Some(self.cmp(o)) - } -} - -impl PartialEq for LicenseFile { - fn eq(&self, o: &Self) -> bool { - self.cmp(o) == cmp::Ordering::Equal - } -} - -impl Eq for LicenseFile {} - -struct KrateLicense<'krate> { - krate: &'krate Crate, - lic_info: LicenseInfo, - license_files: Vec<LicenseFile>, -} - -fn walk_files(dir: &Utf8Path) -> Vec<Utf8PathBuf> { - let mut files = Vec::new(); - let mut stack = vec![dir.to_path_buf()]; - - while let Some(current) = stack.pop() { - let entries = match std::fs::read_dir(¤t) { - Ok(entries) => entries, - Err(e) => { - tracing::warn!("failed to read directory '{current}': {e}"); - continue; - } - }; - - for entry in entries.filter_map(|e| e.ok()) { - let ft = match entry.file_type() { - Ok(ft) => ft, - Err(_) => continue, - }; - - let path = match Utf8PathBuf::from_path_buf(entry.path()) { - Ok(pb) => pb, - Err(e) => { - tracing::warn!("skipping path {}, not a valid utf-8 path", e.display()); - continue; - } - }; - - if ft.is_dir() { - stack.push(path); - } else if ft.is_file() { - files.push(path); - } - } - } - - files -} - -fn scan_files( - root_dir: &Utf8Path, - scanner: &sd::scan::Scanner<'_>, - threshold: f32, -) -> Vec<LicenseFile> { - walk_files(root_dir) - .into_iter() - .filter_map(|path| { - let contents = read_file(&path)?; - check_is_license_file(path, contents, scanner, threshold) - }) - .collect() -} - -fn read_file(path: &Utf8Path) -> Option<String> { - match std::fs::read_to_string(path) { - Err(ref e) if e.kind() == std::io::ErrorKind::InvalidData => { - tracing::debug!("binary file '{path}' detected"); - None - } - Err(e) => { - tracing::error!("failed to read '{path}': {e}"); - None - } - Ok(c) => Some(c), - } -} - -fn check_is_license_file( - path: Utf8PathBuf, - contents: String, - scanner: &sd::scan::Scanner<'_>, - threshold: f32, -) -> Option<LicenseFile> { - match scan_text(&contents, scanner, threshold) { - ScanResult::Header(ided) => { - let license_expr = match Expression::parse(ided.id.name) { - Ok(expr) => expr, - Err(err) => { - tracing::error!( - "failed to parse license '{}' at {path:?} into a valid expression: {err}", - ided.id.name - ); - return None; - } - }; - - Some(LicenseFile { - license_expr, - confidence: ided.confidence, - kind: LicenseFileKind::Header, - }) - } - ScanResult::Text(ided) => { - let license_expr = match Expression::parse(ided.id.name) { - Ok(expr) => expr, - Err(err) => { - tracing::error!( - "failed to parse license '{}' at {path:?} into a valid expression: {err}", - ided.id.name - ); - return None; - } - }; - - Some(LicenseFile { - license_expr, - confidence: ided.confidence, - kind: LicenseFileKind::Text(contents), - }) - } - ScanResult::UnknownId(id_str) => { - tracing::error!("found unknown SPDX identifier '{id_str}' scanning '{path}'"); - None - } - ScanResult::LowLicenseChance(ided) => { - tracing::debug!( - "found '{}' scanning '{path}' but it only has a confidence score of {}", - ided.id.name, - ided.confidence, - ); - None - } - ScanResult::NoLicense => None, - } -} - -struct Identified { - confidence: f32, - id: spdx::LicenseId, -} - -enum ScanResult { - Header(Identified), - Text(Identified), - UnknownId(String), - LowLicenseChance(Identified), - NoLicense, -} - -fn scan_text(contents: &str, strat: &sd::scan::Scanner<'_>, threshold: f32) -> ScanResult { - let text = spdx::detection::TextData::new(contents); - let lic_match = strat.scan(&text); - - let Some(identified) = lic_match.license else { - return ScanResult::NoLicense; - }; - - let lic_id = match spdx::license_id(identified.name) { - Some(id) => Identified { - confidence: lic_match.score, - id, - }, - None => return ScanResult::UnknownId(identified.name.to_owned()), - }; - - use spdx::detection::LicenseType; - - if lic_match.score >= threshold { - match identified.kind { - LicenseType::Header => ScanResult::Header(lic_id), - LicenseType::Original => ScanResult::Text(lic_id), - LicenseType::Alternate => { - panic!("Alternate license detected") - } - } - } else { - ScanResult::LowLicenseChance(lic_id) - } -} - -fn gather_licenses<'k>(krates: &'k Krates, store: &LicenseStore) -> Vec<KrateLicense<'k>> { - use rayon::prelude::*; - - let min_threshold = (CONFIDENCE_THRESHOLD - 0.5).max(0.1); - - let scanner = sd::scan::Scanner::new(store) - .confidence_threshold(min_threshold) - .optimize(false) - .max_passes(1); - - let mut licensed_krates: Vec<_> = krates - .krates() - .par_bridge() - .filter_map(|krate| { - // Skip private/workspace crates - if IGNORE_PRIVATE - && let Some(publish) = &krate.publish - && publish.is_empty() - { - tracing::debug!("ignoring private crate '{krate}'"); - return None; - } - - let lic_info = krate.get_license_expression(); - let root_path = krate.manifest_path.parent().unwrap(); - - let mut license_files = scan_files(root_path, &scanner, CONFIDENCE_THRESHOLD); - - // Condense each license down to the best candidate if - // multiple are found - license_files.sort(); - let mut last_expr = None; - license_files.retain(|lf| { - let dominated = last_expr.as_ref() == Some(&lf.license_expr); - last_expr = Some(lf.license_expr.clone()); - !dominated - }); - - Some(KrateLicense { - krate, - lic_info, - license_files, - }) - }) - .collect(); - - licensed_krates.sort_by(|a, b| a.krate.cmp(b.krate)); - licensed_krates -} - -/// For an OR expression like "MIT OR Apache-2.0", pick the minimal set of -/// licenses to satisfy the expression. Actual license policy validation -/// is handled by `cargo deny`. -fn pick_licenses(expr: &Expression) -> Vec<LicenseReq> { - let accepted: Vec<Licensee> = expr - .requirements() - .filter_map(|r| { - r.req - .license - .id() - .map(|id| Licensee::parse(id.name).unwrap()) - }) - .collect(); - - expr.minimized_requirements(&accepted).unwrap_or_default() -} - -/// For crates without a `license` field, synthesize requirements from -/// scanned license files. -fn synthesize_from_files(files: &[LicenseFile]) -> Vec<LicenseReq> { - let mut reqs = Vec::new(); - for lf in files { - for req in lf.license_expr.requirements() { - if !reqs.contains(&req.req) { - reqs.push(req.req.clone()); - } - } - } - reqs -} - -fn effective_licenses(kl: &KrateLicense<'_>) -> Vec<LicenseReq> { - match &kl.lic_info { - LicenseInfo::Expr(expr) => pick_licenses(expr), - LicenseInfo::Unknown => { - if kl.license_files.is_empty() { - tracing::warn!( - "unable to determine license for '{}': no `license` specified, and no license files were found", - kl.krate - ); - Vec::new() - } else { - synthesize_from_files(&kl.license_files) - } - } - } -} - -#[derive(Clone, Serialize)] -struct UsedBy { - #[serde(rename = "crate")] - krate: UsedByCrate, -} - -#[derive(Clone, Serialize)] -struct UsedByCrate { - name: String, - version: String, - repository: Option<String>, -} - -#[derive(Clone, Serialize)] -struct License { - name: String, - id: String, - first_of_kind: bool, - text: String, - used_by: Vec<UsedBy>, -} - -#[derive(Serialize)] -struct LicenseSet { - count: usize, - name: String, - id: String, -} - -#[derive(Serialize)] -struct LicenseList { - overview: Vec<LicenseSet>, - licenses: Vec<License>, -} - -fn generate(nfos: &[KrateLicense<'_>]) -> LicenseList { - let mut licenses_map: BTreeMap<String, BTreeMap<String, License>> = BTreeMap::new(); - - for krate_license in nfos { - let reqs = effective_licenses(krate_license); - - for license_req in &reqs { - let spdx::LicenseItem::Spdx { id, .. } = license_req.license else { - tracing::warn!( - "{license_req} has no license file for crate '{}'", - krate_license.krate - ); - continue; - }; - - // Try to find actual license text from scanned files - let license_text = krate_license - .license_files - .iter() - .find_map(|lf| { - if !lf - .license_expr - .evaluate(|ereq| ereq.license.id() == Some(id)) - { - return None; - } - - match &lf.kind { - LicenseFileKind::Text(text) => Some(text.clone()), - LicenseFileKind::Header => None, - } - }) - .unwrap_or_else(|| { - tracing::debug!( - "unable to find text for license '{license_req}' for crate '{}', falling back to canonical text", - krate_license.krate - ); - id.text().to_owned() - }); - - let used_by = UsedBy { - krate: UsedByCrate { - name: krate_license.krate.name.clone(), - version: krate_license.krate.version.to_string(), - repository: krate_license.krate.repository.clone(), - }, - }; - - let entry = licenses_map.entry(id.full_name.to_owned()).or_default(); - - let lic = entry - .entry(license_text.clone()) - .or_insert_with(|| License { - name: id.full_name.to_owned(), - id: id.name.to_owned(), - text: license_text, - used_by: Vec::new(), - first_of_kind: false, - }); - lic.used_by.push(used_by); - } - } - - let mut licenses: Vec<_> = licenses_map - .into_values() - .flat_map(|v| v.into_values()) - .collect(); - - for lic in &mut licenses { - lic.used_by.sort_by_key(|used| used.krate.name.len()); - } - - licenses.sort_by(|a, b| a.id.cmp(&b.id)); - - let mut overview_map: BTreeMap<&str, LicenseSet> = BTreeMap::new(); - - for lic in &mut licenses { - let ls = overview_map.entry(&lic.id).or_insert_with(|| { - lic.first_of_kind = true; - LicenseSet { - count: 0, - name: lic.name.clone(), - id: lic.id.clone(), - } - }); - ls.count += lic.used_by.len(); - } - - let mut overview: Vec<_> = overview_map.into_values().collect(); - overview.sort_by(|a, b| a.name.cmp(&b.name)); - - LicenseList { overview, licenses } -} - -/// Gathers all dependency licenses and returns a JSON string. -pub fn generate_json(manifest_path: &str) -> Result<String, Box<dyn std::error::Error>> { - let manifest_path = Utf8PathBuf::from(manifest_path); - if !manifest_path.exists() { - return Err(format!("manifest path '{manifest_path}' does not exist").into()); - } - - let krates = get_all_crates(&manifest_path)?; - let store = load_license_store()?; - let summary = gather_licenses(&krates, &store); - let list = generate(&summary); - - Ok(serde_json::to_string_pretty(&list)?) -} diff --git a/xtask/src/main.rs b/xtask/src/main.rs deleted file mode 100644 --- a/xtask/src/main.rs +++ /dev/null @@ -1,19 +0,0 @@ -// SPDX-FileCopyrightText: 2026 Nikolay Govorov -// SPDX-License-Identifier: AGPL-3.0-or-later - -fn main() -> Result<(), Box<dyn std::error::Error>> { - let args: Vec<String> = std::env::args().skip(1).collect(); - - match args.first().map(|s| s.as_str()) { - Some("licenses") => { - let json = xtask::licenses::generate_json("crates/recluse/Cargo.toml")?; - println!("{json}"); - Ok(()) - } - Some(cmd) => Err(format!("unknown command: {cmd}").into()), - None => Err( - "usage: cargo xtask <command>\n\ncommands:\n licenses Generate dependency license JSON" - .into(), - ), - } -} |