From 9e455c20a929ddae9e142ee0eb1554df1faf30b1 Mon Sep 17 00:00:00 2001 From: Nikolay Govorov Date: Mon, 2 Feb 2026 16:02:33 +0000 Subject: Implement indexes for Go/Zig backend --- Cargo.lock | 97 ++++++ Cargo.toml | 7 +- src/assets/base.css | 16 +- src/backends/go.rs | 652 ++++++++++++++++++++++++++++++-------- src/backends/mod.rs | 272 +++++++++++++++- src/backends/zig.rs | 447 ++++++++++++++++++++++---- src/config.rs | 31 +- src/controller_backend.rs | 30 +- src/controller_web.rs | 140 +++++++- src/main.rs | 123 ++++++- src/proxy.rs | 5 +- src/storage.rs | 5 + src/utils.rs | 61 ++++ 13 files changed, 1621 insertions(+), 265 deletions(-) create mode 100644 src/utils.rs diff --git a/Cargo.lock b/Cargo.lock index 7017aee..9dda733 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -276,6 +276,21 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.10.0" @@ -1008,6 +1023,21 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -1087,6 +1117,7 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", "futures-io", "futures-macro", @@ -3142,6 +3173,25 @@ dependencies = [ "parking_lot", ] +[[package]] +name = "proptest" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand 0.9.2", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.14.3" @@ -3180,6 +3230,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.43" @@ -3260,6 +3316,15 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -3593,6 +3658,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.22" @@ -4772,6 +4849,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.9.0" @@ -4878,6 +4961,15 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -5554,6 +5646,7 @@ checksum = "bd8f3f50b848df28f887acb68e41201b5aea6bc8a8dacc00fb40635ff9a72fea" name = "zorian" version = "0.1.0" dependencies = [ + "async-trait", "axum", "axum-extra", "axum-server", @@ -5562,6 +5655,7 @@ dependencies = [ "cargo-deny", "chrono", "crc32fast", + "futures", "hex", "http-body-util", "hyper", @@ -5574,15 +5668,18 @@ dependencies = [ "opentelemetry-otlp", "opentelemetry-semantic-conventions", "opentelemetry_sdk", + "proptest", "rust-embed", "rustls", "sd-notify", "semver", "serde", + "serde_json", "sqlx", "tempfile", "thiserror 2.0.17", "tokio", + "tokio-util", "toml", "tonic", "tower", diff --git a/Cargo.toml b/Cargo.toml index 52168f9..17571ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ homepage = "https://pkg.earth" repository = "https://github.com/mrdimidium/Zorian" [dependencies] +async-trait = "0.1" axum = { version = "0.8", features = ["http2", "macros", "multipart"] } axum-extra = "0.12" axum-server = { version = "0.8", features = ["tls-rustls-no-provider"] } @@ -16,6 +17,7 @@ bytes = "1.11" bytesize = { version = "2.3", features = ["serde"] } chrono = { version = "0.4", features = ["serde"] } crc32fast = "1.5" +futures = "0.3" hex = "0.4" http-body-util = "0.1" hyper = "1.8" @@ -31,6 +33,7 @@ rustls = { version = "0.23", default-features = false, features = [ ] } semver = "1.0" serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" sqlx = { version = "0.8", features = [ "sqlite", "runtime-tokio", @@ -41,12 +44,13 @@ sqlx = { version = "0.8", features = [ ] } thiserror = "2.0" tokio = { version = "1.49", features = ["full"] } +tokio-util = "0.7" toml = "0.9" tonic = "0.14" tower = { version = "0.5", features = ["full"] } tower-http = { version = "0.6", features = ["full"] } tower_governor = { version = "0.8", features = ["axum", "tracing"] } -url = "2.5" +url = { version = "2.5", features = ["serde"] } uuid = { version = "1.19", features = ["v4", "v5"] } # telemetry @@ -77,4 +81,5 @@ sd-notify = { version = "0.4" } [dev-dependencies] cargo-deny = "0.19" +proptest = "1.6" tempfile = "3.24" diff --git a/src/assets/base.css b/src/assets/base.css index 134c2e3..d102d49 100644 --- a/src/assets/base.css +++ b/src/assets/base.css @@ -35,7 +35,7 @@ html { body { margin: 0 auto; - max-width: 680px; + max-width: 960px; padding: 1.5em 2em; } @@ -45,10 +45,10 @@ pre, code { } h1, h2, h3, h4, h5, h6 { + margin: 1rem 0; font-weight: 700; line-height: 1.2; text-wrap: balance; - margin: 0; } h1 { @@ -59,12 +59,20 @@ h1:first-child { margin-top: 0; } -p { +p, details { + margin: 1em 0; text-wrap: pretty; } -th { +table { + border-collapse: collapse; +} + +th, td { + padding: 7px 8px; + border: 2px solid black; text-align: start; + vertical-align: top; } h3 a { diff --git a/src/backends/go.rs b/src/backends/go.rs index df94df8..41d7a42 100644 --- a/src/backends/go.rs +++ b/src/backends/go.rs @@ -1,66 +1,39 @@ // SPDX-FileCopyrightText: 2026 Nikolay Govorov // SPDX-License-Identifier: AGPL-3.0-or-later -use serde::Deserialize; +use std::sync::Arc; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; use thiserror::Error; +use url::Url; -use super::Backend; +use super::{Archive, Backend, BackendDelegate, FileKind, IndexError, ResolveError, ResolvedFile}; +use crate::utils::deserialize_duration_secs; #[derive(Debug, Clone, Deserialize)] #[serde(default)] pub struct GoConfig { pub enabled: bool, - pub upstream: String, + pub upstream: Url, + #[serde(deserialize_with = "deserialize_duration_secs")] + pub refresh_interval: Duration, } - impl Default for GoConfig { fn default() -> Self { Self { enabled: true, - upstream: String::from("https://dl.google.com/go"), + upstream: Url::parse("https://go.dev/dl/").unwrap(), + refresh_interval: Duration::from_secs(60 * 10), } } } -pub struct GoBackend { - config: GoConfig, - source: String, -} - -impl GoBackend { - pub fn new(config: GoConfig, source: String) -> Self { - Self { config, source } - } -} - -impl Backend for GoBackend { - const ID: &'static str = "go"; - - fn upstream_url(&self, filename: &str) -> Result { - let tarball = Tarball::parse(filename).map_err(|_| ())?; - Ok(tarball.upstream_url(&self.config.upstream, &self.source)) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Archive { - TarGz, - Zip, - Msi, - Pkg, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum ReleaseType { Stable, - ReleaseCandidate(u32), - Beta(u32), -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum TarballType<'a> { - Source, - Binary { os: &'a str, arch: &'a str }, + ReleaseCandidate(u64), + Beta(u64), } #[derive(Debug, Clone, PartialEq, Eq, Error)] @@ -69,23 +42,162 @@ struct ParseError; #[derive(Debug, Clone, PartialEq, Eq)] struct GoVersion { - major: u32, - minor: u32, - patch: Option, + major: u64, + minor: u64, + patch: Option, release_type: ReleaseType, } +impl GoVersion { + /// Parses version string like "go1", "go1.22.3", "go1.23rc1", or "go1.9.2rc2". + fn parse(s: &str) -> Result { + let s = s.strip_prefix("go").ok_or(ParseError)?; + let parts: Vec<&str> = s.split('.').collect(); + let (version, consumed) = Self::from_parts(&parts)?; + if consumed != parts.len() { + return Err(ParseError); + } + Ok(version) + } + + /// Parses version from dot-separated parts, returns (version, parts_consumed). + fn from_parts(parts: &[&str]) -> Result<(Self, usize), ParseError> { + let major = parts + .first() + .ok_or(ParseError)? + .parse() + .map_err(|_| ParseError)?; + + let Some(minor_str) = parts.get(1) else { + return Ok(( + Self { + major, + minor: 0, + patch: None, + release_type: ReleaseType::Stable, + }, + 1, + )); + }; + let (minor, minor_release) = Self::parse_minor_with_release(minor_str)?; + + let Some(patch_str) = parts.get(2) else { + return Ok(( + Self { + major, + minor, + patch: None, + release_type: minor_release, + }, + 2, + )); + }; + if let Ok((p, patch_release)) = Self::parse_version_part(patch_str) { + Ok(( + Self { + major, + minor, + patch: Some(p), + release_type: patch_release.unwrap_or(minor_release), + }, + 3, + )) + } else { + Ok(( + Self { + major, + minor, + patch: None, + release_type: minor_release, + }, + 2, + )) + } + } + + fn sort_key(&self) -> i64 { + let vtype = match self.release_type { + ReleaseType::Beta(n) => super::VersionType::Beta(n), + ReleaseType::ReleaseCandidate(n) => super::VersionType::Rc(n), + ReleaseType::Stable => super::VersionType::Stable, + }; + super::stable_version(self.major, self.minor, self.patch.unwrap_or(0), vtype) + } + + /// Parses a version part (number with optional rc/beta suffix). + /// "25" -> (25, None) + /// "2rc2" -> (2, Some(ReleaseCandidate(2))) + /// "1beta1" -> (1, Some(Beta(1))) + fn parse_version_part(s: &str) -> Result<(u64, Option), ParseError> { + if let Some(idx) = s.find("rc") { + let num = s[..idx].parse::().map_err(|_| ParseError)?; + let rc_num = s[idx + 2..].parse::().map_err(|_| ParseError)?; + return Ok((num, Some(ReleaseType::ReleaseCandidate(rc_num)))); + } + if let Some(idx) = s.find("beta") { + let num = s[..idx].parse::().map_err(|_| ParseError)?; + let beta_num = s[idx + 4..].parse::().map_err(|_| ParseError)?; + return Ok((num, Some(ReleaseType::Beta(beta_num)))); + } + let num = s.parse::().map_err(|_| ParseError)?; + Ok((num, None)) + } + + /// Parses minor version with possible release type suffix. + /// "25" -> (25, Stable) + /// "26rc2" -> (26, ReleaseCandidate(2)) + /// "26beta1" -> (26, Beta(1)) + fn parse_minor_with_release(s: &str) -> Result<(u64, ReleaseType), ParseError> { + let (num, release) = GoVersion::parse_version_part(s)?; + Ok((num, release.unwrap_or(ReleaseType::Stable))) + } +} + +#[cfg(test)] +mod version_tests { + use super::*; + + #[test] + fn test_parse_go1() { + let v = GoVersion::parse("go1").unwrap(); + assert_eq!(v.major, 1); + assert_eq!(v.minor, 0); + assert_eq!(v.patch, None); + assert!(matches!(v.release_type, ReleaseType::Stable)); + } + + #[test] + fn test_parse_patch_rc() { + let v = GoVersion::parse("go1.9.2rc2").unwrap(); + assert_eq!(v.major, 1); + assert_eq!(v.minor, 9); + assert_eq!(v.patch, Some(2)); + assert!(matches!(v.release_type, ReleaseType::ReleaseCandidate(2))); + } + + #[test] + fn test_no_collision_go19_and_go192rc2() { + let v1 = GoVersion::parse("go1.9").unwrap(); + let v2 = GoVersion::parse("go1.9.2rc2").unwrap(); + assert_ne!(v1.sort_key(), v2.sort_key()); + // go1.9 (stable) should sort before go1.9.2rc2 + assert!(v1.sort_key() < v2.sort_key()); + } +} + /// Describes a single file stored at `dl.google.com/go/`. #[derive(Debug, Clone, PartialEq, Eq)] -struct Tarball<'a> { +struct GoFile<'a> { filename: &'a str, version: GoVersion, - tarball_type: TarballType<'a>, + os: Option<&'a str>, + arch: Option<&'a str>, + kind: FileKind, archive: Archive, sha256: bool, } -impl<'a> Tarball<'a> { +impl<'a> GoFile<'a> { pub fn parse(filename: &'a str) -> Result { let mut buffer = filename; let mut sha256 = false; @@ -122,104 +234,72 @@ impl<'a> Tarball<'a> { } // Split by dots: "1.25.6.linux-amd64" -> ["1", "25", "6", "linux-amd64"] - // or "1.26rc2.linux-amd64" -> ["1", "26rc2", "linux-amd64"] let parts: Vec<&str> = buffer.split('.').collect(); if parts.len() < 3 { return Err(ParseError); } - // Parse major version - let major = parts[0].parse::().map_err(|_| ParseError)?; + // Parse version, get how many parts were consumed + let (version, consumed) = GoVersion::from_parts(&parts)?; - // Parse minor version with possible release type (rc/beta) - let (minor, release_type) = parse_minor_with_release(parts[1])?; - - // Determine patch and remainder - let (patch, remainder) = if parts.len() >= 4 { - // Try to parse parts[2] as patch number - if let Ok(p) = parts[2].parse::() { - (Some(p), parts[3]) - } else { - (None, parts[2]) - } - } else { - // parts.len() == 3, no patch - (None, parts[2]) - }; - - // Determine tarball type - let tarball_type = if remainder == "src" { - TarballType::Source + // Remainder must exist and be either "src" or "os-arch" + let remainder = parts.get(consumed).ok_or(ParseError)?; + let (os, arch, kind) = if *remainder == "src" { + (None, None, FileKind::Source) } else if let Some((os, arch)) = remainder.split_once('-') { - TarballType::Binary { os, arch } + let kind = match archive { + Archive::Msi | Archive::Pkg => FileKind::Installer, + _ => FileKind::Archive, + }; + (Some(os), Some(arch), kind) } else { return Err(ParseError); }; - Ok(Tarball { + Ok(GoFile { filename, - version: GoVersion { - major, - minor, - patch, - release_type, - }, - tarball_type, + version, + os, + arch, + kind, archive, sha256, }) } /// Builds the upstream URL for this tarball. - pub fn upstream_url(&self, upstream: &str, source: &str) -> String { - format!("{}/{}?source={}", upstream, self.filename, source) + pub fn upstream_url(&self, upstream: &Url, source: &str) -> Result { + let mut url = upstream.clone(); + url.path_segments_mut() + .map_err(|_| ())? + .pop_if_empty() + .push(self.filename); + url.query_pairs_mut().append_pair("source", source); + Ok(url) } } -/// Parses minor version with possible release type suffix. -/// "25" -> (25, Stable) -/// "26rc2" -> (26, ReleaseCandidate(2)) -/// "26beta1" -> (26, Beta(1)) -fn parse_minor_with_release(s: &str) -> Result<(u32, ReleaseType), ParseError> { - if let Some(idx) = s.find("rc") { - let minor = s[..idx].parse::().map_err(|_| ParseError)?; - let rc_num = s[idx + 2..].parse::().map_err(|_| ParseError)?; - return Ok((minor, ReleaseType::ReleaseCandidate(rc_num))); - } - if let Some(idx) = s.find("beta") { - let minor = s[..idx].parse::().map_err(|_| ParseError)?; - let beta_num = s[idx + 4..].parse::().map_err(|_| ParseError)?; - return Ok((minor, ReleaseType::Beta(beta_num))); - } - let minor = s.parse::().map_err(|_| ParseError)?; - Ok((minor, ReleaseType::Stable)) -} - #[cfg(test)] -mod tests { +mod file_tests { use super::*; #[test] fn test_parse_stable_binary() { - let t = Tarball::parse("go1.25.6.linux-amd64.tar.gz").unwrap(); + let t = GoFile::parse("go1.25.6.linux-amd64.tar.gz").unwrap(); assert_eq!(t.version.major, 1); assert_eq!(t.version.minor, 25); assert_eq!(t.version.patch, Some(6)); assert!(matches!(t.version.release_type, ReleaseType::Stable)); - assert!(matches!( - t.tarball_type, - TarballType::Binary { - os: "linux", - arch: "amd64" - } - )); + assert_eq!(t.os, Some("linux")); + assert_eq!(t.arch, Some("amd64")); + assert!(matches!(t.kind, FileKind::Archive)); assert_eq!(t.archive, Archive::TarGz); assert!(!t.sha256); } #[test] fn test_parse_first_minor_release() { - let t = Tarball::parse("go1.25.linux-amd64.tar.gz").unwrap(); + let t = GoFile::parse("go1.25.linux-amd64.tar.gz").unwrap(); assert_eq!(t.version.major, 1); assert_eq!(t.version.minor, 25); assert_eq!(t.version.patch, None); @@ -228,7 +308,7 @@ mod tests { #[test] fn test_parse_rc() { - let t = Tarball::parse("go1.26rc2.linux-amd64.tar.gz").unwrap(); + let t = GoFile::parse("go1.26rc2.linux-amd64.tar.gz").unwrap(); assert_eq!(t.version.major, 1); assert_eq!(t.version.minor, 26); assert_eq!(t.version.patch, None); @@ -240,7 +320,7 @@ mod tests { #[test] fn test_parse_beta() { - let t = Tarball::parse("go1.26beta1.darwin-arm64.tar.gz").unwrap(); + let t = GoFile::parse("go1.26beta1.darwin-arm64.tar.gz").unwrap(); assert_eq!(t.version.major, 1); assert_eq!(t.version.minor, 26); assert!(matches!(t.version.release_type, ReleaseType::Beta(1))); @@ -248,39 +328,39 @@ mod tests { #[test] fn test_parse_source() { - let t = Tarball::parse("go1.25.6.src.tar.gz").unwrap(); - assert!(matches!(t.tarball_type, TarballType::Source)); + let t = GoFile::parse("go1.25.6.src.tar.gz").unwrap(); + assert!(matches!(t.kind, FileKind::Source)); + assert_eq!(t.os, None); + assert_eq!(t.arch, None); assert_eq!(t.version.patch, Some(6)); } #[test] fn test_parse_windows_zip() { - let t = Tarball::parse("go1.25.6.windows-amd64.zip").unwrap(); + let t = GoFile::parse("go1.25.6.windows-amd64.zip").unwrap(); assert_eq!(t.archive, Archive::Zip); - assert!(matches!( - t.tarball_type, - TarballType::Binary { - os: "windows", - arch: "amd64" - } - )); + assert_eq!(t.os, Some("windows")); + assert_eq!(t.arch, Some("amd64")); + assert!(matches!(t.kind, FileKind::Archive)); } #[test] fn test_parse_msi() { - let t = Tarball::parse("go1.25.6.windows-amd64.msi").unwrap(); + let t = GoFile::parse("go1.25.6.windows-amd64.msi").unwrap(); assert_eq!(t.archive, Archive::Msi); + assert!(matches!(t.kind, FileKind::Installer)); } #[test] fn test_parse_pkg() { - let t = Tarball::parse("go1.25.6.darwin-arm64.pkg").unwrap(); + let t = GoFile::parse("go1.25.6.darwin-arm64.pkg").unwrap(); assert_eq!(t.archive, Archive::Pkg); + assert!(matches!(t.kind, FileKind::Installer)); } #[test] fn test_parse_sha256() { - let t = Tarball::parse("go1.25.6.linux-amd64.tar.gz.sha256").unwrap(); + let t = GoFile::parse("go1.25.6.linux-amd64.tar.gz.sha256").unwrap(); assert!(t.sha256); assert_eq!(t.archive, Archive::TarGz); assert_eq!(t.version.patch, Some(6)); @@ -288,21 +368,333 @@ mod tests { #[test] fn test_upstream_url() { - let t = Tarball::parse("go1.25.6.linux-amd64.tar.gz").unwrap(); - let url = t.upstream_url("https://dl.google.com/go", "zorian:test"); + let t = GoFile::parse("go1.25.6.linux-amd64.tar.gz").unwrap(); + let upstream = Url::parse("https://dl.google.com/go/").unwrap(); + let url = t.upstream_url(&upstream, "zorian:test").unwrap(); assert_eq!( - url, - "https://dl.google.com/go/go1.25.6.linux-amd64.tar.gz?source=zorian:test" + url.as_str(), + "https://dl.google.com/go/go1.25.6.linux-amd64.tar.gz?source=zorian%3Atest" ); } + #[test] + fn test_parse_patch_rc() { + let t = GoFile::parse("go1.9.2rc2.linux-amd64.tar.gz").unwrap(); + assert_eq!(t.version.major, 1); + assert_eq!(t.version.minor, 9); + assert_eq!(t.version.patch, Some(2)); + assert!(matches!( + t.version.release_type, + ReleaseType::ReleaseCandidate(2) + )); + } + #[test] fn test_invalid_prefix() { - assert!(Tarball::parse("rust1.25.6.linux-amd64.tar.gz").is_err()); + assert!(GoFile::parse("rust1.25.6.linux-amd64.tar.gz").is_err()); } #[test] fn test_invalid_extension() { - assert!(Tarball::parse("go1.25.6.linux-amd64.tar.bz2").is_err()); + assert!(GoFile::parse("go1.25.6.linux-amd64.tar.bz2").is_err()); + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GoTarball { + pub filename: String, + pub os: Option, + pub arch: Option, + pub version: Option, + pub sha256: String, + pub size: u64, + pub kind: FileKind, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct GoRelease { + pub version: String, + pub stable: bool, + pub files: Vec, +} + +pub struct GoBackend { + config: GoConfig, + source: String, + delegate: Arc, +} +impl GoBackend { + pub fn new(config: GoConfig, source: String, delegate: Arc) -> Self { + Self { + config, + source, + delegate, + } + } +} +#[async_trait::async_trait] +impl Backend for GoBackend { + const ID: &'static str = "go"; + type Release = self::GoRelease; + + fn enabled(&self) -> bool { + self.config.enabled + } + + fn refresh_interval(&self) -> std::time::Duration { + self.config.refresh_interval + } + + async fn resolve_file(&self, filename: &str) -> Result { + // For .sha256 files, return hash directly from the index + if let Some(base) = filename.strip_suffix(".sha256") { + let result: Result, _> = + sqlx::query_scalar("SELECT sha256 FROM go_files WHERE filename = ?1") + .bind(base) + .fetch_optional(self.delegate.db()) + .await; + + match result { + Ok(Some(hash)) => { + return Ok(ResolvedFile::Content { + data: hash.into(), + mime: "text/plain", + }); + } + Ok(None) => return Err(ResolveError::NotFound), + Err(e) => { + tracing::error!(filename, "failed to query sha256: {e}"); + return Err(ResolveError::Internal); + } + } + } + + // Check that file exists in index before proxying to upstream + let exists: Result, _> = + sqlx::query_scalar("SELECT 1 FROM go_files WHERE filename = ?1") + .bind(filename) + .fetch_optional(self.delegate.db()) + .await; + + match exists { + Ok(None) => return Err(ResolveError::NotFound), + Err(e) => { + tracing::error!(filename, "failed to check file existence: {e}"); + return Err(ResolveError::Internal); + } + Ok(Some(_)) => {} + } + + let file = GoFile::parse(filename).map_err(|_| ResolveError::NotFound)?; + let url = file + .upstream_url(&self.config.upstream, &self.source) + .map_err(|_| ResolveError::Internal)?; + Ok(ResolvedFile::Upstream(url)) + } + + async fn migrate(&self) -> Result<(), IndexError> { + sqlx::query( + "CREATE TABLE IF NOT EXISTS go_versions ( + id INTEGER PRIMARY KEY, + version TEXT NOT NULL UNIQUE, + stable INTEGER NOT NULL + ) STRICT", + ) + .execute(self.delegate.db()) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS go_files ( + version TEXT NOT NULL, + filename TEXT NOT NULL, + os TEXT, + arch TEXT, + sha256 TEXT NOT NULL, + size INTEGER NOT NULL, + kind TEXT NOT NULL, + PRIMARY KEY (version, filename), + FOREIGN KEY (version) REFERENCES go_versions(version) + ) STRICT", + ) + .execute(self.delegate.db()) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + Ok(()) + } + + async fn fetch_index(&self) -> Result<(), IndexError> { + let mut url = self.config.upstream.clone(); + url.query_pairs_mut() + .append_pair("mode", "json") + .append_pair("include", "all"); + let bytes = self.delegate.http_get(&url).await?; + + let versions: Vec = + serde_json::from_slice(&bytes).map_err(|e| IndexError::Parse(e.to_string()))?; + + for version in versions { + if let Err(e) = self.insert_version(&version).await { + tracing::error!(version = version.version, "failed to index version: {e}"); + } + } + + Ok(()) + } + + async fn get_versions(&self) -> Result, IndexError> { + use futures::{StreamExt, TryStreamExt}; + + sqlx::query_as( + " + SELECT + v.version, v.stable, + COALESCE( + json_group_array(json_object( + 'filename', f.filename, 'os', f.os, 'arch', f.arch, + 'version', f.version, 'sha256', f.sha256, 'size', f.size, 'kind', f.kind + )) FILTER (WHERE f.filename IS NOT NULL), + '[]' + ) + FROM go_versions v + LEFT JOIN go_files f ON v.version = f.version + GROUP BY v.version + ORDER BY v.id ASC + ", + ) + .fetch(self.delegate.db()) + .map(|row| { + let (version, stable, files_json): (String, bool, String) = + row.map_err(|e| IndexError::Database(e.to_string()))?; + let files = + serde_json::from_str(&files_json).map_err(|e| IndexError::Parse(e.to_string()))?; + Ok(GoRelease { + version, + stable, + files, + }) + }) + .try_collect() + .await + } +} +impl GoBackend { + async fn insert_version(&self, version: &GoRelease) -> Result<(), IndexError> { + let id = GoVersion::parse(&version.version) + .map(|v| v.sort_key()) + .map_err(|_| IndexError::Parse(format!("invalid go version: {}", version.version)))?; + + let mut tx = self + .delegate + .db() + .begin() + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + sqlx::query( + "INSERT INTO go_versions (id, version, stable) + VALUES (?1, ?2, ?3) + ON CONFLICT(version) DO UPDATE SET + id = excluded.id, + stable = excluded.stable + WHERE id IS NOT excluded.id OR stable IS NOT excluded.stable", + ) + .bind(id) + .bind(&version.version) + .bind(version.stable) + .execute(&mut *tx) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + for file in &version.files { + Self::insert_file(&mut tx, &version.version, file).await?; + } + + tx.commit() + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + Ok(()) + } + + async fn insert_file( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + version: &str, + file: &GoTarball, + ) -> Result<(), IndexError> { + let kind = match file.kind { + FileKind::Source => "source", + FileKind::Archive => "archive", + FileKind::Installer => "installer", + FileKind::Bootstrap => unreachable!("go does not have bootstrap files"), + }; + + let os = file.os.as_deref().filter(|s| !s.is_empty()); + + let exists: Option = + sqlx::query_scalar("SELECT 1 FROM go_files WHERE version = ?1 AND filename = ?2") + .bind(version) + .bind(&file.filename) + .fetch_optional(&mut **tx) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + let changed: Option<(i32,)> = sqlx::query_as( + "INSERT INTO go_files (version, filename, os, arch, sha256, size, kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + ON CONFLICT(version, filename) DO UPDATE SET + os = excluded.os, arch = excluded.arch, sha256 = excluded.sha256, + size = excluded.size, kind = excluded.kind + WHERE os IS NOT excluded.os OR arch IS NOT excluded.arch + OR sha256 IS NOT excluded.sha256 OR size IS NOT excluded.size + OR kind IS NOT excluded.kind + RETURNING 1", + ) + .bind(version) + .bind(&file.filename) + .bind(os) + .bind(&file.arch) + .bind(&file.sha256) + .bind(file.size as i64) + .bind(kind) + .fetch_optional(&mut **tx) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + if exists.is_some() && changed.is_some() { + tracing::warn!(version, filename = file.filename, "go index file changed"); + } + + Ok(()) + } +} + +#[cfg(test)] +mod api_tests { + use super::*; + + #[test] + fn test_deserialize_go_release() { + let json = r#"{ + "version": "go1.22.0", + "stable": true, + "files": [ + { + "filename": "go1.22.0.windows-amd64.msi", + "os": "windows", + "arch": "amd64", + "version": "go1.22.0", + "sha256": "11a47de052db9971359e8c2f3a1667f8d56fa4c6bbec0687cf4cf2403a07628a", + "size": 63172608, + "kind": "installer" + } + ] + }"#; + + let release: GoRelease = serde_json::from_str(json).unwrap(); + assert_eq!(release.version, "go1.22.0"); + assert_eq!(release.files.len(), 1); + assert_eq!(release.files[0].filename, "go1.22.0.windows-amd64.msi"); + assert_eq!(release.files[0].size, 63172608); } } diff --git a/src/backends/mod.rs b/src/backends/mod.rs index 97ac397..0307314 100644 --- a/src/backends/mod.rs +++ b/src/backends/mod.rs @@ -4,14 +4,280 @@ pub mod go; pub mod zig; +use std::time::Duration; + +use async_trait::async_trait; +use bytes::Bytes; +use serde::{Deserialize, Serialize}; +use sqlx::{Pool, Sqlite}; +use thiserror::Error; +use url::Url; + pub use go::{GoBackend, GoConfig}; pub use zig::{ZigBackend, ZigConfig}; -/// Trait for backend-specific logic (parsing, URL building). +/// Error during index operations. +#[derive(Debug, Error)] +pub enum IndexError { + #[error("fetch error: {0}")] + Fetch(String), + + #[error("database error: {0}")] + Database(String), + + #[error("parse error: {0}")] + Parse(String), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Archive { + TarGz, + TarXz, + Zip, + Msi, + Pkg, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum FileKind { + Source, + Bootstrap, + Archive, + Installer, +} + +/// Error during file resolution. +#[derive(Debug, Clone, Copy)] +pub enum ResolveError { + /// File not found (404) + NotFound, + /// Internal error (500) + Internal, +} + +/// Result of resolving a file request. +pub enum ResolvedFile { + /// Proxy request to upstream URL + Upstream(Url), + /// Return content directly + Content { + data: bytes::Bytes, + mime: &'static str, + }, +} + +/// Delegate provides I/O primitives to backends. +#[async_trait] +pub trait BackendDelegate: Send + Sync { + /// Get SQLite pool + fn db(&self) -> &Pool; + + /// HTTP GET request + async fn http_get(&self, url: &Url) -> Result; +} + +/// Trait for backend-specific logic (parsing, URL building, version indexing). +#[async_trait] pub trait Backend: Send + Sync + 'static { /// Fixed unique identifier for storage const ID: &'static str; - /// Validates filename and returns the upstream URL. - fn upstream_url(&self, filename: &str) -> Result; + /// Backend-specific release representation + type Release; + + /// Whether the backend is enabled + fn enabled(&self) -> bool; + + /// Index refresh interval + fn refresh_interval(&self) -> Duration; + + /// Create tables for this backend (called at startup) + async fn migrate(&self) -> Result<(), IndexError>; + + /// Fetch index from upstream and store in DB + async fn fetch_index(&self) -> Result<(), IndexError>; + + /// Resolves filename to upstream URL or direct content. + async fn resolve_file(&self, filename: &str) -> Result; + + /// Load versions from DB + async fn get_versions(&self) -> Result, IndexError>; +} + +/// Version type for ordering. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionType { + Stable, + Rc(u64), + Beta(u64), + Dev(u64), +} + +/// Computes numeric sort key for correct version ordering. +/// +/// Formula: major × 10^12 + minor × 10^9 + patch × 10^6 + type × 10^4 + num +/// Where type: dev=10, beta=25, rc=50, stable=99 +/// +/// Examples: +/// - 1.21.3 stable → 1_021_003_990_000 +/// - 1.22.0 beta(1) → 1_022_000_250_001 +/// - 1.22.0 rc(2) → 1_022_000_500_002 +/// - 0.13.0 dev(1234) → 0_013_000_101_234 +pub fn stable_version(major: u64, minor: u64, patch: u64, vtype: VersionType) -> i64 { + let (type_val, num) = match vtype { + VersionType::Stable => (99, 0), + VersionType::Rc(n) => (50, n), + VersionType::Beta(n) => (25, n), + VersionType::Dev(n) => (10, n), + }; + + (major as i64) * 1_000_000_000_000 + + (minor as i64) * 1_000_000_000 + + (patch as i64) * 1_000_000 + + (type_val as i64) * 10_000 + + (num as i64) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashSet; + + use proptest::prelude::*; + + #[test] + fn test_stable_version_ordering() { + // dev < beta < rc < stable + let dev = stable_version(1, 0, 0, VersionType::Dev(1)); + let beta = stable_version(1, 0, 0, VersionType::Beta(1)); + let rc = stable_version(1, 0, 0, VersionType::Rc(1)); + let stable = stable_version(1, 0, 0, VersionType::Stable); + + assert!(dev < beta); + assert!(beta < rc); + assert!(rc < stable); + } + + #[test] + fn test_stable_version_patch_ordering() { + let v1_0_0 = stable_version(1, 0, 0, VersionType::Stable); + let v1_0_1 = stable_version(1, 0, 1, VersionType::Stable); + let v1_1_0 = stable_version(1, 1, 0, VersionType::Stable); + let v2_0_0 = stable_version(2, 0, 0, VersionType::Stable); + + assert!(v1_0_0 < v1_0_1); + assert!(v1_0_1 < v1_1_0); + assert!(v1_1_0 < v2_0_0); + } + + #[test] + fn test_stable_version_rc_before_stable() { + // 1.22.0-rc1 < 1.22.0-rc2 < 1.22.0 + let rc1 = stable_version(1, 22, 0, VersionType::Rc(1)); + let rc2 = stable_version(1, 22, 0, VersionType::Rc(2)); + let stable = stable_version(1, 22, 0, VersionType::Stable); + + assert!(rc1 < rc2); + assert!(rc2 < stable); + } + + #[test] + fn test_stable_version_no_collisions() { + let mut keys = HashSet::new(); + + // Test a range of realistic versions + for major in 0..3 { + for minor in 0..30 { + for patch in 0..20 { + let key = stable_version(major, minor, patch, VersionType::Stable); + assert!(keys.insert(key), "collision at {major}.{minor}.{patch}"); + } + // Also test pre-release versions + for n in 1..10 { + let rc = stable_version(major, minor, 0, VersionType::Rc(n)); + assert!(keys.insert(rc), "collision at {major}.{minor}.0-rc{n}"); + + let beta = stable_version(major, minor, 0, VersionType::Beta(n)); + assert!(keys.insert(beta), "collision at {major}.{minor}.0-beta{n}"); + + let dev = stable_version(major, minor, 0, VersionType::Dev(n)); + assert!(keys.insert(dev), "collision at {major}.{minor}.0-dev{n}"); + } + } + } + } + + #[test] + fn test_stable_version_expected_values() { + // Verify the documented examples + assert_eq!( + stable_version(1, 21, 3, VersionType::Stable), + 1_021_003_990_000 + ); + assert_eq!( + stable_version(1, 22, 0, VersionType::Beta(1)), + 1_022_000_250_001 + ); + assert_eq!( + stable_version(1, 22, 0, VersionType::Rc(2)), + 1_022_000_500_002 + ); + assert_eq!( + stable_version(0, 13, 0, VersionType::Dev(1234)), + 13_000_101_234 + ); + } + + fn version_type_strategy() -> impl Strategy { + prop_oneof![ + Just(VersionType::Stable), + (1..100u64).prop_map(VersionType::Rc), + (1..100u64).prop_map(VersionType::Beta), + (1..9999u64).prop_map(VersionType::Dev), + ] + } + + fn to_semver(major: u64, minor: u64, patch: u64, vtype: VersionType) -> semver::Version { + let pre = match vtype { + VersionType::Stable => semver::Prerelease::EMPTY, + VersionType::Rc(n) => semver::Prerelease::new(&format!("rc.{n}")).unwrap(), + VersionType::Beta(n) => semver::Prerelease::new(&format!("beta.{n}")).unwrap(), + VersionType::Dev(n) => semver::Prerelease::new(&format!("dev.{n}")).unwrap(), + }; + semver::Version { + major, + minor, + patch, + pre, + build: semver::BuildMetadata::EMPTY, + } + } + + proptest! { + #[test] + fn fuzz_stable_version_matches_semver( + major1 in 0..100u64, + minor1 in 0..1000u64, + patch1 in 0..1000u64, + vtype1 in version_type_strategy(), + major2 in 0..100u64, + minor2 in 0..1000u64, + patch2 in 0..1000u64, + vtype2 in version_type_strategy(), + ) { + let key1 = stable_version(major1, minor1, patch1, vtype1); + let key2 = stable_version(major2, minor2, patch2, vtype2); + + let sem1 = to_semver(major1, minor1, patch1, vtype1); + let sem2 = to_semver(major2, minor2, patch2, vtype2); + + let key_cmp = key1.cmp(&key2); + let sem_cmp = sem1.cmp(&sem2); + + prop_assert_eq!(key_cmp, sem_cmp, + "Mismatch: {} vs {}: stable_version gives {:?}, semver gives {:?}", + sem1, sem2, key_cmp, sem_cmp); + } + } } diff --git a/src/backends/zig.rs b/src/backends/zig.rs index 424fd34..8ffcde3 100644 --- a/src/backends/zig.rs +++ b/src/backends/zig.rs @@ -1,61 +1,73 @@ // SPDX-FileCopyrightText: 2026 Nikolay Govorov // SPDX-License-Identifier: AGPL-3.0-or-later -use semver::Version; -use serde::Deserialize; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::Duration; + +use semver::Version as SemVersion; +use serde::{Deserialize, Serialize}; use thiserror::Error; +use url::Url; -use super::Backend; +use super::{ + Archive, Backend, BackendDelegate, FileKind, IndexError, ResolveError, ResolvedFile, + VersionType, stable_version, +}; +use crate::utils::{deserialize_duration_secs, deserialize_size}; #[derive(Debug, Clone, Deserialize)] #[serde(default)] pub struct ZigConfig { pub enabled: bool, - pub upstream: String, + pub upstream: Url, + #[serde(deserialize_with = "deserialize_duration_secs")] + pub refresh_interval: Duration, } - impl Default for ZigConfig { fn default() -> Self { Self { enabled: true, - upstream: String::from("https://ziglang.org"), + upstream: Url::parse("https://ziglang.org").unwrap(), + refresh_interval: Duration::from_secs(60 * 10), } } } -pub struct ZigBackend { - config: ZigConfig, - source: String, +/// Wrapper for sort key computation. +enum ZigVersion { + Master, + Semver(SemVersion), } - -impl ZigBackend { - pub fn new(config: ZigConfig, source: String) -> Self { - Self { config, source } +impl ZigVersion { + fn parse(s: &str) -> Result { + if s == "master" { + return Ok(Self::Master); + } + SemVersion::parse(s) + .map(Self::Semver) + .map_err(|_| IndexError::Parse(format!("invalid zig version: {s}"))) } -} -impl Backend for ZigBackend { - const ID: &'static str = "zig"; + fn sort_key(&self) -> i64 { + match self { + Self::Master => i64::MAX, + Self::Semver(v) => { + let vtype = if v.pre.is_empty() { + VersionType::Stable + } else if v.pre.as_str().starts_with("dev.") { + let num = v.pre.as_str()[4..].parse().unwrap_or(0); + VersionType::Dev(num) + } else { + VersionType::Stable + }; - fn upstream_url(&self, filename: &str) -> Result { - let tarball = Tarball::parse(filename).map_err(|_| ())?; - Ok(tarball.upstream_url(&self.config.upstream, &self.source)) + stable_version(v.major, v.minor, v.patch, vtype) + } + } } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Archive { - Zip, - TarXz, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum TarballType<'a> { - Source, - Bootstrap, - Binary { os: &'a str, arch: &'a str }, -} - #[derive(Debug, Clone, PartialEq, Eq, Error)] #[error("invalid tarball filename")] struct ParseError; @@ -66,21 +78,22 @@ struct ParseError; /// we standardize the files, but for the reverse operation /// (getting a string from a tarball), we preserve the original path. #[derive(Debug, Clone, PartialEq, Eq)] -struct Tarball<'a> { +struct ZigFile<'a> { filename: &'a str, - tarball_type: TarballType<'a>, + os: Option<&'a str>, + arch: Option<&'a str>, + kind: FileKind, minisig: bool, archive: Archive, - version: Version, + version: SemVersion, development: bool, } -impl<'a> Tarball<'a> { +impl<'a> ZigFile<'a> { pub fn parse(filename: &'a str) -> Result { let mut buffer = filename; let mut minisig = false; let archive; - let tarball_type; // (?:|-bootstrap|-[a-zA-Z0-9_]+-[a-zA-Z0-9_]+)-( // \d+\.\d+\.\d+(?:-dev\.\d+\+[0-9a-f]+)? @@ -118,43 +131,40 @@ impl<'a> Tarball<'a> { let development = last.starts_with("dev"); let version = if !development { - Version::parse(last).map_err(|_| ParseError)? + SemVersion::parse(last).map_err(|_| ParseError)? } else { let semver = it.next().ok_or(ParseError)?; let devver = last; let version_str = format!("{}-{}", semver, devver); - Version::parse(&version_str).map_err(|_| ParseError)? + SemVersion::parse(&version_str).map_err(|_| ParseError)? }; - if let Some(payload) = it.next() { + let (os, arch, kind) = if let Some(payload) = it.next() { if payload == "bootstrap" { - tarball_type = TarballType::Bootstrap; + (None, None, FileKind::Bootstrap) } else { // Version 0.14.0 is the last one to use the OS-ARCH format in names; newer versions use ARCH-OS. - let min_version = Version::new(0, 14, 0); - if version > min_version { - tarball_type = TarballType::Binary { - os: payload, - arch: it.next().ok_or(ParseError)?, - }; + let min_version = SemVersion::new(0, 14, 0); + let (os, arch) = if version > min_version { + (payload, it.next().ok_or(ParseError)?) } else { - tarball_type = TarballType::Binary { - arch: payload, - os: it.next().ok_or(ParseError)?, - }; - } + (it.next().ok_or(ParseError)?, payload) + }; + (Some(os), Some(arch), FileKind::Archive) } } else { - tarball_type = TarballType::Source; - } + (None, None, FileKind::Source) + }; if it.next().is_some() { return Err(ParseError); } - Ok(Tarball { + Ok(ZigFile { filename, - tarball_type, + os, + arch, + kind, minisig, archive, version, @@ -163,14 +173,323 @@ impl<'a> Tarball<'a> { } /// Builds the upstream URL for this tarball. - pub fn upstream_url(&self, upstream: &str, source: &str) -> String { - if self.development { - format!("{}/builds/{}?source={}", upstream, self.filename, source) - } else { - format!( - "{}/download/{}/{}?source={}", - upstream, self.version, self.filename, source, - ) + pub fn upstream_url(&self, upstream: &Url, source: &str) -> Result { + let mut url = upstream.clone(); + { + let mut segments = url.path_segments_mut().map_err(|_| ())?; + segments.pop_if_empty(); + if self.development { + segments.push("builds"); + } else { + segments.push("download").push(&self.version.to_string()); + } + segments.push(self.filename); + } + url.query_pairs_mut().append_pair("source", source); + Ok(url) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ZigTarball { + /// e.g. "zig-x86_64-linux-0.15.2.tar.xz" + /// Note: upstream API returns full URL in "tarball" field, we extract filename when storing + #[serde(alias = "tarball")] + pub filename: String, + + /// e.g. "02aa270f183da276e5b5920b1dac44a63f1a49e55050ebde3aecc9eb82f93239" + pub shasum: String, + + /// e.g. 53733924 + #[serde(deserialize_with = "deserialize_size")] + pub size: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ZigRelease { + /// e.g. "0.15.2" (older releases don't have this field) + #[serde(default)] + pub version: String, + + /// e.g. "2025-10-11" + pub date: Option, + + /// e.g. "https://ziglang.org/documentation/0.15.2/" + pub docs: Option, + + /// e.g. "https://ziglang.org/documentation/0.15.2/std/" + #[serde(rename = "stdDocs")] + pub std_docs: Option, + + /// e.g. "https://ziglang.org/download/0.15.2/release-notes.html" + pub notes: Option, + + /// Source tarball + pub src: Option, + + /// Bootstrap tarball + pub bootstrap: Option, + + /// Platform-specific files (e.g., "x86_64-linux", "aarch64-macos") + #[serde(flatten)] + pub targets: HashMap, +} + +pub struct ZigBackend { + config: ZigConfig, + source: String, + delegate: Arc, +} +impl ZigBackend { + pub fn new(config: ZigConfig, source: String, delegate: Arc) -> Self { + Self { + config, + source, + delegate, } } } +#[async_trait::async_trait] +impl Backend for ZigBackend { + const ID: &'static str = "zig"; + type Release = self::ZigRelease; + + fn enabled(&self) -> bool { + self.config.enabled + } + + fn refresh_interval(&self) -> std::time::Duration { + self.config.refresh_interval + } + + async fn resolve_file(&self, filename: &str) -> Result { + let file = ZigFile::parse(filename).map_err(|_| ResolveError::NotFound)?; + + // For stable builds, check that file exists in index + if !file.development { + let exists: Result, _> = + sqlx::query_scalar("SELECT 1 FROM zig_files WHERE filename = ?1") + .bind(filename) + .fetch_optional(self.delegate.db()) + .await; + + match exists { + Ok(None) => return Err(ResolveError::NotFound), + Err(e) => { + tracing::error!(filename, "failed to check file existence: {e}"); + return Err(ResolveError::Internal); + } + Ok(Some(_)) => {} + } + } + + let url = file + .upstream_url(&self.config.upstream, &self.source) + .map_err(|_| ResolveError::Internal)?; + Ok(ResolvedFile::Upstream(url)) + } + + async fn migrate(&self) -> Result<(), IndexError> { + sqlx::query( + "CREATE TABLE IF NOT EXISTS zig_versions ( + id INTEGER PRIMARY KEY, + version TEXT NOT NULL UNIQUE, + date TEXT, + docs TEXT, + std_docs TEXT, + notes TEXT + ) STRICT", + ) + .execute(self.delegate.db()) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + sqlx::query( + "CREATE TABLE IF NOT EXISTS zig_files ( + version TEXT NOT NULL, + target TEXT NOT NULL, + filename TEXT NOT NULL, + shasum TEXT NOT NULL, + size INTEGER NOT NULL, + PRIMARY KEY (version, target), + FOREIGN KEY (version) REFERENCES zig_versions(version) + ) STRICT", + ) + .execute(self.delegate.db()) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + Ok(()) + } + + async fn fetch_index(&self) -> Result<(), IndexError> { + let mut url = self.config.upstream.clone(); + url.path_segments_mut() + .map_err(|_| IndexError::Parse("cannot-be-a-base URL".into()))? + .pop_if_empty() + .extend(["download", "index.json"]); + let bytes = self.delegate.http_get(&url).await?; + + let index: HashMap = + serde_json::from_slice(&bytes).map_err(|e| IndexError::Parse(e.to_string()))?; + + for (version_str, version) in index { + if let Err(e) = self.insert_version(&version_str, &version).await { + tracing::error!(version = version_str, "failed to index version: {e}"); + } + } + + Ok(()) + } + + async fn get_versions(&self) -> Result, IndexError> { + use futures::{StreamExt, TryStreamExt}; + + #[derive(Deserialize)] + struct FileRow { + target: String, + filename: String, + shasum: String, + size: u64, + } + + sqlx::query_as(" + SELECT + v.version, v.date, v.docs, v.std_docs, v.notes, + COALESCE( + json_group_array(json_object('target', f.target, 'filename', f.filename, 'shasum', f.shasum, 'size', f.size)) FILTER (WHERE f.version IS NOT NULL), + '[]' + ) + FROM zig_versions v + LEFT JOIN zig_files f ON v.version = f.version + GROUP BY v.version + ORDER BY v.id ASC + ") + .fetch(self.delegate.db()) + .map(|row| { + let (version, date, docs, std_docs, notes, files_json): + (String, Option, Option, Option, Option, String) = + row.map_err(|e| IndexError::Database(e.to_string()))?; + + let file_rows: Vec = serde_json::from_str(&files_json) + .map_err(|e| IndexError::Parse(e.to_string()))?; + + let mut src = None; + let mut bootstrap = None; + let mut targets = HashMap::new(); + + for f in file_rows { + let file = ZigTarball { filename: f.filename, shasum: f.shasum, size: f.size }; + match f.target.as_str() { + "src" => src = Some(file), + "bootstrap" => bootstrap = Some(file), + _ => { targets.insert(f.target, file); } + } + } + + Ok(ZigRelease { version, date, docs, std_docs, notes, src, bootstrap, targets }) + }) + .try_collect() + .await + } +} +impl ZigBackend { + async fn insert_version( + &self, + version_str: &str, + version: &ZigRelease, + ) -> Result<(), IndexError> { + let id = ZigVersion::parse(version_str)?.sort_key(); + + let mut tx = self + .delegate + .db() + .begin() + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + sqlx::query( + "INSERT INTO zig_versions (id, version, date, docs, std_docs, notes) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(version) DO UPDATE SET + id = excluded.id, + date = excluded.date, docs = excluded.docs, + std_docs = excluded.std_docs, notes = excluded.notes + WHERE id IS NOT excluded.id OR date IS NOT excluded.date + OR docs IS NOT excluded.docs OR std_docs IS NOT excluded.std_docs + OR notes IS NOT excluded.notes", + ) + .bind(id) + .bind(version_str) + .bind(&version.date) + .bind(&version.docs) + .bind(&version.std_docs) + .bind(&version.notes) + .execute(&mut *tx) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + if let Some(ref file) = version.src { + Self::insert_file(&mut tx, version_str, "src", file).await?; + } + if let Some(ref file) = version.bootstrap { + Self::insert_file(&mut tx, version_str, "bootstrap", file).await?; + } + for (target, file) in &version.targets { + Self::insert_file(&mut tx, version_str, target, file).await?; + } + + tx.commit() + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + Ok(()) + } + + async fn insert_file( + tx: &mut sqlx::Transaction<'_, sqlx::Sqlite>, + version: &str, + target: &str, + file: &ZigTarball, + ) -> Result<(), IndexError> { + let url = Url::parse(&file.filename) + .map_err(|e| IndexError::Parse(format!("invalid tarball URL: {e}")))?; + let filename = url + .path_segments() + .and_then(|mut s| s.next_back()) + .filter(|s| !s.is_empty()) + .ok_or_else(|| IndexError::Parse(format!("no filename in URL: {}", file.filename)))?; + + let exists: Option = + sqlx::query_scalar("SELECT 1 FROM zig_files WHERE version = ?1 AND target = ?2") + .bind(version) + .bind(target) + .fetch_optional(&mut **tx) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + let changed: Option<(i32,)> = sqlx::query_as( + "INSERT INTO zig_files (version, target, filename, shasum, size) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(version, target) DO UPDATE SET + filename = excluded.filename, shasum = excluded.shasum, size = excluded.size + WHERE filename IS NOT excluded.filename + OR shasum IS NOT excluded.shasum + OR size IS NOT excluded.size + RETURNING 1", + ) + .bind(version) + .bind(target) + .bind(filename) + .bind(&file.shasum) + .bind(file.size as i64) + .fetch_optional(&mut **tx) + .await + .map_err(|e| IndexError::Database(e.to_string()))?; + + if exists.is_some() && changed.is_some() { + tracing::warn!(version, target, "zig index file changed"); + } + + Ok(()) + } +} diff --git a/src/config.rs b/src/config.rs index 0fce8be..c7437aa 100644 --- a/src/config.rs +++ b/src/config.rs @@ -12,33 +12,7 @@ use serde::Deserialize; use thiserror::Error; use crate::backends::{GoConfig, ZigConfig}; - -fn deserialize_duration_secs<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - let secs = u64::deserialize(deserializer)?; - Ok(Duration::from_secs(secs)) -} - -fn deserialize_listener_addr<'de, D>(deserializer: D) -> Result -where - D: serde::Deserializer<'de>, -{ - let raw = String::deserialize(deserializer)?; - let raw = raw.trim(); - - // Hostnames are not supported, but localhost shorthands are useful - if let Some(port) = raw.strip_prefix("localhost:") { - return port - .parse::() - .map(|port| SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), port)) - .map_err(|err| serde::de::Error::custom(format!("invalid port in '{raw}': {err}"))); - } - - raw.parse::() - .map_err(|err| serde::de::Error::custom(format!("invalid address '{raw}': {err}",))) -} +use crate::utils::{deserialize_duration_secs, deserialize_listener_addr}; #[derive(Debug, Error)] pub enum ConfigError { @@ -96,7 +70,6 @@ pub struct ServerConfig { /// Rate limit: burst size (max requests allowed in a burst) per client IP pub rate_limit_burst_size: u32, } - impl Default for ServerConfig { fn default() -> Self { Self { @@ -124,7 +97,6 @@ pub struct ListenerConfig { /// Path to TLS private key file (PEM format). If set, tls_crt must also be set. pub tls_key: Option, } - impl Default for ListenerConfig { fn default() -> Self { Self { @@ -217,7 +189,6 @@ pub struct OtelcolConfig { /// HTTP headers for authentication pub headers: HashMap, } - impl Default for OtelcolConfig { fn default() -> Self { Self { diff --git a/src/controller_backend.rs b/src/controller_backend.rs index ace9cdc..dfbfeb1 100644 --- a/src/controller_backend.rs +++ b/src/controller_backend.rs @@ -6,20 +6,20 @@ use std::sync::Arc; use axum::{Router, body, extract, http, response, routing}; use tracing::error; -use crate::backends::Backend; +use crate::backends::{Backend, ResolveError, ResolvedFile}; use crate::proxy; use crate::storage; /// Generic controller for backend HTTP handling. pub struct BackendController { - backend: B, + backend: Arc, storage: Arc, upstream: Arc, } impl BackendController { pub fn new( - backend: B, + backend: Arc, storage: Arc, upstream: Arc, ) -> Self { @@ -40,12 +40,19 @@ impl BackendController { extract::State(controller): extract::State>, extract::Path(filename): extract::Path, ) -> Result { - let url = match controller.backend.upstream_url(&filename) { - Ok(url) => url, - Err(()) => { - error!(backend = B::ID, filename, "invalid filename"); + let url = match controller.backend.resolve_file(&filename).await { + Ok(ResolvedFile::Content { data, mime }) => { + return Ok(Self::build_response_with_mime(data, mime)); + } + Ok(ResolvedFile::Upstream(url)) => url, + Err(ResolveError::NotFound) => { + error!(backend = B::ID, filename, "file not found"); return Err(http::StatusCode::NOT_FOUND); } + Err(ResolveError::Internal) => { + error!(backend = B::ID, filename, "internal error resolving file"); + return Err(http::StatusCode::INTERNAL_SERVER_ERROR); + } }; match controller.storage.get(B::ID, &filename).await { @@ -92,4 +99,13 @@ impl BackendController { .body(body::Body::from(bytes)) .unwrap() } + + fn build_response_with_mime(bytes: bytes::Bytes, mime: &'static str) -> response::Response { + response::Response::builder() + .status(http::StatusCode::OK) + .header(http::header::CONTENT_TYPE, mime) + .header(http::header::CONTENT_LENGTH, bytes.len()) + .body(body::Body::from(bytes)) + .unwrap() + } } diff --git a/src/controller_web.rs b/src/controller_web.rs index 072a6d2..877e243 100644 --- a/src/controller_web.rs +++ b/src/controller_web.rs @@ -10,12 +10,14 @@ use axum::body::Body; use axum::http::{HeaderValue, Method, Request, Response, StatusCode, header}; use axum::{extract, routing}; use chrono::{DateTime, Utc}; -use maud::html; +use maud::{Markup, html}; use rust_embed::Embed; use sqlx::types::chrono; use tower::{Layer, Service}; use tracing::error; +use crate::backends::{Backend, GoBackend, ZigBackend}; + #[derive(Embed)] #[folder = "src/assets/"] struct Assets; @@ -23,8 +25,16 @@ struct Assets; const CSP: &str = "default-src 'self'; base-uri 'none'; img-src 'self'; font-src 'self'; style-src 'self'; script-src 'self'; object-src 'none'; frame-ancestors 'none'"; /// Handles html pages rendering and static files -#[derive(Default)] -pub struct WebController {} +pub struct WebController { + zig: Option>, + go: Option>, +} + +impl WebController { + pub fn new(zig: Option>, go: Option>) -> Self { + Self { zig, go } + } +} impl WebController { pub fn router(self: Arc) -> axum::Router { @@ -33,7 +43,8 @@ impl WebController { .layer(tower_http::set_header::SetResponseHeaderLayer::overriding( header::CACHE_CONTROL, HeaderValue::from_static("no-cache"), - )); + )) + .with_state(self.clone()); let assets = axum::Router::new().route("/assets/{*path}", routing::get(Self::assets)); @@ -45,7 +56,6 @@ impl WebController { header::CONTENT_SECURITY_POLICY, HeaderValue::from_static(CSP), )) - .with_state(self) } async fn assets(extract::Path(path): extract::Path) -> Response { @@ -75,7 +85,31 @@ impl WebController { } } - async fn index() -> maud::Markup { + async fn index(extract::State(ctrl): extract::State>) -> Markup { + let zig_versions = if let Some(ref backend) = ctrl.zig { + match backend.get_versions().await { + Ok(v) => v, + Err(e) => { + error!("failed to get zig versions: {e}"); + Vec::new() + } + } + } else { + Vec::new() + }; + + let go_versions = if let Some(ref backend) = ctrl.go { + match backend.get_versions().await { + Ok(v) => v, + Err(e) => { + error!("failed to get go versions: {e}"); + Vec::new() + } + } + } else { + Vec::new() + }; + html! { (maud::DOCTYPE) @@ -113,6 +147,60 @@ impl WebController { a href="#zig" { "Zig" } } + @if !zig_versions.is_empty() { + details { + summary { "Available versions (" (zig_versions.len()) ")" } + + p { "You can take actual minisig public key at " a href="https://ziglang.org/download/" { "ziglang.org/download" } "." } + table { + thead { + tr { + th { "Version" } + th { "Date" } + th { "Docs" } + th { "Targets" } + } + } + tbody { + @for v in zig_versions.iter().rev() { + tr { + td { (v.version) } + td { (v.date.as_deref().unwrap_or("-")) } + td { + @if let Some(ref url) = v.docs { + a href=(url) { "docs" } + } + " " + @if let Some(ref url) = v.std_docs { + a href=(url) { "std" } + } + " " + @if let Some(ref url) = v.notes { + a href=(url) { "notes" } + } + } + td { + @if let Some(ref src) = v.src { + a href=(format!("/zig/{}", src.filename)) { code { "src" } } + " " + } + @if let Some(ref bootstrap) = v.bootstrap { + a href=(format!("/zig/{}", bootstrap.filename)) { code { "bootstrap" } } + " " + } + @for (target, tarball) in v.targets.iter() { + a href=(format!("/zig/{}", tarball.filename)) { code { (target) } } + " " + } + } + } + } + } + } + } + } + + p { "Read more about community mirrors in the " a href="https://ziglang.org/download/community-mirrors/" { "blog post" } ". " "Information on how to deploy your own mirror is available " a href="https://github.com/ziglang/www.ziglang.org/blob/main/MIRRORS.md" { "in the documentation" } "." @@ -132,21 +220,49 @@ impl WebController { li { "unpack archive:" br; code { "tar -xf 'zig-x86_64-linux-0.15.1.tar.xz'" } ";" } li { "check installed zig:" br; code { "./zig-x86_64-linux-0.15.1/zig --version" } ";" } } - "You can take actual minisig public key at " a href="https://ziglang.org/download/" { "ziglang.org/download" } "." } h3 id="go" { a href="#go" { "Go" } } - p { - "To install manually:" - ol { + @if !go_versions.is_empty() { + details { + summary { "Available versions (" (go_versions.len()) ")" } + + p { "You can find available versions at " a href="https://go.dev/dl/" { "go.dev/dl" } "." } + table { + thead { + tr { + th { "Version" } + th { "Stable" } + th { "Files" } + } + } + tbody { + @for v in go_versions.iter().rev() { + tr { + td { (v.version) } + td { @if v.stable { "✓" } @else { "" } } + td { + @for file in &v.files { + a href=(format!("/go/{}", file.filename)) { code { (file.filename) } } + " " + } + } + } + } + } + } + } + } + + p { "To install manually:" } + + ol { li { "download go dist file:" br; code { "wget https://pkg.earth/go/go1.23.0.linux-amd64.tar.gz" } ";" } li { "download go sha256 file:" br; code { "wget https://pkg.earth/go/go1.23.0.linux-amd64.tar.gz.sha256" } ";" } li { "check archive integrity:" br; code { "sha256sum -c go1.23.0.linux-amd64.tar.gz.sha256" } ";" } li { "unpack archive:" br; code { "tar -xzf go1.23.0.linux-amd64.tar.gz" } ";" } li { "check installed go:" br; code { "./go/bin/go version" } ";" } - } - "You can find available versions at " a href="https://go.dev/dl/" { "go.dev/dl" } "." } h2 { "Privacy policy" } diff --git a/src/main.rs b/src/main.rs index 955c4f4..bc364db 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,6 +6,7 @@ mod config; mod proxy; mod storage; mod telemetry; +mod utils; mod controller_backend; mod controller_web; @@ -30,10 +31,82 @@ use tokio::signal; use tracing::{error, info, trace}; use tracing_subscriber::registry::LookupSpan; -use crate::backends::{GoBackend, ZigBackend}; +use crate::backends::{Backend, BackendDelegate, GoBackend, IndexError, ZigBackend}; use crate::controller_backend::BackendController; use crate::controller_web::WebController; +/// Implementation of BackendDelegate for the application. +struct AppDelegate { + proxy: Arc, + storage: Arc, +} + +#[async_trait::async_trait] +impl BackendDelegate for AppDelegate { + async fn http_get(&self, url: &url::Url) -> Result { + self.proxy + .fetch(proxy::DownloadRequest { url: url.clone() }) + .await + .map(|f| f.bytes) + .map_err(|e| IndexError::Fetch(e.to_string())) + } + + fn db(&self) -> &sqlx::Pool { + self.storage.db() + } +} + +async fn init_backend( + backend: B, + index_tasks: &mut tokio::task::JoinSet<()>, + index_cancel: tokio_util::sync::CancellationToken, +) -> Option> { + if !backend.enabled() { + return None; + } + let backend = Arc::new(backend); + if let Err(e) = backend.migrate().await { + error!(backend = B::ID, "migration failed: {e}"); + std::process::exit(1); + } + let interval = backend.refresh_interval(); + if !interval.is_zero() { + index_tasks.spawn(run_index_refresh( + B::ID, + backend.clone(), + interval, + index_cancel, + )); + } + Some(backend) +} + +async fn run_index_refresh( + name: &'static str, + backend: Arc, + interval: std::time::Duration, + cancel: tokio_util::sync::CancellationToken, +) { + let mut ticker = tokio::time::interval(interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + loop { + tokio::select! { + _ = cancel.cancelled() => { + info!(backend = name, "index refresh stopped"); + break; + } + _ = ticker.tick() => { + info!(backend = name, "refreshing index"); + match backend.fetch_index().await { + Ok(()) => info!(backend = name, "index refreshed"), + Err(e) => error!(backend = name, "index refresh failed: {e}"), + } + } + } + } +} + const VERSION: &str = env!("CARGO_PKG_VERSION"); const HELP: &str = "\ Usage: zorian [--config=] @@ -121,10 +194,14 @@ async fn main() { let storage = Arc::new(storage::StorageService::new(config.clone()).await.unwrap()); let upstream = Arc::new(proxy::ProxyService::new()); - let web_controller = Arc::new(WebController::default()); let source = format!("zorian:{}", config.appname()); let backends = config.backends(); + let delegate: Arc = Arc::new(AppDelegate { + proxy: upstream.clone(), + storage: storage.clone(), + }); + const REQUEST_ID_HEADER: http::HeaderName = http::HeaderName::from_static("x-request-id"); let trace_layer = tower_http::trace::TraceLayer::new_for_http() @@ -219,22 +296,38 @@ async fn main() { .finish() .unwrap(); + let mut index_tasks = tokio::task::JoinSet::new(); + let index_cancel = tokio_util::sync::CancellationToken::new(); + + let zig_backend = init_backend( + ZigBackend::new(backends.zig.clone(), source.clone(), delegate.clone()), + &mut index_tasks, + index_cancel.clone(), + ) + .await; + + let go_backend = init_backend( + GoBackend::new(backends.go.clone(), source.clone(), delegate.clone()), + &mut index_tasks, + index_cancel.clone(), + ) + .await; + + let web_controller = Arc::new(WebController::new(zig_backend.clone(), go_backend.clone())); let mut app = axum::Router::new().merge(web_controller.router()); - if backends.zig.enabled { - let backend = ZigBackend::new(backends.zig.clone(), source.clone()); + if let Some(ref backend) = zig_backend { let ctrl = Arc::new(BackendController::new( - backend, + backend.clone(), storage.clone(), upstream.clone(), )); app = app.nest("/zig", ctrl.router()); } - if backends.go.enabled { - let backend = GoBackend::new(backends.go.clone(), source.clone()); + if let Some(ref backend) = go_backend { let ctrl = Arc::new(BackendController::new( - backend, + backend.clone(), storage.clone(), upstream.clone(), )); @@ -327,7 +420,7 @@ async fn main() { ); } - let mut watchdog_ticker = tokio::time::interval(std::time::Duration::from_mins(1)); + let mut watchdog_ticker = tokio::time::interval(std::time::Duration::from_secs(60)); #[cfg(target_os = "linux")] if sd_notify::booted().unwrap_or(false) { @@ -395,25 +488,31 @@ async fn main() { #[cfg(target_os = "linux")] sd_notify::notify(false, &[NotifyState::Stopping]).ok(); + index_cancel.cancel(); handle.graceful_shutdown(None); - // Wait for all listeners to finish with timeout + // Wait for all tasks to finish with timeout let shutdown_result = tokio::time::timeout(config.server().shutdown_timeout, async { while let Some(result) = tasks.join_next().await { if let Err(e) = result { error!("listener task failed: {e}"); } } + while let Some(result) = index_tasks.join_next().await { + if let Err(e) = result { + error!("index task failed: {e}"); + } + } }) .await; if shutdown_result.is_err() { error!( - "shutdown timeout after {:?}, aborting {} remaining tasks", + "shutdown timeout after {:?}, aborting remaining tasks", config.server().shutdown_timeout, - tasks.len() ); tasks.abort_all(); + index_tasks.abort_all(); } else { info!("shutdown complete"); } diff --git a/src/proxy.rs b/src/proxy.rs index 444ffaa..e627323 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -7,10 +7,11 @@ use hyper::{Request, http}; use hyper_tls::HttpsConnector; use hyper_util::client::legacy::{Client, connect::HttpConnector}; use hyper_util::rt::TokioExecutor; +use url::Url; #[derive(Clone)] pub struct DownloadRequest { - pub url: String, + pub url: Url, } #[derive(Clone)] @@ -32,7 +33,7 @@ impl ProxyService { pub async fn fetch(&self, request: DownloadRequest) -> Result { let request = Request::builder() .method(http::Method::GET) - .uri(&request.url) + .uri(request.url.as_str()) .header(http::header::USER_AGENT, "zorian/0.1") .body(Empty::::new()) .unwrap(); diff --git a/src/storage.rs b/src/storage.rs index 9117d23..ae5b056 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -259,6 +259,11 @@ impl StorageService { Ok(storage) } + /// Get SQLite pool for use by backends + pub fn db(&self) -> &Pool { + &self.sqlite + } + /// Synchronously traverses the tree and removes temporary files. /// Must run before the application starts. async fn doctor(&self) -> Result<(), StorageError> { diff --git a/src/utils.rs b/src/utils.rs new file mode 100644 index 0000000..955a492 --- /dev/null +++ b/src/utils.rs @@ -0,0 +1,61 @@ +// SPDX-FileCopyrightText: 2026 Nikolay Govorov +// SPDX-License-Identifier: AGPL-3.0-or-later + +use std::net::SocketAddr; +use std::time::Duration; + +use serde::de::{self, Deserialize, Visitor}; + +/// Deserializes a duration from seconds (u64). +pub fn deserialize_duration_secs<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let secs = u64::deserialize(deserializer)?; + Ok(Duration::from_secs(secs)) +} + +/// Deserializes a u64 from either a number or a string. +pub fn deserialize_size<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + struct SizeVisitor; + impl<'de> Visitor<'de> for SizeVisitor { + type Value = u64; + + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a number or string") + } + + fn visit_u64(self, v: u64) -> Result { + Ok(v) + } + + fn visit_str(self, v: &str) -> Result { + v.parse().map_err(de::Error::custom) + } + } + + deserializer.deserialize_any(SizeVisitor) +} + +/// Deserializes a SocketAddr from a string. +pub fn deserialize_listener_addr<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let raw = String::deserialize(deserializer)?; + let raw = raw.trim(); + + // Hostnames are not supported, but localhost shorthands are useful + if let Some(port) = raw.strip_prefix("localhost:") { + return port + .parse::() + .map(|port| SocketAddr::new(std::net::Ipv4Addr::LOCALHOST.into(), port)) + .map_err(|err| serde::de::Error::custom(format!("invalid port in '{raw}': {err}"))); + } + + raw.parse::() + .map_err(|err| serde::de::Error::custom(format!("invalid address '{raw}': {err}",))) +} -- Gilti