diff options
Diffstat
| -rw-r--r-- | Cargo.lock | 171 | +171 −0 |
| -rw-r--r-- | Cargo.toml | 2 | +2 −0 |
| -rw-r--r-- | src/config.rs | 11 | +11 −0 |
| -rw-r--r-- | src/main.rs | 2 | +1 −1 |
| -rw-r--r-- | src/storage.rs | 315 | +315 −0 |
5 files changed, 500 insertions, 1 deletions
diff --git a/Cargo.lock b/Cargo.lock index e9ea97e..e3c104a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -342,6 +342,18 @@ dependencies = [ ] [[package]] +name = "cargo-config2" +version = "0.1.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3795d3a48839a46854805f56c8fe9c558f10804bcf57df53925ca843d87c788f" +dependencies = [ + "serde", + "serde_derive", + "toml", + "windows-sys 0.61.2", +] + +[[package]] name = "cargo-deny" version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -386,6 +398,33 @@ dependencies = [ ] [[package]] +name = "cargo-llvm-cov" +version = "0.6.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a19af1cb5512737613b5caa313dd14cd56459732867c53a755da4b967ffefe6" +dependencies = [ + "anyhow", + "camino", + "cargo-config2", + "duct", + "fs-err", + "glob", + "lcov2cobertura", + "lexopt", + "opener", + "regex", + "rustc-demangle", + "ruzstd", + "serde", + "serde_derive", + "serde_json", + "shell-escape", + "tar", + "termcolor", + "walkdir", +] + +[[package]] name = "cargo-lock" version = "11.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -752,6 +791,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" [[package]] +name = "duct" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e66e9c0c03d094e1a0ba1be130b849034aa80c3a2ab8ee94316bc809f3fa684" +dependencies = [ + "libc", + "os_pipe", + "shared_child", + "shared_thread", +] + +[[package]] name = "dunce" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1806,6 +1857,12 @@ dependencies = [ ] [[package]] +name = "glob" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" + +[[package]] name = "globset" version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2357,6 +2414,24 @@ dependencies = [ ] [[package]] +name = "lcov2cobertura" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaa0cf456e88a45378a5737f228c0800175d94be6856908dc4718b3a91c7c9f8" +dependencies = [ + "anyhow", + "quick-xml", + "regex", + "rustc-demangle", +] + +[[package]] +name = "lexopt" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" + +[[package]] name = "libc" version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2549,6 +2624,15 @@ dependencies = [ ] [[package]] +name = "normpath" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf23ab2b905654b4cb177e30b629937b3868311d4e1cba859f899c041046e69b" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] name = "nu-ansi-term" version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2622,6 +2706,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] +name = "opener" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9024962ab91e00c89d2a14352a8d0fc1a64346bf96f1839b45c09149564e47" +dependencies = [ + "bstr", + "normpath", + "windows-sys 0.60.2", +] + +[[package]] name = "openssl" version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2672,6 +2767,16 @@ dependencies = [ ] [[package]] +name = "os_pipe" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] name = "parking" version = "2.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2838,6 +2943,15 @@ dependencies = [ ] [[package]] +name = "quick-xml" +version = "0.37.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "331e97a1af0bf59823e6eadffe373d7b27f485be8748f71471c662c1f269b7fb" +dependencies = [ + "memchr", +] + +[[package]] name = "quote" version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3029,6 +3143,12 @@ dependencies = [ ] [[package]] +name = "rustc-demangle" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" + +[[package]] name = "rustc-stable-hash" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3144,6 +3264,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] +name = "ruzstd" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ff0cc5e135c8870a775d3320910cd9b564ec036b4dc0b8741629020be63f01" + +[[package]] name = "ryu" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3362,6 +3488,28 @@ dependencies = [ ] [[package]] +name = "shared_child" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e362d9935bc50f019969e2f9ecd66786612daae13e8f277be7bfb66e8bed3f7" +dependencies = [ + "libc", + "windows-sys 0.60.2", +] + +[[package]] +name = "shared_thread" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52b86057fcb5423f5018e331ac04623e32d6b5ce85e33300f92c79a1973928b0" + +[[package]] +name = "shell-escape" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" + +[[package]] name = "shell-words" version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3799,6 +3947,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + +[[package]] name = "target-lexicon" version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4846,6 +5005,16 @@ dependencies = [ ] [[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] name = "yoke" version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4968,6 +5137,7 @@ dependencies = [ "axum-extra", "bytes", "cargo-deny", + "cargo-llvm-cov", "chrono", "crc32fast", "hex", @@ -4979,6 +5149,7 @@ dependencies = [ "semver", "serde", "sqlx", + "tempfile", "thiserror 2.0.17", "tokio", "toml", diff --git a/Cargo.toml b/Cargo.toml index 85f596c..6a541e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,3 +34,5 @@ uuid = { version = "1.19.0", features = ["v4","v5"] } [dev-dependencies] cargo-deny = "0.19.0" +cargo-llvm-cov = "0.6.23" +tempfile = "3" diff --git a/src/config.rs b/src/config.rs index dd55bfc..4be4026 100644 --- a/src/config.rs +++ b/src/config.rs @@ -91,3 +91,14 @@ impl ConfigService { &self.dirname } } + +#[cfg(test)] +impl ConfigService { + pub fn for_test(dirname: PathBuf) -> Self { + Self { + listen: "127.0.0.1:0".to_string(), + appname: "test".to_string(), + dirname, + } + } +} diff --git a/src/main.rs b/src/main.rs index d2317c8..c8f593a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -255,7 +255,7 @@ impl WebController { response.headers_mut().insert( http::header::CONTENT_SECURITY_POLICY, http::HeaderValue::from_static( - "img-src 'self'; base-uri 'none'; font-src 'self'; style-src 'self'; script-src 'self'; object-src 'none'; default-src 'self'; frame-ancestors 'none'", + "default-src 'self'; base-uri 'none'; img-src 'self'; font-src 'self'; style-src 'self'; script-src 'self'; object-src 'none'; frame-ancestors 'none'", ), ); diff --git a/src/storage.rs b/src/storage.rs index 960afc5..9117d23 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -495,3 +495,318 @@ impl StorageService { result } } + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + const BIG_FILE_SIZE: usize = INLINE_THRESHOLD * 64; + + async fn create_storage_at(path: &Path) -> StorageService { + let config = sync::Arc::new(ConfigService::for_test(path.to_path_buf())); + StorageService::new(config).await.unwrap() + } + + async fn create_test_storage() -> (StorageService, TempDir) { + let tmp = TempDir::new().unwrap(); + let storage = create_storage_at(tmp.path()).await; + (storage, tmp) + } + + /// Count files in a directory recursively + fn count_files(dir: &Path, ext: Option<&str>) -> usize { + let mut count = 0; + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + count += count_files(&path, ext); + } else if let Some(extension) = ext { + if path.extension().is_some_and(|ext| ext == extension) { + count += 1; + } + } else { + count += 1; + } + } + } + count + } + + /// Recursively delete or corrupt all files in a directory + fn damage_blobs(dir: &Path, corrupt: bool) { + if let Ok(entries) = std::fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + damage_blobs(&path, corrupt); + } else if corrupt { + if let Ok(meta) = path.metadata() { + let _ = std::fs::write(&path, vec![0x00; meta.len() as usize]); + } + } else { + let _ = std::fs::remove_file(&path); + } + } + } + } + + #[tokio::test] + async fn test_write_read_small_file() { + let (storage, _tmp) = create_test_storage().await; + + // 1 KB file - should be stored inline + let data = Bytes::from(vec![0xAB; 1024]); + storage.put("test-scope", "small.bin", &data).await.unwrap(); + + let file = storage.get("test-scope", "small.bin").await.unwrap(); + assert!(file.is_some()); + + let file = file.unwrap(); + assert!(file.inlined); + assert_eq!(file.file_bytes.0, data); + assert_eq!(file.file_size, 1024); + } + + #[tokio::test] + async fn test_write_read_large_file() { + let (storage, tmp) = create_test_storage().await; + + // Large file - should be stored on disk + let data = Bytes::from(vec![0xCD; BIG_FILE_SIZE]); + storage.put("test-scope", "large.bin", &data).await.unwrap(); + + let file = storage.get("test-scope", "large.bin").await.unwrap(); + assert!(file.is_some()); + + let file = file.unwrap(); + assert!(!file.inlined); + assert_eq!(file.file_bytes.0, data); + assert_eq!(file.file_size, (BIG_FILE_SIZE) as i64); + + // Verify file is stored on disk, not inline in SQLite + let db_size = std::fs::metadata(tmp.path().join("index.sqlite")) + .unwrap() + .len(); + assert!( + db_size < data.len() as u64, + "database ({db_size} bytes) should be smaller than file ({} bytes)", + data.len() + ); + } + + #[tokio::test] + async fn test_write_read_boundary_file() { + let (storage, _tmp) = create_test_storage().await; + + // Exactly 256 KB - should be stored inline (threshold is <=) + let data = Bytes::from(vec![0xEF; INLINE_THRESHOLD]); + storage + .put("test-scope", "boundary.bin", &data) + .await + .unwrap(); + + let file = storage.get("test-scope", "boundary.bin").await.unwrap(); + assert!(file.is_some()); + + let file = file.unwrap(); + assert!(file.inlined); + assert_eq!(file.file_bytes.0, data); + assert_eq!(file.file_size, INLINE_THRESHOLD as i64); + } + + #[tokio::test] + async fn test_read_nonexistent_file() { + let (storage, _tmp) = create_test_storage().await; + + let file = storage.get("test-scope", "nonexistent.bin").await.unwrap(); + assert!(file.is_none()); + } + + #[tokio::test] + async fn test_persistence_after_restart() { + let tmp = TempDir::new().unwrap(); + + let small_data = Bytes::from(vec![0x11; 1024]); + let large_data = Bytes::from(vec![0x22; BIG_FILE_SIZE]); + + // First "session": write files + { + let storage = create_storage_at(tmp.path()).await; + storage + .put("persist", "small.bin", &small_data) + .await + .unwrap(); + storage + .put("persist", "large.bin", &large_data) + .await + .unwrap(); + // storage is dropped here, simulating shutdown + } + + // Second "session": verify files persist + { + let storage = create_storage_at(tmp.path()).await; + + let small = storage.get("persist", "small.bin").await.unwrap(); + assert!(small.is_some()); + assert_eq!(small.unwrap().file_bytes.0, small_data); + + let large = storage.get("persist", "large.bin").await.unwrap(); + assert!(large.is_some()); + assert_eq!(large.unwrap().file_bytes.0, large_data); + } + } + + #[tokio::test] + async fn test_error_blob_not_found() { + let (storage, tmp) = create_test_storage().await; + + // Write a large file (stored on disk) + let data = Bytes::from(vec![0xAA; BIG_FILE_SIZE]); + storage + .put("test-scope", "to-delete.bin", &data) + .await + .unwrap(); + + // Delete all blob files from disk + damage_blobs(&tmp.path().join("objects"), false); + + // Reading should fail with BlobNotFound + let result = storage.get("test-scope", "to-delete.bin").await; + assert!(matches!(result, Err(StorageError::BlobNotFound(_)))); + } + + #[tokio::test] + async fn test_error_blob_corrupted() { + let (storage, tmp) = create_test_storage().await; + + // Write a large file (stored on disk) + let data = Bytes::from(vec![0xBB; BIG_FILE_SIZE]); + storage + .put("test-scope", "to-corrupt.bin", &data) + .await + .unwrap(); + + // Corrupt all blob files on disk + damage_blobs(&tmp.path().join("objects"), true); + + // Reading should fail with IntegrityError + let result = storage.get("test-scope", "to-corrupt.bin").await; + assert!(matches!(result, Err(StorageError::IntegrityError))); + } + + #[tokio::test] + async fn test_duplicate_put_same_content() { + let (storage, tmp) = create_test_storage().await; + + // Write a large file + let data = Bytes::from(vec![0xCC; BIG_FILE_SIZE]); + storage.put("test-scope", "dup.bin", &data).await.unwrap(); + + // Write the same file again with identical content - should succeed + let result = storage.put("test-scope", "dup.bin", &data).await; + assert!(result.is_ok()); + + // Verify no temp files left behind + assert_eq!(count_files(&tmp.path().join("objects"), Some("part")), 0); + + // Verify file is still readable with correct content + let file = storage.get("test-scope", "dup.bin").await.unwrap(); + assert!(file.is_some()); + assert_eq!(file.unwrap().file_bytes.0, data); + } + + #[tokio::test] + async fn test_duplicate_put_different_content() { + let (storage, tmp) = create_test_storage().await; + + // Write a large file + let data1 = Bytes::from(vec![0xDD; BIG_FILE_SIZE]); + storage.put("test-scope", "dup.bin", &data1).await.unwrap(); + + // Write the same filename with different content - should fail + let data2 = Bytes::from(vec![0xEE; BIG_FILE_SIZE]); + let result = storage.put("test-scope", "dup.bin", &data2).await; + assert!(matches!(result, Err(StorageError::AlreadyExists(_, _)))); + + // Verify no temp files left behind + assert_eq!(count_files(&tmp.path().join("objects"), Some("part")), 0); + + // Verify original file is still intact + let file = storage.get("test-scope", "dup.bin").await.unwrap(); + assert!(file.is_some()); + assert_eq!(file.unwrap().file_bytes.0, data1); + } + + #[tokio::test] + async fn test_doctor_removes_temp_files() { + let tmp = TempDir::new().unwrap(); + + // First session: write a large file + let data = Bytes::from(vec![0xAA; BIG_FILE_SIZE]); + { + let storage = create_storage_at(tmp.path()).await; + storage.put("test-scope", "valid.bin", &data).await.unwrap(); + } + + // Manually create temp files in the objects directory + let objects_dir = tmp.path().join("objects"); + std::fs::create_dir_all(objects_dir.join("ab/cd")).unwrap(); + std::fs::write(objects_dir.join("ab/cd/test.part"), b"temp1").unwrap(); + std::fs::write(objects_dir.join("ab/cd/another.12345.part"), b"temp2").unwrap(); + + assert_eq!(count_files(&objects_dir, Some("part")), 2); + + // Second session: doctor should clean up temp files + { + let storage = create_storage_at(tmp.path()).await; + + // Temp files should be gone + assert_eq!(count_files(&objects_dir, Some("part")), 0); + + // Valid file should still be readable + let file = storage.get("test-scope", "valid.bin").await.unwrap(); + assert!(file.is_some()); + assert_eq!(file.unwrap().file_bytes.0, data); + } + } + + #[tokio::test] + async fn test_doctor_removes_orphan_blobs() { + let tmp = TempDir::new().unwrap(); + + // First session: write a large file + let data = Bytes::from(vec![0xBB; BIG_FILE_SIZE]); + { + let storage = create_storage_at(tmp.path()).await; + storage.put("test-scope", "valid.bin", &data).await.unwrap(); + } + + // Count files before adding orphan + let objects_dir = tmp.path().join("objects"); + let files_before = count_files(&objects_dir, None); + assert_eq!(files_before, 1); // Only the valid blob + + // Manually create an orphan blob (valid hex name but no DB record) + std::fs::create_dir_all(objects_dir.join("de/ad")).unwrap(); + let orphan_name = "deadbeefdeadbeefdeadbeefdeadbeef"; // 32 hex chars + std::fs::write(objects_dir.join("de/ad").join(orphan_name), b"orphan").unwrap(); + + assert_eq!(count_files(&objects_dir, None), 2); + + // Second session: doctor should clean up orphan blob + { + let storage = create_storage_at(tmp.path()).await; + + // Only the valid file should remain + assert_eq!(count_files(&objects_dir, None), 1); + + // Valid file should still be readable + let file = storage.get("test-scope", "valid.bin").await.unwrap(); + assert!(file.is_some()); + assert_eq!(file.unwrap().file_bytes.0, data); + } + } +} |