aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat (limited to 'src/storage.rs')
-rw-r--r--src/storage.rs812+812 −0
1 files changed, 812 insertions, 0 deletions
diff --git a/src/storage.rs b/src/storage.rs
new file mode 100644
--- /dev/null
+++ b/src/storage.rs
@@ -0,0 +1,812 @@
+// SPDX-FileCopyrightText: 2026 Nikolay Govorov <me@govorov.online>
+// SPDX-License-Identifier: AGPL-3.0-or-later
+
+// This class stores uploaded files and associated metadata.
+// Files are immutable but can be deleted.
+//
+// A unique file is defined by a `scope` and a `filename`.
+// The filename can be any valid UTF-8 string up to 1KB
+// in size and does not have to be a valid filesystem name.
+//
+// The list of files and their metadata are stored in a single
+// SQLite database table. Small files are stored in a BLOB column
+// in SQLite; for larger files, a checksum is stored in the database,
+// and the file itself is stored on disk.
+//
+// On disk new files are written in two steps:
+// - the file is written to a temporary file in the same directory;
+// - a transaction is opened and a new file entry is created
+// - temp file is renamed to the final filename
+// - the transaction is committed.
+//
+// This scheme guarantees that if a record exists in the table, the file is written.
+// However if the server crashes after rename but before commit, an orphan file will remain on the disk.
+// To combat this, when the server starts, we check that there is an entry
+// in the database for each file on the disk, and we also delete all temporary (non-renamed) files.
+
+use std::path::{Path, PathBuf};
+use std::sync;
+
+use bytes::Bytes;
+use crc32fast::Hasher as Crc32Hasher;
+use sqlx::encode::{Encode, IsNull};
+use sqlx::error::BoxDynError;
+use sqlx::{FromRow, Pool, Sqlite, query, query_as, query_scalar, sqlite};
+use thiserror::Error;
+use tokio::fs;
+use tokio::io::AsyncWriteExt;
+use tracing::{debug, instrument, warn};
+use uuid::Uuid;
+
+use super::config::ConfigService;
+
+const SQLITE_POOL_SIZE: u32 = 16;
+const INLINE_THRESHOLD: usize = 256 * 1024; // 256 KB
+
+// You cannot change this ID, this will desynchronize the records in the database and the files on the disk.
+const UUID_ROOT_NAMESPACE: Uuid = Uuid::from_bytes([
+ 0x8b, 0x06, 0x3c, 0x4c, 0x6b, 0x5c, 0x4a, 0x8b, 0x92, 0x8f, 0x75, 0x8b, 0x0e, 0x63, 0xc3, 0x5d,
+]);
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub struct Id(pub Uuid);
+impl std::ops::Deref for Id {
+ type Target = Uuid;
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+impl sqlx::Type<Sqlite> for Id {
+ fn type_info() -> sqlite::SqliteTypeInfo {
+ <Vec<u8> as sqlx::Type<Sqlite>>::type_info()
+ }
+
+ fn compatible(ty: &sqlite::SqliteTypeInfo) -> bool {
+ <Vec<u8> as sqlx::Type<Sqlite>>::compatible(ty)
+ }
+}
+impl<'r> sqlx::decode::Decode<'r, Sqlite> for Id {
+ fn decode(
+ value: sqlite::SqliteValueRef<'r>,
+ ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+ let value: Vec<u8> = <Vec<u8> as sqlx::decode::Decode<Sqlite>>::decode(value)?;
+ let uuid = Uuid::from_slice(&value)?;
+ Ok(Id(uuid))
+ }
+}
+impl<'q> Encode<'q, Sqlite> for Id {
+ fn encode_by_ref(
+ &self,
+ buf: &mut <Sqlite as sqlx::database::Database>::ArgumentBuffer<'q>,
+ ) -> Result<IsNull, BoxDynError> {
+ let value = self.0.as_bytes().to_vec();
+ <Vec<u8> as Encode<Sqlite>>::encode(value, buf)
+ }
+}
+
+#[derive(Debug, Clone)]
+pub struct Blob(pub Bytes);
+impl std::ops::Deref for Blob {
+ type Target = Bytes;
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+impl sqlx::Type<Sqlite> for Blob {
+ fn type_info() -> sqlite::SqliteTypeInfo {
+ <Vec<u8> as sqlx::Type<Sqlite>>::type_info() // BLOB
+ }
+
+ fn compatible(ty: &sqlite::SqliteTypeInfo) -> bool {
+ <Vec<u8> as sqlx::Type<Sqlite>>::compatible(ty)
+ }
+}
+impl<'r> sqlx::decode::Decode<'r, Sqlite> for Blob {
+ fn decode(
+ value: sqlite::SqliteValueRef<'r>,
+ ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+ let slice: &'r [u8] = <&'r [u8] as sqlx::decode::Decode<Sqlite>>::decode(value)?;
+ Ok(Blob(Bytes::copy_from_slice(slice)))
+ }
+}
+
+#[derive(Debug, Error)]
+pub enum StorageError {
+ #[error("file already exists: {0}/{1}")]
+ AlreadyExists(String, String),
+
+ #[error("blob integrity check failed")]
+ IntegrityError,
+
+ #[error("blob file missing on disk: {0}")]
+ BlobNotFound(PathBuf),
+
+ #[error("failed to connect index db: {0}")]
+ DbError(#[from] sqlx::Error),
+
+ #[error("io error: {0}")]
+ IoError(#[from] std::io::Error),
+}
+
+#[allow(unused)]
+#[derive(Debug, Clone, FromRow)]
+pub struct File {
+ pub id: Id,
+ pub scope: String,
+ pub created_at: chrono::DateTime<chrono::Utc>,
+
+ pub file_name: String,
+ pub file_size: i64,
+ pub file_bytes: Blob,
+ pub inlined: bool,
+}
+
+impl File {
+ fn uuid(scope: &str, file: &str) -> Id {
+ let scope_ns = Uuid::new_v5(&UUID_ROOT_NAMESPACE, scope.as_bytes());
+ Id(Uuid::new_v5(&scope_ns, file.as_bytes()))
+ }
+
+ fn hash(scope: &str, file: &str, blob: &[u8]) -> Vec<u8> {
+ let mut hasher = Crc32Hasher::new();
+
+ hasher.update(b"/");
+ hasher.update(scope.as_bytes());
+
+ hasher.update(b"/");
+ hasher.update(file.as_bytes());
+
+ hasher.update(b"/");
+ hasher.update(blob);
+ hasher.update(b"\0");
+
+ hasher.finalize().to_be_bytes().to_vec()
+ }
+}
+
+struct FileSystem {
+ objects: PathBuf,
+ database: PathBuf,
+}
+
+impl FileSystem {
+ fn new(root: &Path) -> Self {
+ Self {
+ objects: root.join("objects"),
+ database: root.join("index.sqlite"),
+ }
+ }
+
+ fn database(&self) -> &Path {
+ &self.database
+ }
+
+ fn objects_root(&self) -> &Path {
+ &self.objects
+ }
+
+ fn object(&self, scope: &str, file: &str) -> PathBuf {
+ let id = File::uuid(scope, file);
+ let hash_hex = hex::encode(id.0.as_bytes());
+ self.objects_root()
+ .join(&hash_hex[0..2])
+ .join(&hash_hex[2..4])
+ .join(&hash_hex)
+ }
+
+ async fn objects_walk<F, Fut>(&self, mut f: F) -> Result<(), StorageError>
+ where
+ F: FnMut(PathBuf) -> Fut,
+ Fut: std::future::Future<Output = Result<(), StorageError>>,
+ {
+ if !self.objects.exists() {
+ return Ok(());
+ }
+
+ let mut stack = vec![self.objects.to_path_buf()];
+ while let Some(dir) = stack.pop() {
+ let mut entries = fs::read_dir(&dir).await?;
+ while let Some(entry) = entries.next_entry().await? {
+ let path = entry.path();
+ let file_type = entry.file_type().await?;
+
+ if file_type.is_dir() {
+ stack.push(path);
+ } else if file_type.is_file() {
+ f(path).await?;
+ }
+ }
+ }
+
+ Ok(())
+ }
+}
+
+pub struct StorageService {
+ blobfs: FileSystem,
+ sqlite: Pool<Sqlite>,
+}
+
+impl StorageService {
+ pub async fn new(config: sync::Arc<ConfigService>) -> Result<Self, StorageError> {
+ let blobfs = FileSystem::new(config.dirname());
+
+ let connection = format!("sqlite:{}?mode=rwc", blobfs.database().to_str().unwrap());
+ let sqlite: Pool<Sqlite> = sqlite::SqlitePoolOptions::new()
+ .max_connections(SQLITE_POOL_SIZE)
+ .after_connect(|conn, _meta| {
+ Box::pin(async move {
+ // Connection-specific PRAGMAs (must be set on each connection)
+ sqlx::query("PRAGMA foreign_keys = ON;")
+ .execute(&mut *conn)
+ .await?;
+ sqlx::query("PRAGMA busy_timeout = 5000;")
+ .execute(&mut *conn)
+ .await?;
+ Ok(())
+ })
+ })
+ .connect(&connection)
+ .await?;
+
+ // WAL mode is database-wide and persists, only needs to be set once
+ query("PRAGMA journal_mode = WAL;").execute(&sqlite).await?;
+
+ let storage = Self { sqlite, blobfs };
+ storage.migrations().await?;
+ storage.doctor().await?;
+
+ Ok(storage)
+ }
+
+ /// Synchronously traverses the tree and removes temporary files.
+ /// Must run before the application starts.
+ async fn doctor(&self) -> Result<(), StorageError> {
+ let result = self.blobfs.objects_walk(|path| async move {
+ let name = match path.file_name().and_then(|name| name.to_str()) {
+ Some(name) => name.to_string(),
+ None => {
+ warn!(path = %path.display(), "cleanup: invalid blob name");
+ let _ = fs::remove_file(path).await;
+ return Ok(());
+ }
+ };
+
+ if name.ends_with(".part") {
+ warn!(path = %path.display(), "cleanup: removing temp file");
+ let _ = fs::remove_file(path).await;
+ return Ok(());
+ }
+
+ if name.len() != 32 {
+ warn!(path = %path.display(), "cleanup: invalid blob name length");
+ let _ = fs::remove_file(path).await;
+ return Ok(());
+ }
+
+ let id_bytes = match hex::decode(&name) {
+ Ok(bytes) => bytes,
+ Err(_) => {
+ warn!(path = %path.display(), "cleanup: invalid blob name hex");
+ let _ = fs::remove_file(path).await;
+ return Ok(());
+ }
+ };
+
+ let id = match Uuid::from_slice(&id_bytes) {
+ Ok(uuid) => Id(uuid),
+ Err(_) => {
+ warn!(path = %path.display(), "cleanup: invalid blob uuid");
+ let _ = fs::remove_file(path).await;
+ return Ok(());
+ }
+ };
+
+ let exists: Option<i64> = query_scalar("SELECT 1 FROM datafiles WHERE id = ?1")
+ .bind(id)
+ .fetch_optional(&self.sqlite)
+ .await?;
+
+ if exists.is_none() {
+ warn!(path = %path.display(), "cleanup: removing orphan blob");
+ let _ = fs::remove_file(path).await;
+ }
+
+ Ok(())
+ });
+ result.await?;
+
+ Ok(())
+ }
+
+ async fn migrations(&self) -> Result<(), StorageError> {
+ query(
+ "
+ CREATE TABLE IF NOT EXISTS datafiles(
+ id BLOB PRIMARY KEY CHECK (length(id) = 16),
+ scope TEXT NOT NULL,
+ created_at TEXT DEFAULT (datetime('now')),
+ file_name TEXT NOT NULL,
+ file_size INTEGER NOT NULL,
+ file_bytes BLOB,
+ inlined INTEGER NOT NULL,
+
+ UNIQUE (scope, file_name)
+ ) STRICT;
+ ",
+ )
+ .execute(&self.sqlite)
+ .await?;
+
+ Ok(())
+ }
+
+ #[instrument(skip(self))]
+ pub async fn get(&self, scope: &str, filename: &str) -> Result<Option<File>, StorageError> {
+ let file: Option<File> =
+ query_as("SELECT * FROM datafiles WHERE scope = ?1 AND file_name = ?2")
+ .bind(scope)
+ .bind(filename)
+ .fetch_optional(&self.sqlite)
+ .await?;
+
+ match file {
+ None => {
+ debug!("get: file not found");
+ Ok(None)
+ }
+ Some(mut file) if !file.inlined => {
+ let obj = self.blobfs.object(scope, filename);
+ let bytes = match fs::read(&obj).await {
+ Ok(b) => Bytes::from(b),
+ Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+ return Err(StorageError::BlobNotFound(obj));
+ }
+ Err(e) => return Err(e.into()),
+ };
+
+ let hash = File::hash(scope, filename, &bytes);
+ if hash != file.file_bytes.to_vec() {
+ return Err(StorageError::IntegrityError);
+ }
+
+ file.file_bytes = Blob(bytes);
+ debug!(size = file.file_size, "get: loaded from disk");
+ Ok(Some(file))
+ }
+ Some(file) => {
+ debug!(size = file.file_size, "get: loaded inline");
+ Ok(Some(file))
+ }
+ }
+ }
+
+ #[instrument(skip(self, bytes), fields(size = bytes.len()))]
+ pub async fn put(
+ &self,
+ scope: &str,
+ filename: &str,
+ bytes: &Bytes,
+ ) -> Result<(), StorageError> {
+ let inlined = bytes.len() <= INLINE_THRESHOLD;
+
+ let obj = self.blobfs.object(scope, filename);
+ let tmp = obj.with_extension(format!("{}.part", Uuid::new_v4()));
+
+ // write temp file
+ if !inlined {
+ if let Some(parent) = obj.parent() {
+ fs::create_dir_all(parent).await?;
+ }
+
+ let mut file = fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(&tmp)
+ .await?;
+
+ file.write_all(bytes).await?;
+ file.sync_all().await?;
+ }
+
+ // bytes for small files or hash for large ones
+ let payload: Vec<u8> = if inlined {
+ bytes.to_vec()
+ } else {
+ File::hash(scope, filename, bytes)
+ };
+
+ let result = async {
+ let mut tx = self.sqlite.begin().await?;
+ let id = File::uuid(scope, filename);
+
+ let result = query(
+ "
+ INSERT INTO datafiles (
+ id, scope, file_name, file_size, file_bytes, inlined
+ ) VALUES (
+ ?1, ?2, ?3, ?4, ?5, ?6
+ );
+ ",
+ )
+ .bind(id)
+ .bind(scope)
+ .bind(filename)
+ .bind(bytes.len() as i64)
+ .bind(&payload)
+ .bind(inlined)
+ .execute(tx.as_mut())
+ .await;
+
+ match result {
+ Ok(_) => {
+ if !inlined {
+ fs::rename(&tmp, &obj).await?;
+ if let Some(parent) = obj.parent() {
+ let dir = fs::File::open(parent).await?;
+ dir.sync_all().await?;
+ }
+ }
+
+ tx.commit().await?;
+ debug!("put: a new file has been commited");
+ Ok(())
+ }
+ Err(sqlx::Error::Database(ref db_err)) if db_err.is_unique_violation() => {
+ let existing: Option<File> =
+ query_as("SELECT * FROM datafiles WHERE scope = ?1 AND file_name = ?2")
+ .bind(scope)
+ .bind(filename)
+ .fetch_optional(tx.as_mut())
+ .await?;
+
+ match existing {
+ Some(file) if file.file_bytes.to_vec() == payload => {
+ if !inlined {
+ let _ = fs::remove_file(&tmp).await;
+ tx.rollback().await?;
+ }
+
+ debug!("put: identical file already exists");
+ return Ok(());
+ }
+ Some(_) => {
+ warn!("put: file already exists with different content");
+ }
+ None => {
+ // unique_violation but row doesn't exist - shouldn't happen
+ warn!("corrupted! 'is_unique_violation' received, but data cannot be selected");
+ }
+ }
+
+ Err(StorageError::AlreadyExists(
+ scope.to_string(),
+ filename.to_string(),
+ ))
+ }
+ Err(e) => Err(e.into()),
+ }
+ }.await;
+
+ if result.is_err() && !inlined {
+ let _ = fs::remove_file(&tmp).await;
+ }
+
+ result
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use tempfile::TempDir;
+
+ const BIG_FILE_SIZE: usize = INLINE_THRESHOLD * 64;
+
+ async fn create_storage_at(path: &Path) -> StorageService {
+ let config = sync::Arc::new(ConfigService::for_test(path.to_path_buf()));
+ StorageService::new(config).await.unwrap()
+ }
+
+ async fn create_test_storage() -> (StorageService, TempDir) {
+ let tmp = TempDir::new().unwrap();
+ let storage = create_storage_at(tmp.path()).await;
+ (storage, tmp)
+ }
+
+ /// Count files in a directory recursively
+ fn count_files(dir: &Path, ext: Option<&str>) -> usize {
+ let mut count = 0;
+ if let Ok(entries) = std::fs::read_dir(dir) {
+ for entry in entries.flatten() {
+ let path = entry.path();
+ if path.is_dir() {
+ count += count_files(&path, ext);
+ } else if let Some(extension) = ext {
+ if path.extension().is_some_and(|ext| ext == extension) {
+ count += 1;
+ }
+ } else {
+ count += 1;
+ }
+ }
+ }
+ count
+ }
+
+ /// Recursively delete or corrupt all files in a directory
+ fn damage_blobs(dir: &Path, corrupt: bool) {
+ if let Ok(entries) = std::fs::read_dir(dir) {
+ for entry in entries.flatten() {
+ let path = entry.path();
+ if path.is_dir() {
+ damage_blobs(&path, corrupt);
+ } else if corrupt {
+ if let Ok(meta) = path.metadata() {
+ let _ = std::fs::write(&path, vec![0x00; meta.len() as usize]);
+ }
+ } else {
+ let _ = std::fs::remove_file(&path);
+ }
+ }
+ }
+ }
+
+ #[tokio::test]
+ async fn test_write_read_small_file() {
+ let (storage, _tmp) = create_test_storage().await;
+
+ // 1 KB file - should be stored inline
+ let data = Bytes::from(vec![0xAB; 1024]);
+ storage.put("test-scope", "small.bin", &data).await.unwrap();
+
+ let file = storage.get("test-scope", "small.bin").await.unwrap();
+ assert!(file.is_some());
+
+ let file = file.unwrap();
+ assert!(file.inlined);
+ assert_eq!(file.file_bytes.0, data);
+ assert_eq!(file.file_size, 1024);
+ }
+
+ #[tokio::test]
+ async fn test_write_read_large_file() {
+ let (storage, tmp) = create_test_storage().await;
+
+ // Large file - should be stored on disk
+ let data = Bytes::from(vec![0xCD; BIG_FILE_SIZE]);
+ storage.put("test-scope", "large.bin", &data).await.unwrap();
+
+ let file = storage.get("test-scope", "large.bin").await.unwrap();
+ assert!(file.is_some());
+
+ let file = file.unwrap();
+ assert!(!file.inlined);
+ assert_eq!(file.file_bytes.0, data);
+ assert_eq!(file.file_size, (BIG_FILE_SIZE) as i64);
+
+ // Verify file is stored on disk, not inline in SQLite
+ let db_size = std::fs::metadata(tmp.path().join("index.sqlite"))
+ .unwrap()
+ .len();
+ assert!(
+ db_size < data.len() as u64,
+ "database ({db_size} bytes) should be smaller than file ({} bytes)",
+ data.len()
+ );
+ }
+
+ #[tokio::test]
+ async fn test_write_read_boundary_file() {
+ let (storage, _tmp) = create_test_storage().await;
+
+ // Exactly 256 KB - should be stored inline (threshold is <=)
+ let data = Bytes::from(vec![0xEF; INLINE_THRESHOLD]);
+ storage
+ .put("test-scope", "boundary.bin", &data)
+ .await
+ .unwrap();
+
+ let file = storage.get("test-scope", "boundary.bin").await.unwrap();
+ assert!(file.is_some());
+
+ let file = file.unwrap();
+ assert!(file.inlined);
+ assert_eq!(file.file_bytes.0, data);
+ assert_eq!(file.file_size, INLINE_THRESHOLD as i64);
+ }
+
+ #[tokio::test]
+ async fn test_read_nonexistent_file() {
+ let (storage, _tmp) = create_test_storage().await;
+
+ let file = storage.get("test-scope", "nonexistent.bin").await.unwrap();
+ assert!(file.is_none());
+ }
+
+ #[tokio::test]
+ async fn test_persistence_after_restart() {
+ let tmp = TempDir::new().unwrap();
+
+ let small_data = Bytes::from(vec![0x11; 1024]);
+ let large_data = Bytes::from(vec![0x22; BIG_FILE_SIZE]);
+
+ // First "session": write files
+ {
+ let storage = create_storage_at(tmp.path()).await;
+ storage
+ .put("persist", "small.bin", &small_data)
+ .await
+ .unwrap();
+ storage
+ .put("persist", "large.bin", &large_data)
+ .await
+ .unwrap();
+ // storage is dropped here, simulating shutdown
+ }
+
+ // Second "session": verify files persist
+ {
+ let storage = create_storage_at(tmp.path()).await;
+
+ let small = storage.get("persist", "small.bin").await.unwrap();
+ assert!(small.is_some());
+ assert_eq!(small.unwrap().file_bytes.0, small_data);
+
+ let large = storage.get("persist", "large.bin").await.unwrap();
+ assert!(large.is_some());
+ assert_eq!(large.unwrap().file_bytes.0, large_data);
+ }
+ }
+
+ #[tokio::test]
+ async fn test_error_blob_not_found() {
+ let (storage, tmp) = create_test_storage().await;
+
+ // Write a large file (stored on disk)
+ let data = Bytes::from(vec![0xAA; BIG_FILE_SIZE]);
+ storage
+ .put("test-scope", "to-delete.bin", &data)
+ .await
+ .unwrap();
+
+ // Delete all blob files from disk
+ damage_blobs(&tmp.path().join("objects"), false);
+
+ // Reading should fail with BlobNotFound
+ let result = storage.get("test-scope", "to-delete.bin").await;
+ assert!(matches!(result, Err(StorageError::BlobNotFound(_))));
+ }
+
+ #[tokio::test]
+ async fn test_error_blob_corrupted() {
+ let (storage, tmp) = create_test_storage().await;
+
+ // Write a large file (stored on disk)
+ let data = Bytes::from(vec![0xBB; BIG_FILE_SIZE]);
+ storage
+ .put("test-scope", "to-corrupt.bin", &data)
+ .await
+ .unwrap();
+
+ // Corrupt all blob files on disk
+ damage_blobs(&tmp.path().join("objects"), true);
+
+ // Reading should fail with IntegrityError
+ let result = storage.get("test-scope", "to-corrupt.bin").await;
+ assert!(matches!(result, Err(StorageError::IntegrityError)));
+ }
+
+ #[tokio::test]
+ async fn test_duplicate_put_same_content() {
+ let (storage, tmp) = create_test_storage().await;
+
+ // Write a large file
+ let data = Bytes::from(vec![0xCC; BIG_FILE_SIZE]);
+ storage.put("test-scope", "dup.bin", &data).await.unwrap();
+
+ // Write the same file again with identical content - should succeed
+ let result = storage.put("test-scope", "dup.bin", &data).await;
+ assert!(result.is_ok());
+
+ // Verify no temp files left behind
+ assert_eq!(count_files(&tmp.path().join("objects"), Some("part")), 0);
+
+ // Verify file is still readable with correct content
+ let file = storage.get("test-scope", "dup.bin").await.unwrap();
+ assert!(file.is_some());
+ assert_eq!(file.unwrap().file_bytes.0, data);
+ }
+
+ #[tokio::test]
+ async fn test_duplicate_put_different_content() {
+ let (storage, tmp) = create_test_storage().await;
+
+ // Write a large file
+ let data1 = Bytes::from(vec![0xDD; BIG_FILE_SIZE]);
+ storage.put("test-scope", "dup.bin", &data1).await.unwrap();
+
+ // Write the same filename with different content - should fail
+ let data2 = Bytes::from(vec![0xEE; BIG_FILE_SIZE]);
+ let result = storage.put("test-scope", "dup.bin", &data2).await;
+ assert!(matches!(result, Err(StorageError::AlreadyExists(_, _))));
+
+ // Verify no temp files left behind
+ assert_eq!(count_files(&tmp.path().join("objects"), Some("part")), 0);
+
+ // Verify original file is still intact
+ let file = storage.get("test-scope", "dup.bin").await.unwrap();
+ assert!(file.is_some());
+ assert_eq!(file.unwrap().file_bytes.0, data1);
+ }
+
+ #[tokio::test]
+ async fn test_doctor_removes_temp_files() {
+ let tmp = TempDir::new().unwrap();
+
+ // First session: write a large file
+ let data = Bytes::from(vec![0xAA; BIG_FILE_SIZE]);
+ {
+ let storage = create_storage_at(tmp.path()).await;
+ storage.put("test-scope", "valid.bin", &data).await.unwrap();
+ }
+
+ // Manually create temp files in the objects directory
+ let objects_dir = tmp.path().join("objects");
+ std::fs::create_dir_all(objects_dir.join("ab/cd")).unwrap();
+ std::fs::write(objects_dir.join("ab/cd/test.part"), b"temp1").unwrap();
+ std::fs::write(objects_dir.join("ab/cd/another.12345.part"), b"temp2").unwrap();
+
+ assert_eq!(count_files(&objects_dir, Some("part")), 2);
+
+ // Second session: doctor should clean up temp files
+ {
+ let storage = create_storage_at(tmp.path()).await;
+
+ // Temp files should be gone
+ assert_eq!(count_files(&objects_dir, Some("part")), 0);
+
+ // Valid file should still be readable
+ let file = storage.get("test-scope", "valid.bin").await.unwrap();
+ assert!(file.is_some());
+ assert_eq!(file.unwrap().file_bytes.0, data);
+ }
+ }
+
+ #[tokio::test]
+ async fn test_doctor_removes_orphan_blobs() {
+ let tmp = TempDir::new().unwrap();
+
+ // First session: write a large file
+ let data = Bytes::from(vec![0xBB; BIG_FILE_SIZE]);
+ {
+ let storage = create_storage_at(tmp.path()).await;
+ storage.put("test-scope", "valid.bin", &data).await.unwrap();
+ }
+
+ // Count files before adding orphan
+ let objects_dir = tmp.path().join("objects");
+ let files_before = count_files(&objects_dir, None);
+ assert_eq!(files_before, 1); // Only the valid blob
+
+ // Manually create an orphan blob (valid hex name but no DB record)
+ std::fs::create_dir_all(objects_dir.join("de/ad")).unwrap();
+ let orphan_name = "deadbeefdeadbeefdeadbeefdeadbeef"; // 32 hex chars
+ std::fs::write(objects_dir.join("de/ad").join(orphan_name), b"orphan").unwrap();
+
+ assert_eq!(count_files(&objects_dir, None), 2);
+
+ // Second session: doctor should clean up orphan blob
+ {
+ let storage = create_storage_at(tmp.path()).await;
+
+ // Only the valid file should remain
+ assert_eq!(count_files(&objects_dir, None), 1);
+
+ // Valid file should still be readable
+ let file = storage.get("test-scope", "valid.bin").await.unwrap();
+ assert!(file.is_some());
+ assert_eq!(file.unwrap().file_bytes.0, data);
+ }
+ }
+}