aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNikolay Govorov <me@govorov.online>2026-01-14 00:36:26 +0000
committerNikolay Govorov <me@govorov.online>2026-01-14 01:50:27 +0000
commita49e094d3e85b0b0fb10cf6c5b1a8a4f4259dfae (patch)
treeb12e0fd308b94e4f11a7f70c3dd1a6236cc13d88
parent9eb7972aa06168a449c7e1b468761a295c0f3c8f (diff)
downloadtar
tar.gz
tar.bz2
tar.lz
tar.xz
tar.zst
zip
Designs a more robust storage based on sqlite
Diffstat
-rw-r--r--Cargo.lock806+806 −0
-rw-r--r--Cargo.toml4+4 −0
-rw-r--r--deny.toml8+4 −4
-rw-r--r--src/controller_zig.rs22+10 −12
-rw-r--r--src/main.rs11+9 −2
-rw-r--r--src/service_storage.rs263+237 −26
6 files changed, 1070 insertions, 44 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 4618da4..62f2084 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -24,6 +24,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
+[[package]]
name = "anstream"
version = "0.6.21"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -101,6 +110,15 @@ dependencies = [
]
[[package]]
+name = "atoi"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
name = "atomic-waker"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -184,10 +202,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
+name = "base64ct"
+version = "1.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
+
+[[package]]
name = "bitflags"
version = "2.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
+dependencies = [
+ "serde_core",
+]
[[package]]
name = "bitvec"
@@ -354,6 +381,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
+name = "chrono"
+version = "0.4.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
+dependencies = [
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "wasm-bindgen",
+ "windows-link",
+]
+
+[[package]]
name = "clap"
version = "4.5.54"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -454,6 +494,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d"
[[package]]
+name = "concurrent-queue"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "const-oid"
+version = "0.9.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8"
+
+[[package]]
name = "core-foundation"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -489,6 +544,21 @@ dependencies = [
]
[[package]]
+name = "crc"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d"
+dependencies = [
+ "crc-catalog",
+]
+
+[[package]]
+name = "crc-catalog"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5"
+
+[[package]]
name = "crc32fast"
version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -573,6 +643,17 @@ dependencies = [
]
[[package]]
+name = "der"
+version = "0.7.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
+dependencies = [
+ "const-oid",
+ "pem-rfc7468",
+ "zeroize",
+]
+
+[[package]]
name = "deranged"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -588,7 +669,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
dependencies = [
"block-buffer",
+ "const-oid",
"crypto-common",
+ "subtle",
]
[[package]]
@@ -603,6 +686,12 @@ dependencies = [
]
[[package]]
+name = "dotenvy"
+version = "0.15.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b"
+
+[[package]]
name = "dunce"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -613,6 +702,9 @@ name = "either"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+dependencies = [
+ "serde",
+]
[[package]]
name = "encoding_rs"
@@ -640,6 +732,28 @@ dependencies = [
]
[[package]]
+name = "etcetera"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943"
+dependencies = [
+ "cfg-if",
+ "home",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "event-listener"
+version = "5.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
+dependencies = [
+ "concurrent-queue",
+ "parking",
+ "pin-project-lite",
+]
+
+[[package]]
name = "faster-hex"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -699,6 +813,17 @@ dependencies = [
]
[[package]]
+name = "flume"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+ "spin",
+]
+
+[[package]]
name = "fnv"
version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -772,6 +897,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-intrusive"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
+dependencies = [
+ "futures-core",
+ "lock_api",
+ "parking_lot",
+]
+
+[[package]]
name = "futures-io"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1656,6 +1803,8 @@ version = "0.15.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
dependencies = [
+ "allocator-api2",
+ "equivalent",
"foldhash 0.1.5",
]
@@ -1671,6 +1820,15 @@ dependencies = [
]
[[package]]
+name = "hashlink"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
+dependencies = [
+ "hashbrown 0.15.5",
+]
+
+[[package]]
name = "heapless"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1687,6 +1845,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
+name = "hex"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
+
+[[package]]
+name = "hkdf"
+version = "0.12.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7"
+dependencies = [
+ "hmac",
+]
+
+[[package]]
+name = "hmac"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e"
+dependencies = [
+ "digest",
+]
+
+[[package]]
name = "home"
version = "0.5.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1820,6 +2002,30 @@ dependencies = [
]
[[package]]
+name = "iana-time-zone"
+version = "0.1.64"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "log",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
name = "icu_collections"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2076,12 +2282,27 @@ dependencies = [
]
[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+dependencies = [
+ "spin",
+]
+
+[[package]]
name = "libc"
version = "0.2.180"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc"
[[package]]
+name = "libm"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
+
+[[package]]
name = "libredox"
version = "0.1.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2093,6 +2314,17 @@ dependencies = [
]
[[package]]
+name = "libsqlite3-sys"
+version = "0.30.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
+dependencies = [
+ "cc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
name = "linux-raw-sys"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2137,6 +2369,16 @@ dependencies = [
]
[[package]]
+name = "md-5"
+version = "0.10.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
+dependencies = [
+ "cfg-if",
+ "digest",
+]
+
+[[package]]
name = "memchr"
version = "2.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2231,12 +2473,58 @@ dependencies = [
]
[[package]]
+name = "num-bigint-dig"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7"
+dependencies = [
+ "lazy_static",
+ "libm",
+ "num-integer",
+ "num-iter",
+ "num-traits",
+ "rand",
+ "smallvec",
+ "zeroize",
+]
+
+[[package]]
name = "num-conv"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9"
[[package]]
+name = "num-integer"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "num-iter"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
+dependencies = [
+ "autocfg",
+ "libm",
+]
+
+[[package]]
name = "once_cell"
version = "1.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2299,6 +2587,12 @@ dependencies = [
]
[[package]]
+name = "parking"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
+
+[[package]]
name = "parking_lot"
version = "0.12.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2322,6 +2616,15 @@ dependencies = [
]
[[package]]
+name = "pem-rfc7468"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412"
+dependencies = [
+ "base64ct",
+]
+
+[[package]]
name = "percent-encoding"
version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2351,6 +2654,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
+name = "pkcs1"
+version = "0.7.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f"
+dependencies = [
+ "der",
+ "pkcs8",
+ "spki",
+]
+
+[[package]]
+name = "pkcs8"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
+dependencies = [
+ "der",
+ "spki",
+]
+
+[[package]]
name = "pkg-config"
version = "0.3.32"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2402,6 +2726,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
+name = "ppv-lite86"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
name = "proc-macro2"
version = "1.0.105"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2441,6 +2774,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.17",
+]
+
+[[package]]
name = "rayon"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2561,6 +2924,26 @@ dependencies = [
]
[[package]]
+name = "rsa"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d"
+dependencies = [
+ "const-oid",
+ "digest",
+ "num-bigint-dig",
+ "num-integer",
+ "num-traits",
+ "pkcs1",
+ "pkcs8",
+ "rand_core",
+ "signature",
+ "spki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
name = "rustc-stable-hash"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2921,6 +3304,16 @@ dependencies = [
]
[[package]]
+name = "signature"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de"
+dependencies = [
+ "digest",
+ "rand_core",
+]
+
+[[package]]
name = "simd-adler32"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2937,6 +3330,9 @@ name = "smallvec"
version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
+dependencies = [
+ "serde",
+]
[[package]]
name = "smol_str"
@@ -2976,6 +3372,211 @@ name = "spin"
version = "0.9.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+dependencies = [
+ "lock_api",
+]
+
+[[package]]
+name = "spki"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
+dependencies = [
+ "base64ct",
+ "der",
+]
+
+[[package]]
+name = "sqlx"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc"
+dependencies = [
+ "sqlx-core",
+ "sqlx-macros",
+ "sqlx-mysql",
+ "sqlx-postgres",
+ "sqlx-sqlite",
+]
+
+[[package]]
+name = "sqlx-core"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6"
+dependencies = [
+ "base64",
+ "bytes",
+ "chrono",
+ "crc",
+ "crossbeam-queue",
+ "either",
+ "event-listener",
+ "futures-core",
+ "futures-intrusive",
+ "futures-io",
+ "futures-util",
+ "hashbrown 0.15.5",
+ "hashlink",
+ "indexmap",
+ "log",
+ "memchr",
+ "once_cell",
+ "percent-encoding",
+ "serde",
+ "serde_json",
+ "sha2",
+ "smallvec",
+ "thiserror 2.0.17",
+ "tokio",
+ "tokio-stream",
+ "tracing",
+ "url",
+]
+
+[[package]]
+name = "sqlx-macros"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "sqlx-core",
+ "sqlx-macros-core",
+ "syn",
+]
+
+[[package]]
+name = "sqlx-macros-core"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b"
+dependencies = [
+ "dotenvy",
+ "either",
+ "heck",
+ "hex",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "sha2",
+ "sqlx-core",
+ "sqlx-mysql",
+ "sqlx-postgres",
+ "sqlx-sqlite",
+ "syn",
+ "tokio",
+ "url",
+]
+
+[[package]]
+name = "sqlx-mysql"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526"
+dependencies = [
+ "atoi",
+ "base64",
+ "bitflags",
+ "byteorder",
+ "bytes",
+ "chrono",
+ "crc",
+ "digest",
+ "dotenvy",
+ "either",
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-util",
+ "generic-array",
+ "hex",
+ "hkdf",
+ "hmac",
+ "itoa",
+ "log",
+ "md-5",
+ "memchr",
+ "once_cell",
+ "percent-encoding",
+ "rand",
+ "rsa",
+ "serde",
+ "sha1",
+ "sha2",
+ "smallvec",
+ "sqlx-core",
+ "stringprep",
+ "thiserror 2.0.17",
+ "tracing",
+ "whoami",
+]
+
+[[package]]
+name = "sqlx-postgres"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46"
+dependencies = [
+ "atoi",
+ "base64",
+ "bitflags",
+ "byteorder",
+ "chrono",
+ "crc",
+ "dotenvy",
+ "etcetera",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "hex",
+ "hkdf",
+ "hmac",
+ "home",
+ "itoa",
+ "log",
+ "md-5",
+ "memchr",
+ "once_cell",
+ "rand",
+ "serde",
+ "serde_json",
+ "sha2",
+ "smallvec",
+ "sqlx-core",
+ "stringprep",
+ "thiserror 2.0.17",
+ "tracing",
+ "whoami",
+]
+
+[[package]]
+name = "sqlx-sqlite"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea"
+dependencies = [
+ "atoi",
+ "chrono",
+ "flume",
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-intrusive",
+ "futures-util",
+ "libsqlite3-sys",
+ "log",
+ "percent-encoding",
+ "serde",
+ "serde_urlencoded",
+ "sqlx-core",
+ "thiserror 2.0.17",
+ "tracing",
+ "url",
+]
[[package]]
name = "stable_deref_trait"
@@ -2990,6 +3591,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
+name = "stringprep"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+ "unicode-properties",
+]
+
+[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3259,6 +3871,17 @@ dependencies = [
]
[[package]]
+name = "tokio-stream"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
name = "tokio-util"
version = "0.7.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3379,10 +4002,22 @@ checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100"
dependencies = [
"log",
"pin-project-lite",
+ "tracing-attributes",
"tracing-core",
]
[[package]]
+name = "tracing-attributes"
+version = "0.1.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
name = "tracing-core"
version = "0.1.36"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3410,6 +4045,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
[[package]]
+name = "unicode-bidi"
+version = "0.3.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5"
+
+[[package]]
name = "unicode-bom"
version = "2.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3431,6 +4072,12 @@ dependencies = [
]
[[package]]
+name = "unicode-properties"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
+
+[[package]]
name = "unicode-width"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3514,6 +4161,12 @@ dependencies = [
]
[[package]]
+name = "wasite"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b"
+
+[[package]]
name = "wasm-bindgen"
version = "0.2.106"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3591,6 +4244,16 @@ dependencies = [
]
[[package]]
+name = "whoami"
+version = "1.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d4a4db5077702ca3015d3d02d74974948aba2ad9e12ab7df718ee64ccd7e97d"
+dependencies = [
+ "libredox",
+ "wasite",
+]
+
+[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3622,12 +4285,65 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
+name = "windows-core"
+version = "0.62.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
+dependencies = [
+ "windows-implement",
+ "windows-interface",
+ "windows-link",
+ "windows-result",
+ "windows-strings",
+]
+
+[[package]]
+name = "windows-implement"
+version = "0.60.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "windows-interface"
+version = "0.59.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
name = "windows-link"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
[[package]]
+name = "windows-result"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "windows-strings"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
name = "windows-sys"
version = "0.45.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3638,6 +4354,15 @@ dependencies = [
[[package]]
name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
@@ -3680,6 +4405,21 @@ dependencies = [
[[package]]
name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
@@ -3719,6 +4459,12 @@ checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
[[package]]
name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
@@ -3737,6 +4483,12 @@ checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
[[package]]
name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
@@ -3755,6 +4507,12 @@ checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
[[package]]
name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
@@ -3785,6 +4543,12 @@ checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
[[package]]
name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
@@ -3803,6 +4567,12 @@ checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
[[package]]
name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
@@ -3821,6 +4591,12 @@ checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
[[package]]
name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
@@ -3839,6 +4615,12 @@ checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
[[package]]
name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
@@ -3903,6 +4685,26 @@ dependencies = [
]
[[package]]
+name = "zerocopy"
+version = "0.8.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "668f5168d10b9ee831de31933dc111a459c97ec93225beb307aed970d1372dfd"
+dependencies = [
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.8.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2c7962b26b0a8685668b671ee4b54d007a67d4eaf05fda79ac0ecf41e32270f1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
name = "zerofrom"
version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3981,6 +4783,8 @@ dependencies = [
"axum",
"bytes",
"cargo-deny",
+ "chrono",
+ "hex",
"http-body-util",
"hyper",
"hyper-tls",
@@ -3988,6 +4792,8 @@ dependencies = [
"minijinja",
"semver",
"serde",
+ "sha2",
+ "sqlx",
"thiserror 2.0.17",
"tokio",
"toml",
diff --git a/Cargo.toml b/Cargo.toml
index ea0261a..5417587 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -11,13 +11,17 @@ repository = "https://github.com/mrdimidium/Zorian"
[dependencies]
axum = { version = "0.8.8", features = ["http2", "macros", "multipart"] }
bytes = "1.11.0"
+chrono = "0.4"
+hex = "0.4"
http-body-util = "0.1.3"
hyper = "1.8.1"
hyper-tls = "0.6.0"
hyper-util = { version = "0.1.19", features = ["client", "http1", "http2", "tokio"] }
minijinja = "2.14.0"
semver = "1.0"
+sha2 = "0.10"
serde = { version = "1.0", features = ["derive"] }
+sqlx = { version = "0.8.6", features = ["sqlite", "runtime-tokio", "macros", "derive", "chrono", "json"] }
thiserror = "2.0"
tokio = { version = "1.49.0", features = ["full"] }
toml = "0.9.8"
diff --git a/deny.toml b/deny.toml
index 3db906e..28938d0 100644
--- a/deny.toml
+++ b/deny.toml
@@ -8,6 +8,10 @@ allow = [
"Apache-2.0",
"BSD-3-Clause",
+ # Custom but acceptable free licenses
+ "Zlib",
+ "Unicode-3.0",
+
# GPL versions
"GPL-2.0-or-later",
"GPL-3.0-or-later",
@@ -18,10 +22,6 @@ allow = [
exceptions = [
# The project itself is distributed under the AGPL, but avoid it in dependencies
{ crate = "zorian", allow = ["AGPL-3.0-or-later"] },
-
- # Acceptable custom free licenses (for specific products only)
- { crate = "foldhash", allow = ["Zlib"] },
- { crate = "unicode-ident", allow = ["Unicode-3.0"] },
]
unused-allowed-license = "allow"
diff --git a/src/controller_zig.rs b/src/controller_zig.rs
index 41b8b0a..7403998 100644
--- a/src/controller_zig.rs
+++ b/src/controller_zig.rs
@@ -171,9 +171,12 @@ impl ZigController {
let tarball = Tarball::parse(&filename).map_err(|_| http::StatusCode::NOT_FOUND)?;
let url = tarball.upstream_url(controller.config.appname());
- match controller.storage.get(&filename).await {
+ match controller.storage.get("zig", &filename).await {
Ok(Some(entry)) => {
- return Ok(Self::build_response(http::StatusCode::OK, entry));
+ return Ok(Self::build_response(
+ http::StatusCode::OK,
+ entry.file_bytes.0,
+ ));
}
Ok(None) => {}
Err(_) => {
@@ -185,28 +188,23 @@ impl ZigController {
.upstream
.fetch(service_upstream::DownloadRequest { url })
.await?;
- let cache_entry = service_storage::File {
- bytes: entry.bytes.clone(),
- };
+ // bytes: entry.bytes.clone(),
- match controller.storage.put(&filename, &cache_entry).await {
+ match controller.storage.put("zig", &filename, &entry.bytes).await {
Ok(()) => {}
Err(_) => {
return Err(http::StatusCode::OK);
}
}
- Ok(Self::build_response(http::StatusCode::OK, cache_entry))
+ Ok(Self::build_response(http::StatusCode::OK, entry.bytes))
}
- fn build_response(
- status: http::StatusCode,
- entry: service_storage::File,
- ) -> response::Response {
+ fn build_response(status: http::StatusCode, bytes: bytes::Bytes) -> response::Response {
response::Response::builder()
.status(status)
.header(http::header::CONTENT_TYPE, "application/octet-stream")
- .body(body::Body::from(entry.bytes))
+ .body(body::Body::from(bytes))
.unwrap()
}
}
diff --git a/src/main.rs b/src/main.rs
index 6dd25a1..8edafe3 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -46,11 +46,18 @@ async fn main() {
None => service_config::ConfigService::default(),
});
config.validate().unwrap_or_else(|e| {
- eprintln!("error: {e}");
+ eprintln!("config error: {e}");
std::process::exit(1);
});
- let storage = Arc::new(service_storage::StorageService::new(config.clone()));
+ let storage = Arc::new(
+ service_storage::StorageService::new(config.clone())
+ .await
+ .unwrap_or_else(|e| {
+ eprintln!("storage error: {e}");
+ std::process::exit(1);
+ }),
+ );
let upstream = Arc::new(service_upstream::UpstreamService::new());
let web_controller = Arc::new(controller_web::WebController::new());
diff --git a/src/service_storage.rs b/src/service_storage.rs
index bce2c79..fbf0bca 100644
--- a/src/service_storage.rs
+++ b/src/service_storage.rs
@@ -1,50 +1,261 @@
// SPDX-FileCopyrightText: 2026 Nikolay Govorov <me@govorov.online>
// SPDX-License-Identifier: AGPL-3.0-or-later
-use std::{io, path, sync};
-use tokio::fs;
+use std::path::PathBuf;
+use std::sync;
+
+use tokio::io::AsyncWriteExt;
+
+use sha2::{Digest, Sha256};
+use sqlx::{FromRow, Pool, Sqlite, query, query_as, sqlite};
+use thiserror::Error;
use super::service_config::ConfigService;
-#[derive(Clone)]
+const SQLITE_POOL_SIZE: u32 = 16;
+const INLINE_THRESHOLD: usize = 256 * 1024; // 256 KB
+
+fn hash_blob(scope: &str, filename: &str, bytes: &[u8]) -> Vec<u8> {
+ let mut hasher = Sha256::new();
+ hasher.update(scope.as_bytes());
+ hasher.update(b"/");
+ hasher.update(filename.as_bytes());
+ hasher.update(b"\0");
+ hasher.update(bytes);
+ hasher.finalize().to_vec()
+}
+
+#[derive(Debug, Clone)]
+pub struct Blob(pub bytes::Bytes);
+
+impl std::ops::Deref for Blob {
+ type Target = bytes::Bytes;
+ fn deref(&self) -> &Self::Target {
+ &self.0
+ }
+}
+
+impl sqlx::Type<Sqlite> for Blob {
+ fn type_info() -> sqlite::SqliteTypeInfo {
+ <Vec<u8> as sqlx::Type<Sqlite>>::type_info() // BLOB
+ }
+
+ fn compatible(ty: &sqlite::SqliteTypeInfo) -> bool {
+ <Vec<u8> as sqlx::Type<Sqlite>>::compatible(ty)
+ }
+}
+
+impl<'r> sqlx::decode::Decode<'r, Sqlite> for Blob {
+ fn decode(
+ value: sqlite::SqliteValueRef<'r>,
+ ) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
+ let slice: &'r [u8] = <&'r [u8] as sqlx::decode::Decode<Sqlite>>::decode(value)?;
+ Ok(Blob(bytes::Bytes::copy_from_slice(slice)))
+ }
+}
+
+#[allow(unused)]
+#[derive(Debug, Clone, FromRow)]
pub struct File {
- pub bytes: bytes::Bytes,
+ pub id: u64,
+ pub scope: String,
+ pub created_at: chrono::DateTime<chrono::Utc>,
+
+ pub file_name: String,
+ pub file_size: i64,
+ pub file_bytes: Blob,
+ pub inlined: bool,
+}
+
+#[derive(Debug, Error)]
+pub enum StorageError {
+ #[error("file already exists: {0}/{1}")]
+ AlreadyExists(String, String),
+
+ #[error("blob integrity check failed")]
+ IntegrityError,
+
+ #[error("failed to connect index db: {0}")]
+ DbError(#[from] sqlx::Error),
+
+ #[error("io error: {0}")]
+ IoError(#[from] std::io::Error),
}
pub struct StorageService {
- config: sync::Arc<ConfigService>,
+ sqlite: Pool<Sqlite>,
+ storage_dir: PathBuf,
}
impl StorageService {
- pub fn new(config: sync::Arc<ConfigService>) -> Self {
- Self { config }
+ pub async fn new(config: sync::Arc<ConfigService>) -> Result<Self, StorageError> {
+ let storage_dir = config.dirname().to_path_buf();
+ let connection = format!(
+ "sqlite:{}?mode=rwc",
+ storage_dir.join("index.sqlite").to_str().unwrap()
+ );
+
+ let pool: Pool<Sqlite> = sqlite::SqlitePoolOptions::new()
+ .max_connections(SQLITE_POOL_SIZE)
+ .connect(&connection)
+ .await?;
+
+ let storage = Self {
+ sqlite: pool,
+ storage_dir,
+ };
+ storage.msetupdb().await?;
+
+ Ok(storage)
}
- pub async fn get(&self, filename: &str) -> Result<Option<File>, io::Error> {
- let data_path = self.data_path(filename)?;
- let bytes = match fs::read(&data_path).await {
- Ok(bytes) => bytes,
- Err(err) => {
- return if err.kind() == io::ErrorKind::NotFound {
- Ok(None)
- } else {
- Err(err)
- };
- }
+ async fn msetupdb(&self) -> Result<(), StorageError> {
+ query("PRAGMA foreign_keys = ON;")
+ .execute(&self.sqlite)
+ .await?;
+ query("PRAGMA journal_mode = WAL;")
+ .execute(&self.sqlite)
+ .await?;
+ query("PRAGMA busy_timeout = 5000;")
+ .execute(&self.sqlite)
+ .await?;
+
+ query(
+ "CREATE TABLE IF NOT EXISTS datafiles(
+ id INTEGER PRIMARY KEY,
+ scope TEXT NOT NULL,
+ created_at TEXT DEFAULT (datetime('now')),
+ file_name TEXT NOT NULL,
+ file_size INTEGER NOT NULL,
+ file_bytes BLOB,
+ inlined INTEGER NOT NULL,
+
+ UNIQUE (scope, file_name)
+ ) STRICT;",
+ )
+ .execute(&self.sqlite)
+ .await?;
+
+ Ok(())
+ }
+
+ pub async fn put(
+ &self,
+ scope: &str,
+ filename: &str,
+ bytes: &bytes::Bytes,
+ ) -> Result<(), StorageError> {
+ let inlined = bytes.len() <= INLINE_THRESHOLD;
+
+ let stored_bytes: Vec<u8> = if inlined {
+ bytes.to_vec()
+ } else {
+ self.write_blob(scope, filename, bytes).await?
};
- Ok(Some(File {
- bytes: bytes.into(),
- }))
+ let result = query("INSERT INTO datafiles (scope, file_name, file_size, file_bytes, inlined) VALUES (?1, ?2, ?3, ?4, ?5);")
+ .bind(scope)
+ .bind(filename)
+ .bind(bytes.len() as i64)
+ .bind(&stored_bytes)
+ .bind(inlined)
+ .execute(&self.sqlite)
+ .await;
+
+ match result {
+ Ok(_) => Ok(()),
+ Err(e) => {
+ if !inlined {
+ let _ = self.delete_blob(&stored_bytes).await;
+ }
+ match e {
+ sqlx::Error::Database(ref db_err) if db_err.is_unique_violation() => Err(
+ StorageError::AlreadyExists(scope.to_string(), filename.to_string()),
+ ),
+ _ => Err(e.into()),
+ }
+ }
+ }
}
- pub async fn put(&self, filename: &str, entry: &File) -> Result<(), io::Error> {
- let data_path = self.data_path(filename)?;
- fs::write(&data_path, &entry.bytes).await?;
+ async fn write_blob(
+ &self,
+ scope: &str,
+ filename: &str,
+ bytes: &bytes::Bytes,
+ ) -> Result<Vec<u8>, StorageError> {
+ let hash = hash_blob(scope, filename, bytes);
+ let hash_hex = hex::encode(&hash);
+
+ let blob_dir = self.storage_dir.join(&hash_hex[0..2]).join(&hash_hex[2..4]);
+ let blob_path = blob_dir.join(&hash_hex);
+
+ tokio::fs::create_dir_all(&blob_dir).await?;
+
+ let mut file = tokio::fs::OpenOptions::new()
+ .write(true)
+ .create_new(true)
+ .open(&blob_path)
+ .await
+ .map_err(|e| {
+ if e.kind() == std::io::ErrorKind::AlreadyExists {
+ StorageError::AlreadyExists(scope.to_string(), filename.to_string())
+ } else {
+ StorageError::IoError(e)
+ }
+ })?;
+
+ file.write_all(bytes).await?;
+ file.sync_all().await?;
+
+ Ok(hash)
+ }
+
+ fn blob_path(&self, hash: &[u8]) -> PathBuf {
+ let hash_hex = hex::encode(hash);
+ self.storage_dir
+ .join(&hash_hex[0..2])
+ .join(&hash_hex[2..4])
+ .join(&hash_hex)
+ }
+
+ async fn delete_blob(&self, hash: &[u8]) -> Result<(), StorageError> {
+ tokio::fs::remove_file(self.blob_path(hash)).await?;
Ok(())
}
- fn data_path(&self, filename: &str) -> Result<path::PathBuf, io::Error> {
- Ok(self.config.dirname().join(filename))
+ pub async fn get(&self, scope: &str, filename: &str) -> Result<Option<File>, StorageError> {
+ let file: Option<File> =
+ query_as("SELECT * FROM datafiles WHERE scope = ?1 AND file_name = ?2")
+ .bind(scope)
+ .bind(filename)
+ .fetch_optional(&self.sqlite)
+ .await?;
+
+ match file {
+ None => Ok(None),
+ Some(file) if file.inlined => Ok(Some(file)),
+ Some(mut file) => {
+ let bytes = self.read_blob(scope, filename, &file.file_bytes).await?;
+ file.file_bytes = Blob(bytes);
+ Ok(Some(file))
+ }
+ }
+ }
+
+ async fn read_blob(
+ &self,
+ scope: &str,
+ filename: &str,
+ expected_hash: &[u8],
+ ) -> Result<bytes::Bytes, StorageError> {
+ let bytes = tokio::fs::read(self.blob_path(expected_hash)).await?;
+
+ let actual_hash = hash_blob(scope, filename, &bytes);
+ if actual_hash != expected_hash {
+ return Err(StorageError::IntegrityError);
+ }
+
+ Ok(bytes::Bytes::from(bytes))
}
}
generated by Gilti