diff --git a/Cargo.lock b/Cargo.lock index 4c6f862..0b89258 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + [[package]] name = "android_system_properties" version = "0.1.5" @@ -186,23 +195,11 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" -dependencies = [ - "serde_core", -] [[package]] name = "block-buffer" @@ -226,11 +223,11 @@ dependencies = [ "color-eyre", "dashmap", "headers", - "heed", "hmac", "http", + "http-body-util", "img-parts", - "rand 0.9.4", + "rand", "serde", "serde_with", "sha2", @@ -239,10 +236,11 @@ dependencies = [ "tokio-stream", "tokio-util", "toml", + "tower", "tracing", "tracing-subscriber", + "tracing-test", "twox-hash", - "walkdir", ] [[package]] @@ -251,12 +249,6 @@ version = "3.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - [[package]] name = "bytes" version = "1.11.1" @@ -342,15 +334,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-queue" -version = "0.3.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -436,26 +419,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "doxygen-rs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" -dependencies = [ - "phf", -] - [[package]] name = "dyn-clone" version = "1.0.20" @@ -494,12 +457,6 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "form_urlencoded" version = "1.2.2" @@ -576,25 +533,6 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" -[[package]] -name = "h2" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" -dependencies = [ - "atomic-waker", - "bytes", - "fnv", - "futures-core", - "futures-sink", - "http", - "indexmap 2.14.0", - "slab", - "tokio", - "tokio-util", - "tracing", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -637,44 +575,6 @@ dependencies = [ "http", ] -[[package]] -name = "heed" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad82d6598ccf1dac15c8b758a1bd282b755b6776be600429176757190a1b0202" -dependencies = [ - "bitflags", - "byteorder", - "heed-traits", - "heed-types", - "libc", - "lmdb-master-sys", - "once_cell", - "page_size", - "serde", - "synchronoise", - "url", -] - -[[package]] -name = "heed-traits" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff" - -[[package]] -name = "heed-types" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d" -dependencies = [ - "bincode", - "byteorder", - "heed-traits", - "serde", - "serde_json", -] - [[package]] name = "hex" version = "0.4.3" @@ -745,7 +645,6 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", "http", "http-body", "httparse", @@ -795,115 +694,12 @@ dependencies = [ "cc", ] -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - [[package]] name = "ident_case" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - [[package]] name = "img-parts" version = "0.3.3" @@ -974,23 +770,6 @@ version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "lmdb-master-sys" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaeb9bd22e73bd1babffff614994b341e9b2008de7bb73bf1f7e9154f1978f8b" -dependencies = [ - "cc", - "doxygen-rs", - "libc", -] - [[package]] name = "lock_api" version = "0.4.14" @@ -1006,6 +785,15 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -1089,16 +877,6 @@ version = "4.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d" -[[package]] -name = "page_size" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" -dependencies = [ - "libc", - "winapi", -] - [[package]] name = "parking_lot_core" version = "0.9.12" @@ -1118,48 +896,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "phf" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" -dependencies = [ - "phf_shared", - "rand 0.8.6", -] - -[[package]] -name = "phf_macros" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "phf_shared" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project-lite" version = "0.2.17" @@ -1172,15 +908,6 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - [[package]] name = "powerfmt" version = "0.2.0" @@ -1220,15 +947,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "rand" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand" version = "0.9.4" @@ -1236,7 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha", - "rand_core 0.9.5", + "rand_core", ] [[package]] @@ -1246,15 +964,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", - "rand_core 0.9.5", + "rand_core", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" - [[package]] name = "rand_core" version = "0.9.5" @@ -1293,6 +1005,23 @@ dependencies = [ "syn", ] +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -1311,15 +1040,6 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - [[package]] name = "schemars" version = "0.9.0" @@ -1406,11 +1126,11 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "0.6.9" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -1503,12 +1223,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649" - [[package]] name = "slab" version = "0.4.12" @@ -1531,12 +1245,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - [[package]] name = "strsim" version = "0.11.1" @@ -1566,26 +1274,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -[[package]] -name = "synchronoise" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2" -dependencies = [ - "crossbeam-queue", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "thread_local" version = "1.1.9" @@ -1646,16 +1334,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - [[package]] name = "tokio" version = "1.52.2" @@ -1709,44 +1387,42 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.23" +version = "0.9.12+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863" dependencies = [ - "serde", + "indexmap 2.14.0", + "serde_core", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_parser", + "toml_writer", + "winnow 0.7.15", ] [[package]] name = "toml_datetime" -version = "0.6.11" +version = "0.7.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" dependencies = [ - "serde", + "serde_core", ] [[package]] -name = "toml_edit" -version = "0.22.27" +name = "toml_parser" +version = "1.1.2+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526" dependencies = [ - "indexmap 2.14.0", - "serde", - "serde_spanned", - "toml_datetime", - "toml_write", - "winnow", + "winnow 1.0.2", ] [[package]] -name = "toml_write" -version = "0.1.2" +name = "toml_writer" +version = "1.1.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" [[package]] name = "tower" @@ -1836,21 +1512,46 @@ version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] +[[package]] +name = "tracing-test" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051" +dependencies = [ + "tracing-core", + "tracing-subscriber", + "tracing-test-macro", +] + +[[package]] +name = "tracing-test-macro" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "twox-hash" version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.4", + "rand", ] [[package]] @@ -1865,24 +1566,6 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - [[package]] name = "valuable" version = "0.1.1" @@ -1895,16 +1578,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1965,37 +1638,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - [[package]] name = "windows-core" version = "0.62.2" @@ -2069,9 +1711,12 @@ name = "winnow" version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" -dependencies = [ - "memchr", -] + +[[package]] +name = "winnow" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0" [[package]] name = "wit-bindgen" @@ -2079,35 +1724,6 @@ version = "0.57.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - [[package]] name = "zerocopy" version = "0.8.48" @@ -2128,60 +1744,6 @@ dependencies = [ "syn", ] -[[package]] -name = "zerofrom" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "synstructure", -] - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 64b4a8e..ba99b51 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ tikv-jemalloc-sys = { opt-level = 3 } [dependencies] argh = "0.1.12" atomic-time = "0.1.4" -axum = { version = "0.8.9", features = ["macros", "http2"] } +axum = { version = "0.8.9", features = ["macros"] } axum-extra = { version = "0.12.6", default-features = false, features = [ "tracing", "typed-header", @@ -19,7 +19,6 @@ bytes = "1" color-eyre = "0.6" dashmap = { version = "6.1.0", features = ["inline"] } headers = "0.4" -heed = "0.22.1" hmac = "0.12.1" http = "1.2" img-parts = "0.3" @@ -32,16 +31,25 @@ tokio = { version = "1", features = [ "macros", "net", "fs", + "io-util", "signal", "test-util", ] } tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["io"] } -toml = "0.8.2" +toml = { version = "0.9", default-features = false, features = [ + "std", + "parse", + "serde", +] } tracing = "0.1" tracing-subscriber = "0.3" twox-hash = "2" -walkdir = "2" + +[dev-dependencies] +http-body-util = "0.1" +tower = "0.5" +tracing-test = "0.2" [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = "0.6" diff --git a/src/cache.rs b/src/cache.rs index 3e0bdb0..45aaf60 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,15 +1,22 @@ use std::{ - sync::atomic::{AtomicUsize, Ordering}, - time::{Duration, SystemTime}, + sync::atomic::{AtomicU64, AtomicUsize, Ordering}, + time::Duration, }; -use atomic_time::AtomicSystemTime; use bytes::Bytes; +use color_eyre::eyre::{self, bail}; use dashmap::{DashMap, mapref::one::Ref}; use tokio::time; use crate::config; +#[cfg(not(test))] +use atomic_time::AtomicSystemTime; +#[cfg(not(test))] +use std::time::SystemTime; +#[cfg(test)] +use tests::{MockAtomicSystemTime as AtomicSystemTime, MockSystemTime as SystemTime}; + /// An entry stored in the cache. /// /// It contains basic metadata and the actual value. @@ -61,18 +68,29 @@ pub struct Cache { /// Total length of data stored in cache currently length: AtomicUsize, + /// How many times the scanner has ran, + /// for testing purposes + scan_count: AtomicU64, + /// How should it behave cfg: config::CacheConfig, } impl Cache { - pub fn with_config(cfg: config::CacheConfig) -> Self { - Self { + pub fn with_config(cfg: config::CacheConfig) -> eyre::Result { + // Sanity check chosen limits + if cfg.mem_capacity < cfg.max_length { + bail!("`max_length` should not exceed `mem_capacity`"); + } + + // Return + Ok(Self { map: DashMap::with_capacity(64), length: AtomicUsize::new(0), + scan_count: AtomicU64::new(0), cfg, - } + }) } /// Figure out who should be bumped out of cache next @@ -221,11 +239,13 @@ impl Cache { /// letting each entry keep track of expiry with its own task pub async fn scanner(&self) { let mut interval = time::interval(self.cfg.scan_freq); + interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip); interval.tick().await; // Skip first tick loop { // We put this first so that it doesn't scan the instant the server starts interval.tick().await; + self.scan_count.fetch_add(1, Ordering::Relaxed); // Save current timestamp so we aren't retrieving it constantly // If we don't do this it'll be a LOT of system api calls @@ -257,3 +277,225 @@ impl Cache { } } } + +#[cfg(test)] +mod tests { + use std::{ + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, + }; + + use bytes::Bytes; + + use crate::{cache::Cache, config::CacheConfig}; + + thread_local! { + static MOCK_CLOCK: AtomicU64 = AtomicU64::new(0); + } + fn get_clock() -> u64 { + MOCK_CLOCK.with(|mc| mc.load(Ordering::Relaxed)) + } + fn advance_clock(ms: u64) { + MOCK_CLOCK.with(|mc| mc.fetch_add(ms, Ordering::Relaxed)); + } + async fn advance_clock_async(ms: u64) { + advance_clock(ms); + tokio::time::advance(Duration::from_millis(ms)).await; + tokio::task::yield_now().await; // make sure scanner tick runs + } + + pub struct MockSystemTimeError; + + #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug)] + pub(super) struct MockSystemTime(u64); + impl MockSystemTime { + pub fn now() -> Self { + Self(get_clock()) + } + + pub fn duration_since( + &self, + earlier: MockSystemTime, + ) -> Result { + if self.0 >= earlier.0 { + Ok(Duration::from_millis(self.0 - earlier.0)) + } else { + Err(MockSystemTimeError) + } + } + + pub fn elapsed(&self) -> Result { + Self::now().duration_since(*self) + } + } + + pub(super) struct MockAtomicSystemTime(AtomicU64); + impl MockAtomicSystemTime { + pub fn now() -> Self { + Self(AtomicU64::new(get_clock())) + } + + pub fn load(&self, order: Ordering) -> MockSystemTime { + MockSystemTime(self.0.load(order)) + } + pub fn store(&self, system_time: MockSystemTime, order: Ordering) { + self.0.store(system_time.0, order); + } + } + + const KEY: &str = "abcdef.png"; + const VALUE: Bytes = Bytes::from_static(&[0, 1, 2, 3, 4, 5, 6, 7]); + + fn simple() -> Cache { + return Cache::with_config(CacheConfig { + max_length: 10_000_000, + mem_capacity: 100_000_000, + scan_freq: Duration::from_secs(5), + upload_lifetime: Duration::from_secs(15), + }) + .unwrap(); + } + + async fn scanning() -> Arc { + let cache = Arc::new(simple()); + + tokio::spawn({ + let cache = cache.clone(); + async move { cache.scanner().await } + }); + // allow 0ms scanner tick to run + tokio::task::yield_now().await; + + cache + } + + /// Make sure that cache use check + /// decides properly for multiple lengths + #[test] + fn will_use() { + let cache = simple(); + + // use something + assert!(cache.will_use(4_000_000)); + + // don't use something + assert!(!cache.will_use(12_000_001)); + + // use something edge + assert!(cache.will_use(10_000_000)); + + // use something mini + assert!(cache.will_use(0)); + } + + /// Make sure that [`Cache::add`]'s return value + /// is `false` when an entry was replaced + #[test] + fn store_replacement() { + let cache = simple(); + + // store + assert!(cache.add(KEY, VALUE)); + + // store w replace + assert!(!cache.add(KEY, VALUE)); + } + + /// Make sure that the scanner ticks at + /// the right times, and removes entries + /// when expected. + #[tokio::test(start_paused = true)] + async fn store_expire_on_hit_with_scanner() { + let cache = scanning().await; + + // store + assert!(cache.add(KEY, VALUE)); + + // get again so that scanner timing + // doesn't align w expiration + advance_clock_async(4999).await; + assert_eq!(cache.scan_count.load(Ordering::Relaxed), 0); + assert_eq!(cache.get(KEY), Some(VALUE)); + + // next scanner tick + advance_clock_async(1).await; + assert_eq!(cache.scan_count.load(Ordering::Relaxed), 1); + + // advance a bit more + // make sure we don't expire early + advance_clock_async(7000).await; + assert_eq!(cache.scan_count.load(Ordering::Relaxed), 2); + assert!(cache.map.get(KEY).is_some()); + + // advance to next scanner tick + advance_clock_async(3000).await; + assert_eq!(cache.scan_count.load(Ordering::Relaxed), 3); + + // advance to after expiry + advance_clock_async(4999).await; + assert_eq!(cache.scan_count.load(Ordering::Relaxed), 3); + + // it should be there because we + // offset ourselves by 1ms + assert!(cache.map.get(KEY).is_some()); + assert_eq!(cache.get(KEY), None); + } + + /// Make sure that the scanner removes + /// expired entries. + #[tokio::test(start_paused = true)] + async fn store_expire_by_scanner() { + let cache = scanning().await; + + // store + assert!(cache.add(KEY, VALUE)); + + // make sure we don't expire early + advance_clock_async(6500).await; + assert!(cache.map.get(KEY).is_some()); + + // advance to after expiry + advance_clock_async(8500).await; + + // it should get hit by scanner + assert!(cache.map.get(KEY).is_none()); + } + + /// Make sure that entries expire on hit, + /// even when there is no scanner + #[test] + fn store_get_expire_on_hit() { + let cache = simple(); + + // store, get + let added_at = MockSystemTime::now(); + assert!(cache.add(KEY, VALUE)); + assert_eq!(cache.get(KEY), Some(VALUE)); + + // get after delay + // (upload gets used) + advance_clock(2000); + assert_eq!(cache.map.get(KEY).unwrap().last_used(), added_at); + assert_eq!(cache.get(KEY), Some(VALUE)); + assert_eq!( + cache.map.get(KEY).unwrap().last_used(), + MockSystemTime::now() + ); + + // get after longer delay + // (upload should have been used so no expire) + advance_clock(14000); + assert_eq!(cache.get(KEY), Some(VALUE)); + assert_eq!( + cache.map.get(KEY).unwrap().last_used(), + MockSystemTime::now() + ); + + // get after expiration + advance_clock(15000); + assert!(cache.get(KEY).is_none()); + } +} diff --git a/src/config.rs b/src/config.rs index cae81f2..41aedcb 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,6 +7,8 @@ use tracing_subscriber::filter::LevelFilter; #[derive(Deserialize)] pub struct Config { pub engine: EngineConfig, + pub cache: CacheConfig, + pub disk: DiskConfig, pub http: HttpConfig, pub logger: LoggerConfig, } @@ -33,12 +35,6 @@ pub struct EngineConfig { /// If this secret is leaked, anyone can delete any file. Be careful!!! pub deletion_secret: Option, - /// Configuration for disk system - pub disk: DiskConfig, - - /// Configuration for cache system - pub cache: CacheConfig, - /// Maximum size of an upload that will be accepted. /// Files above this size can not be uploaded. pub max_upload_len: Option, diff --git a/src/disk.rs b/src/disk.rs index cdccb4c..2de56fc 100644 --- a/src/disk.rs +++ b/src/disk.rs @@ -6,7 +6,6 @@ use tokio::{ io::{self, AsyncWriteExt}, sync::mpsc, }; -use walkdir::WalkDir; use crate::config; @@ -22,11 +21,14 @@ impl Disk { } /// Counts the number of files saved to disk we have - pub fn count(&self) -> usize { - WalkDir::new(&self.cfg.save_path) - .min_depth(1) - .into_iter() - .count() + pub fn count(&self) -> io::Result { + std::fs::read_dir(&self.cfg.save_path)?.try_fold(0, |acc, x| { + Ok(if x?.file_type()?.is_file() { + acc + 1 + } else { + acc + }) + }) } /// Formats the path on disk for a `saved_name`. @@ -67,31 +69,47 @@ impl Disk { } /// Create a background I/O task - pub fn start_save(&self, saved_name: &str) -> mpsc::UnboundedSender { + pub fn start_save< + Fut: Future + Send + 'static, + F: FnOnce(io::Error) -> Fut + Send + 'static, + >( + &self, + saved_name: &str, + fail_callback: F, + ) -> mpsc::Sender { // start a task that handles saving files to disk (we can save to cache/disk in parallel that way) - let (tx, mut rx): (mpsc::UnboundedSender, mpsc::UnboundedReceiver) = - mpsc::unbounded_channel(); + // a large buffer size is chosen so uploads can be received quickly, + // but with less possibility of running out of memory. + // (thats probably only possible w very high link speed tho......) + let (tx, mut rx): (mpsc::Sender, mpsc::Receiver) = mpsc::channel(30000); let p = self.path_for(saved_name); tokio::spawn(async move { // create file to save upload to - let file = File::create(p).await; - - if let Err(err) = file { - tracing::error!(%err, "could not open file! make sure your upload path is valid"); - return; - } - let mut file = file.unwrap(); + let mut file = match File::create(p).await { + Ok(f) => f, + Err(err) => { + tracing::error!(%err, "could not open file! make sure your upload path is valid"); + return; + } + }; // receive chunks and save them to file while let Some(chunk) = rx.recv().await { tracing::debug!(length = chunk.len(), "writing chunk to disk"); if let Err(err) = file.write_all(&chunk).await { - tracing::error!(%err, "error while writing file to disk"); - break; + drop(rx); + fail_callback(err).await; + return; } } + + // flush to disk + // this should catch "no space left on device" i hope... + if let Err(err) = file.flush().await { + fail_callback(err).await; + } }); tx diff --git a/src/engine.rs b/src/engine.rs index 153bc7d..7ef1197 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -90,7 +90,23 @@ pub struct Engine { cache: Arc, /// An interface to the on-disk upload store - disk: disk::Disk, + disk: Arc, +} + +/// Wipe out an upload from all storage. +/// * Intended for deletion URLs and failed uploads +/// * Separated from [`Engine`] for use in [`disk::Disk`] +async fn remove(cache: &cache::Cache, disk: &disk::Disk, saved_name: &str) -> eyre::Result<()> { + info!(saved_name, "!! removing upload"); + + cache.remove(saved_name); + disk.remove(saved_name) + .await + .wrap_err("failed to remove file from disk")?; + + info!("!! successfully removed upload"); + + Ok(()) } /// Try to parse a `Range` header into an easier format to work with @@ -173,30 +189,26 @@ fn calculate_hash(len: u64, data_sample: Bytes) -> u128 { impl Engine { /// Creates a new instance of the engine - pub fn with_config(cfg: config::EngineConfig) -> Self { + pub fn new( + cfg: config::EngineConfig, + cache: Arc, + disk: disk::Disk, + ) -> std::io::Result { let deletion_hmac = cfg .deletion_secret .as_ref() .map(|s| HmacSha256::new_from_slice(s.as_bytes()).unwrap()); - let cache = cache::Cache::with_config(cfg.cache.clone()); - let disk = disk::Disk::with_config(cfg.disk.clone()); - - let cache = Arc::new(cache); - - let cache_scanner = cache.clone(); - tokio::spawn(async move { cache_scanner.scanner().await }); - - Self { + Ok(Self { // initialise our cached upload count. this doesn't include temp uploads! - upl_count: AtomicUsize::new(disk.count()), + upl_count: AtomicUsize::new(disk.count()?), deletion_hmac, cfg, cache, - disk, - } + disk: Arc::new(disk), + }) } /// Fetch an upload. @@ -378,17 +390,7 @@ impl Engine { /// /// (Intended for deletion URLs and failed uploads) pub async fn remove(&self, saved_name: &str) -> eyre::Result<()> { - info!(saved_name, "!! removing upload"); - - self.cache.remove(saved_name); - self.disk - .remove(saved_name) - .await - .wrap_err("failed to remove file from disk")?; - - info!("!! successfully removed upload"); - - Ok(()) + remove(&self.cache, &self.disk, saved_name).await } /// Save a file to disk, and optionally cache. @@ -412,7 +414,19 @@ impl Engine { // don't begin a disk save if we're using temporary lifetimes let tx = if lifetime.is_none() { - Some(self.disk.start_save(saved_name)) + Some(self.disk.start_save(saved_name, { + let cache = self.cache.clone(); + let disk = self.disk.clone(); + let saved_name = saved_name.to_string(); + + async move |err| { + // try to delete the failed upload + error!(%saved_name, %err, "error while saving file to disk"); + if let Err(err) = remove(&cache, &disk, &saved_name).await { + error!(%saved_name, %err, "IO error callback failed to remove upload"); + } + } + })) } else { None }; @@ -445,6 +459,7 @@ impl Engine { if !coalesce_and_strip && let Some(ref tx) = tx { debug!("sending chunk to i/o task"); tx.send(chunk.clone()) + .await .wrap_err("failed to send chunk to i/o task!")?; } @@ -499,6 +514,7 @@ impl Engine { if let Some(ref tx) = tx { debug!("sending filled buffer to i/o task"); tx.send(data.clone()) + .await .wrap_err("failed to send coalesced buffer to i/o task!")?; } diff --git a/src/main.rs b/src/main.rs index 8d05ec0..41570df 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,8 @@ mod view; #[cfg(not(target_env = "msvc"))] use tikv_jemallocator::Jemalloc; +use crate::{cache::Cache, disk::Disk}; + #[cfg(not(target_env = "msvc"))] #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; @@ -72,7 +74,7 @@ async fn main() -> eyre::Result<()> { // Check config { - let save_path = cfg.engine.disk.save_path.clone(); + let save_path = cfg.disk.save_path.clone(); if !save_path.exists() || !save_path.is_dir() { bail!("the save path does not exist or is not a directory! this is invalid"); } @@ -81,8 +83,18 @@ async fn main() -> eyre::Result<()> { warn!("engine upload_key is empty! no key will be required for uploading new files"); } + // Create backends + let cache = Arc::new(Cache::with_config(cfg.cache)?); + let disk = Disk::with_config(cfg.disk); + + // Start cache scanner + tokio::spawn({ + let cache = cache.clone(); + async move { cache.scanner().await } + }); + // Create engine - let engine = Engine::with_config(cfg.engine); + let engine = Engine::new(cfg.engine, cache, disk)?; // Build main router let app = router(engine);