handle io errors + test p1
This commit is contained in:
parent
2d429c75b7
commit
b53680bf58
SSH Key Fingerprint:
SHA256:mf+pUTmK92Y57BuCjlkBdd82LqztTfDCQIUp0fCKABc
|
|
@ -17,6 +17,15 @@ version = "2.0.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
|
|
@ -186,23 +195,11 @@ version = "0.22.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
|
|
@ -226,11 +223,11 @@ dependencies = [
|
|||
"color-eyre",
|
||||
"dashmap",
|
||||
"headers",
|
||||
"heed",
|
||||
"hmac",
|
||||
"http",
|
||||
"http-body-util",
|
||||
"img-parts",
|
||||
"rand 0.9.4",
|
||||
"rand",
|
||||
"serde",
|
||||
"serde_with",
|
||||
"sha2",
|
||||
|
|
@ -239,10 +236,11 @@ dependencies = [
|
|||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"toml",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"tracing-test",
|
||||
"twox-hash",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -251,12 +249,6 @@ version = "3.20.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.1"
|
||||
|
|
@ -342,15 +334,6 @@ dependencies = [
|
|||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-queue"
|
||||
version = "0.3.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115"
|
||||
dependencies = [
|
||||
"crossbeam-utils",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crossbeam-utils"
|
||||
version = "0.8.21"
|
||||
|
|
@ -436,26 +419,6 @@ dependencies = [
|
|||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "doxygen-rs"
|
||||
version = "0.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9"
|
||||
dependencies = [
|
||||
"phf",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dyn-clone"
|
||||
version = "1.0.20"
|
||||
|
|
@ -494,12 +457,6 @@ version = "0.1.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
|
||||
|
||||
[[package]]
|
||||
name = "fnv"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
|
||||
|
||||
[[package]]
|
||||
name = "form_urlencoded"
|
||||
version = "1.2.2"
|
||||
|
|
@ -576,25 +533,6 @@ version = "0.32.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
|
||||
|
||||
[[package]]
|
||||
name = "h2"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733"
|
||||
dependencies = [
|
||||
"atomic-waker",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
"http",
|
||||
"indexmap 2.14.0",
|
||||
"slab",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.12.3"
|
||||
|
|
@ -637,44 +575,6 @@ dependencies = [
|
|||
"http",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heed"
|
||||
version = "0.22.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad82d6598ccf1dac15c8b758a1bd282b755b6776be600429176757190a1b0202"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"heed-types",
|
||||
"libc",
|
||||
"lmdb-master-sys",
|
||||
"once_cell",
|
||||
"page_size",
|
||||
"serde",
|
||||
"synchronoise",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heed-traits"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb3130048d404c57ce5a1ac61a903696e8fcde7e8c2991e9fcfc1f27c3ef74ff"
|
||||
|
||||
[[package]]
|
||||
name = "heed-types"
|
||||
version = "0.21.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13c255bdf46e07fb840d120a36dcc81f385140d7191c76a7391672675c01a55d"
|
||||
dependencies = [
|
||||
"bincode",
|
||||
"byteorder",
|
||||
"heed-traits",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hex"
|
||||
version = "0.4.3"
|
||||
|
|
@ -745,7 +645,6 @@ dependencies = [
|
|||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"h2",
|
||||
"http",
|
||||
"http-body",
|
||||
"httparse",
|
||||
|
|
@ -795,115 +694,12 @@ dependencies = [
|
|||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"potential_utf",
|
||||
"utf8_iter",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_locale_core"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"litemap",
|
||||
"tinystr",
|
||||
"writeable",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_normalizer_data",
|
||||
"icu_properties",
|
||||
"icu_provider",
|
||||
"smallvec",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_normalizer_data"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38"
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de"
|
||||
dependencies = [
|
||||
"icu_collections",
|
||||
"icu_locale_core",
|
||||
"icu_properties_data",
|
||||
"icu_provider",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_properties_data"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14"
|
||||
|
||||
[[package]]
|
||||
name = "icu_provider"
|
||||
version = "2.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"icu_locale_core",
|
||||
"writeable",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerotrie",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "idna"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
|
||||
dependencies = [
|
||||
"idna_adapter",
|
||||
"smallvec",
|
||||
"utf8_iter",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "idna_adapter"
|
||||
version = "1.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714"
|
||||
dependencies = [
|
||||
"icu_normalizer",
|
||||
"icu_properties",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "img-parts"
|
||||
version = "0.3.3"
|
||||
|
|
@ -974,23 +770,6 @@ version = "0.2.186"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66"
|
||||
|
||||
[[package]]
|
||||
name = "litemap"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0"
|
||||
|
||||
[[package]]
|
||||
name = "lmdb-master-sys"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aaeb9bd22e73bd1babffff614994b341e9b2008de7bb73bf1f7e9154f1978f8b"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"doxygen-rs",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.14"
|
||||
|
|
@ -1006,6 +785,15 @@ version = "0.4.29"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
|
||||
dependencies = [
|
||||
"regex-automata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.8.4"
|
||||
|
|
@ -1089,16 +877,6 @@ version = "4.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d211803b9b6b570f68772237e415a029d5a50c65d382910b879fb19d3271f94d"
|
||||
|
||||
[[package]]
|
||||
name = "page_size"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.12"
|
||||
|
|
@ -1118,48 +896,6 @@ version = "2.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
|
||||
|
||||
[[package]]
|
||||
name = "phf"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078"
|
||||
dependencies = [
|
||||
"phf_macros",
|
||||
"phf_shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_generator"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d"
|
||||
dependencies = [
|
||||
"phf_shared",
|
||||
"rand 0.8.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_macros"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216"
|
||||
dependencies = [
|
||||
"phf_generator",
|
||||
"phf_shared",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "phf_shared"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5"
|
||||
dependencies = [
|
||||
"siphasher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pin-project-lite"
|
||||
version = "0.2.17"
|
||||
|
|
@ -1172,15 +908,6 @@ version = "1.13.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49"
|
||||
|
||||
[[package]]
|
||||
name = "potential_utf"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564"
|
||||
dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "powerfmt"
|
||||
version = "0.2.0"
|
||||
|
|
@ -1220,15 +947,6 @@ version = "5.3.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
||||
dependencies = [
|
||||
"rand_core 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.9.4"
|
||||
|
|
@ -1236,7 +954,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea"
|
||||
dependencies = [
|
||||
"rand_chacha",
|
||||
"rand_core 0.9.5",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1246,15 +964,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core 0.9.5",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.9.5"
|
||||
|
|
@ -1293,6 +1005,23 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-automata"
|
||||
version = "0.4.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.8.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.27"
|
||||
|
|
@ -1311,15 +1040,6 @@ version = "1.0.23"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "0.9.0"
|
||||
|
|
@ -1406,11 +1126,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "serde_spanned"
|
||||
version = "0.6.9"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
|
||||
checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1503,12 +1223,6 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "siphasher"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ee5873ec9cce0195efcb7a4e9507a04cd49aec9c83d0389df45b1ef7ba2e649"
|
||||
|
||||
[[package]]
|
||||
name = "slab"
|
||||
version = "0.4.12"
|
||||
|
|
@ -1531,12 +1245,6 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "stable_deref_trait"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
|
|
@ -1566,26 +1274,6 @@ version = "1.0.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
|
||||
|
||||
[[package]]
|
||||
name = "synchronoise"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3dbc01390fc626ce8d1cffe3376ded2b72a11bb70e1c75f404a210e4daa4def2"
|
||||
dependencies = [
|
||||
"crossbeam-queue",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.1.9"
|
||||
|
|
@ -1646,16 +1334,6 @@ dependencies = [
|
|||
"time-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinystr"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.52.2"
|
||||
|
|
@ -1709,44 +1387,42 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.8.23"
|
||||
version = "0.9.12+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
|
||||
checksum = "cf92845e79fc2e2def6a5d828f0801e29a2f8acc037becc5ab08595c7d5e9863"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"indexmap 2.14.0",
|
||||
"serde_core",
|
||||
"serde_spanned",
|
||||
"toml_datetime",
|
||||
"toml_edit",
|
||||
"toml_parser",
|
||||
"toml_writer",
|
||||
"winnow 0.7.15",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "0.6.11"
|
||||
version = "0.7.5+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
|
||||
checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.22.27"
|
||||
name = "toml_parser"
|
||||
version = "1.1.2+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
|
||||
checksum = "a2abe9b86193656635d2411dc43050282ca48aa31c2451210f4202550afb7526"
|
||||
dependencies = [
|
||||
"indexmap 2.14.0",
|
||||
"serde",
|
||||
"serde_spanned",
|
||||
"toml_datetime",
|
||||
"toml_write",
|
||||
"winnow",
|
||||
"winnow 1.0.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_write"
|
||||
version = "0.1.2"
|
||||
name = "toml_writer"
|
||||
version = "1.1.1+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
|
||||
checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db"
|
||||
|
||||
[[package]]
|
||||
name = "tower"
|
||||
|
|
@ -1836,21 +1512,46 @@ version = "0.3.23"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319"
|
||||
dependencies = [
|
||||
"matchers",
|
||||
"nu-ansi-term",
|
||||
"once_cell",
|
||||
"regex-automata",
|
||||
"sharded-slab",
|
||||
"smallvec",
|
||||
"thread_local",
|
||||
"tracing",
|
||||
"tracing-core",
|
||||
"tracing-log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-test"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19a4c448db514d4f24c5ddb9f73f2ee71bfb24c526cf0c570ba142d1119e0051"
|
||||
dependencies = [
|
||||
"tracing-core",
|
||||
"tracing-subscriber",
|
||||
"tracing-test-macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-test-macro"
|
||||
version = "0.2.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ad06847b7afb65c7866a36664b75c40b895e318cea4f71299f013fb22965329d"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "twox-hash"
|
||||
version = "2.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
|
||||
dependencies = [
|
||||
"rand 0.9.4",
|
||||
"rand",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -1865,24 +1566,6 @@ version = "1.0.24"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
|
||||
|
||||
[[package]]
|
||||
name = "url"
|
||||
version = "2.5.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"
|
||||
dependencies = [
|
||||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "utf8_iter"
|
||||
version = "1.0.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
|
||||
|
||||
[[package]]
|
||||
name = "valuable"
|
||||
version = "0.1.1"
|
||||
|
|
@ -1895,16 +1578,6 @@ version = "0.9.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.11.1+wasi-snapshot-preview1"
|
||||
|
|
@ -1965,37 +1638,6 @@ dependencies = [
|
|||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
|
|
@ -2069,9 +1711,12 @@ name = "winnow"
|
|||
version = "0.7.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee1708bef14716a11bae175f579062d4554d95be2c6829f518df847b7b3fdd0"
|
||||
|
||||
[[package]]
|
||||
name = "wit-bindgen"
|
||||
|
|
@ -2079,35 +1724,6 @@ version = "0.57.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e"
|
||||
|
||||
[[package]]
|
||||
name = "writeable"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4"
|
||||
|
||||
[[package]]
|
||||
name = "yoke"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "abe8c5fda708d9ca3df187cae8bfb9ceda00dd96231bed36e445a1a48e66f9ca"
|
||||
dependencies = [
|
||||
"stable_deref_trait",
|
||||
"yoke-derive",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "yoke-derive"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.48"
|
||||
|
|
@ -2128,60 +1744,6 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69faa1f2a1ea75661980b013019ed6687ed0e83d069bc1114e2cc74c6c04c4df"
|
||||
dependencies = [
|
||||
"zerofrom-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerofrom-derive"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerotrie"
|
||||
version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf"
|
||||
dependencies = [
|
||||
"displaydoc",
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec"
|
||||
version = "0.11.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239"
|
||||
dependencies = [
|
||||
"yoke",
|
||||
"zerofrom",
|
||||
"zerovec-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerovec-derive"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmij"
|
||||
version = "1.0.21"
|
||||
|
|
|
|||
16
Cargo.toml
16
Cargo.toml
|
|
@ -9,7 +9,7 @@ tikv-jemalloc-sys = { opt-level = 3 }
|
|||
[dependencies]
|
||||
argh = "0.1.12"
|
||||
atomic-time = "0.1.4"
|
||||
axum = { version = "0.8.9", features = ["macros", "http2"] }
|
||||
axum = { version = "0.8.9", features = ["macros"] }
|
||||
axum-extra = { version = "0.12.6", default-features = false, features = [
|
||||
"tracing",
|
||||
"typed-header",
|
||||
|
|
@ -19,7 +19,6 @@ bytes = "1"
|
|||
color-eyre = "0.6"
|
||||
dashmap = { version = "6.1.0", features = ["inline"] }
|
||||
headers = "0.4"
|
||||
heed = "0.22.1"
|
||||
hmac = "0.12.1"
|
||||
http = "1.2"
|
||||
img-parts = "0.3"
|
||||
|
|
@ -32,16 +31,25 @@ tokio = { version = "1", features = [
|
|||
"macros",
|
||||
"net",
|
||||
"fs",
|
||||
"io-util",
|
||||
"signal",
|
||||
"test-util",
|
||||
] }
|
||||
tokio-stream = "0.1"
|
||||
tokio-util = { version = "0.7", features = ["io"] }
|
||||
toml = "0.8.2"
|
||||
toml = { version = "0.9", default-features = false, features = [
|
||||
"std",
|
||||
"parse",
|
||||
"serde",
|
||||
] }
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = "0.3"
|
||||
twox-hash = "2"
|
||||
walkdir = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
http-body-util = "0.1"
|
||||
tower = "0.5"
|
||||
tracing-test = "0.2"
|
||||
|
||||
[target.'cfg(not(target_env = "msvc"))'.dependencies]
|
||||
tikv-jemallocator = "0.6"
|
||||
|
|
|
|||
254
src/cache.rs
254
src/cache.rs
|
|
@ -1,15 +1,22 @@
|
|||
use std::{
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
time::{Duration, SystemTime},
|
||||
sync::atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use atomic_time::AtomicSystemTime;
|
||||
use bytes::Bytes;
|
||||
use color_eyre::eyre::{self, bail};
|
||||
use dashmap::{DashMap, mapref::one::Ref};
|
||||
use tokio::time;
|
||||
|
||||
use crate::config;
|
||||
|
||||
#[cfg(not(test))]
|
||||
use atomic_time::AtomicSystemTime;
|
||||
#[cfg(not(test))]
|
||||
use std::time::SystemTime;
|
||||
#[cfg(test)]
|
||||
use tests::{MockAtomicSystemTime as AtomicSystemTime, MockSystemTime as SystemTime};
|
||||
|
||||
/// An entry stored in the cache.
|
||||
///
|
||||
/// It contains basic metadata and the actual value.
|
||||
|
|
@ -61,18 +68,29 @@ pub struct Cache {
|
|||
/// Total length of data stored in cache currently
|
||||
length: AtomicUsize,
|
||||
|
||||
/// How many times the scanner has ran,
|
||||
/// for testing purposes
|
||||
scan_count: AtomicU64,
|
||||
|
||||
/// How should it behave
|
||||
cfg: config::CacheConfig,
|
||||
}
|
||||
|
||||
impl Cache {
|
||||
pub fn with_config(cfg: config::CacheConfig) -> Self {
|
||||
Self {
|
||||
pub fn with_config(cfg: config::CacheConfig) -> eyre::Result<Self> {
|
||||
// Sanity check chosen limits
|
||||
if cfg.mem_capacity < cfg.max_length {
|
||||
bail!("`max_length` should not exceed `mem_capacity`");
|
||||
}
|
||||
|
||||
// Return
|
||||
Ok(Self {
|
||||
map: DashMap::with_capacity(64),
|
||||
length: AtomicUsize::new(0),
|
||||
scan_count: AtomicU64::new(0),
|
||||
|
||||
cfg,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Figure out who should be bumped out of cache next
|
||||
|
|
@ -221,11 +239,13 @@ impl Cache {
|
|||
/// letting each entry keep track of expiry with its own task
|
||||
pub async fn scanner(&self) {
|
||||
let mut interval = time::interval(self.cfg.scan_freq);
|
||||
interval.set_missed_tick_behavior(time::MissedTickBehavior::Skip);
|
||||
interval.tick().await; // Skip first tick
|
||||
|
||||
loop {
|
||||
// We put this first so that it doesn't scan the instant the server starts
|
||||
interval.tick().await;
|
||||
self.scan_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
// Save current timestamp so we aren't retrieving it constantly
|
||||
// If we don't do this it'll be a LOT of system api calls
|
||||
|
|
@ -257,3 +277,225 @@ impl Cache {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use bytes::Bytes;
|
||||
|
||||
use crate::{cache::Cache, config::CacheConfig};
|
||||
|
||||
thread_local! {
|
||||
static MOCK_CLOCK: AtomicU64 = AtomicU64::new(0);
|
||||
}
|
||||
fn get_clock() -> u64 {
|
||||
MOCK_CLOCK.with(|mc| mc.load(Ordering::Relaxed))
|
||||
}
|
||||
fn advance_clock(ms: u64) {
|
||||
MOCK_CLOCK.with(|mc| mc.fetch_add(ms, Ordering::Relaxed));
|
||||
}
|
||||
async fn advance_clock_async(ms: u64) {
|
||||
advance_clock(ms);
|
||||
tokio::time::advance(Duration::from_millis(ms)).await;
|
||||
tokio::task::yield_now().await; // make sure scanner tick runs
|
||||
}
|
||||
|
||||
pub struct MockSystemTimeError;
|
||||
|
||||
#[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug)]
|
||||
pub(super) struct MockSystemTime(u64);
|
||||
impl MockSystemTime {
|
||||
pub fn now() -> Self {
|
||||
Self(get_clock())
|
||||
}
|
||||
|
||||
pub fn duration_since(
|
||||
&self,
|
||||
earlier: MockSystemTime,
|
||||
) -> Result<Duration, MockSystemTimeError> {
|
||||
if self.0 >= earlier.0 {
|
||||
Ok(Duration::from_millis(self.0 - earlier.0))
|
||||
} else {
|
||||
Err(MockSystemTimeError)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn elapsed(&self) -> Result<Duration, MockSystemTimeError> {
|
||||
Self::now().duration_since(*self)
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct MockAtomicSystemTime(AtomicU64);
|
||||
impl MockAtomicSystemTime {
|
||||
pub fn now() -> Self {
|
||||
Self(AtomicU64::new(get_clock()))
|
||||
}
|
||||
|
||||
pub fn load(&self, order: Ordering) -> MockSystemTime {
|
||||
MockSystemTime(self.0.load(order))
|
||||
}
|
||||
pub fn store(&self, system_time: MockSystemTime, order: Ordering) {
|
||||
self.0.store(system_time.0, order);
|
||||
}
|
||||
}
|
||||
|
||||
const KEY: &str = "abcdef.png";
|
||||
const VALUE: Bytes = Bytes::from_static(&[0, 1, 2, 3, 4, 5, 6, 7]);
|
||||
|
||||
fn simple() -> Cache {
|
||||
return Cache::with_config(CacheConfig {
|
||||
max_length: 10_000_000,
|
||||
mem_capacity: 100_000_000,
|
||||
scan_freq: Duration::from_secs(5),
|
||||
upload_lifetime: Duration::from_secs(15),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn scanning() -> Arc<Cache> {
|
||||
let cache = Arc::new(simple());
|
||||
|
||||
tokio::spawn({
|
||||
let cache = cache.clone();
|
||||
async move { cache.scanner().await }
|
||||
});
|
||||
// allow 0ms scanner tick to run
|
||||
tokio::task::yield_now().await;
|
||||
|
||||
cache
|
||||
}
|
||||
|
||||
/// Make sure that cache use check
|
||||
/// decides properly for multiple lengths
|
||||
#[test]
|
||||
fn will_use() {
|
||||
let cache = simple();
|
||||
|
||||
// use something
|
||||
assert!(cache.will_use(4_000_000));
|
||||
|
||||
// don't use something
|
||||
assert!(!cache.will_use(12_000_001));
|
||||
|
||||
// use something edge
|
||||
assert!(cache.will_use(10_000_000));
|
||||
|
||||
// use something mini
|
||||
assert!(cache.will_use(0));
|
||||
}
|
||||
|
||||
/// Make sure that [`Cache::add`]'s return value
|
||||
/// is `false` when an entry was replaced
|
||||
#[test]
|
||||
fn store_replacement() {
|
||||
let cache = simple();
|
||||
|
||||
// store
|
||||
assert!(cache.add(KEY, VALUE));
|
||||
|
||||
// store w replace
|
||||
assert!(!cache.add(KEY, VALUE));
|
||||
}
|
||||
|
||||
/// Make sure that the scanner ticks at
|
||||
/// the right times, and removes entries
|
||||
/// when expected.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn store_expire_on_hit_with_scanner() {
|
||||
let cache = scanning().await;
|
||||
|
||||
// store
|
||||
assert!(cache.add(KEY, VALUE));
|
||||
|
||||
// get again so that scanner timing
|
||||
// doesn't align w expiration
|
||||
advance_clock_async(4999).await;
|
||||
assert_eq!(cache.scan_count.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(cache.get(KEY), Some(VALUE));
|
||||
|
||||
// next scanner tick
|
||||
advance_clock_async(1).await;
|
||||
assert_eq!(cache.scan_count.load(Ordering::Relaxed), 1);
|
||||
|
||||
// advance a bit more
|
||||
// make sure we don't expire early
|
||||
advance_clock_async(7000).await;
|
||||
assert_eq!(cache.scan_count.load(Ordering::Relaxed), 2);
|
||||
assert!(cache.map.get(KEY).is_some());
|
||||
|
||||
// advance to next scanner tick
|
||||
advance_clock_async(3000).await;
|
||||
assert_eq!(cache.scan_count.load(Ordering::Relaxed), 3);
|
||||
|
||||
// advance to after expiry
|
||||
advance_clock_async(4999).await;
|
||||
assert_eq!(cache.scan_count.load(Ordering::Relaxed), 3);
|
||||
|
||||
// it should be there because we
|
||||
// offset ourselves by 1ms
|
||||
assert!(cache.map.get(KEY).is_some());
|
||||
assert_eq!(cache.get(KEY), None);
|
||||
}
|
||||
|
||||
/// Make sure that the scanner removes
|
||||
/// expired entries.
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn store_expire_by_scanner() {
|
||||
let cache = scanning().await;
|
||||
|
||||
// store
|
||||
assert!(cache.add(KEY, VALUE));
|
||||
|
||||
// make sure we don't expire early
|
||||
advance_clock_async(6500).await;
|
||||
assert!(cache.map.get(KEY).is_some());
|
||||
|
||||
// advance to after expiry
|
||||
advance_clock_async(8500).await;
|
||||
|
||||
// it should get hit by scanner
|
||||
assert!(cache.map.get(KEY).is_none());
|
||||
}
|
||||
|
||||
/// Make sure that entries expire on hit,
|
||||
/// even when there is no scanner
|
||||
#[test]
|
||||
fn store_get_expire_on_hit() {
|
||||
let cache = simple();
|
||||
|
||||
// store, get
|
||||
let added_at = MockSystemTime::now();
|
||||
assert!(cache.add(KEY, VALUE));
|
||||
assert_eq!(cache.get(KEY), Some(VALUE));
|
||||
|
||||
// get after delay
|
||||
// (upload gets used)
|
||||
advance_clock(2000);
|
||||
assert_eq!(cache.map.get(KEY).unwrap().last_used(), added_at);
|
||||
assert_eq!(cache.get(KEY), Some(VALUE));
|
||||
assert_eq!(
|
||||
cache.map.get(KEY).unwrap().last_used(),
|
||||
MockSystemTime::now()
|
||||
);
|
||||
|
||||
// get after longer delay
|
||||
// (upload should have been used so no expire)
|
||||
advance_clock(14000);
|
||||
assert_eq!(cache.get(KEY), Some(VALUE));
|
||||
assert_eq!(
|
||||
cache.map.get(KEY).unwrap().last_used(),
|
||||
MockSystemTime::now()
|
||||
);
|
||||
|
||||
// get after expiration
|
||||
advance_clock(15000);
|
||||
assert!(cache.get(KEY).is_none());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@ use tracing_subscriber::filter::LevelFilter;
|
|||
#[derive(Deserialize)]
|
||||
pub struct Config {
|
||||
pub engine: EngineConfig,
|
||||
pub cache: CacheConfig,
|
||||
pub disk: DiskConfig,
|
||||
pub http: HttpConfig,
|
||||
pub logger: LoggerConfig,
|
||||
}
|
||||
|
|
@ -33,12 +35,6 @@ pub struct EngineConfig {
|
|||
/// If this secret is leaked, anyone can delete any file. Be careful!!!
|
||||
pub deletion_secret: Option<String>,
|
||||
|
||||
/// Configuration for disk system
|
||||
pub disk: DiskConfig,
|
||||
|
||||
/// Configuration for cache system
|
||||
pub cache: CacheConfig,
|
||||
|
||||
/// Maximum size of an upload that will be accepted.
|
||||
/// Files above this size can not be uploaded.
|
||||
pub max_upload_len: Option<u64>,
|
||||
|
|
|
|||
54
src/disk.rs
54
src/disk.rs
|
|
@ -6,7 +6,6 @@ use tokio::{
|
|||
io::{self, AsyncWriteExt},
|
||||
sync::mpsc,
|
||||
};
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use crate::config;
|
||||
|
||||
|
|
@ -22,11 +21,14 @@ impl Disk {
|
|||
}
|
||||
|
||||
/// Counts the number of files saved to disk we have
|
||||
pub fn count(&self) -> usize {
|
||||
WalkDir::new(&self.cfg.save_path)
|
||||
.min_depth(1)
|
||||
.into_iter()
|
||||
.count()
|
||||
pub fn count(&self) -> io::Result<usize> {
|
||||
std::fs::read_dir(&self.cfg.save_path)?.try_fold(0, |acc, x| {
|
||||
Ok(if x?.file_type()?.is_file() {
|
||||
acc + 1
|
||||
} else {
|
||||
acc
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Formats the path on disk for a `saved_name`.
|
||||
|
|
@ -67,31 +69,47 @@ impl Disk {
|
|||
}
|
||||
|
||||
/// Create a background I/O task
|
||||
pub fn start_save(&self, saved_name: &str) -> mpsc::UnboundedSender<Bytes> {
|
||||
pub fn start_save<
|
||||
Fut: Future + Send + 'static,
|
||||
F: FnOnce(io::Error) -> Fut + Send + 'static,
|
||||
>(
|
||||
&self,
|
||||
saved_name: &str,
|
||||
fail_callback: F,
|
||||
) -> mpsc::Sender<Bytes> {
|
||||
// start a task that handles saving files to disk (we can save to cache/disk in parallel that way)
|
||||
let (tx, mut rx): (mpsc::UnboundedSender<Bytes>, mpsc::UnboundedReceiver<Bytes>) =
|
||||
mpsc::unbounded_channel();
|
||||
// a large buffer size is chosen so uploads can be received quickly,
|
||||
// but with less possibility of running out of memory.
|
||||
// (thats probably only possible w very high link speed tho......)
|
||||
let (tx, mut rx): (mpsc::Sender<Bytes>, mpsc::Receiver<Bytes>) = mpsc::channel(30000);
|
||||
|
||||
let p = self.path_for(saved_name);
|
||||
|
||||
tokio::spawn(async move {
|
||||
// create file to save upload to
|
||||
let file = File::create(p).await;
|
||||
|
||||
if let Err(err) = file {
|
||||
tracing::error!(%err, "could not open file! make sure your upload path is valid");
|
||||
return;
|
||||
}
|
||||
let mut file = file.unwrap();
|
||||
let mut file = match File::create(p).await {
|
||||
Ok(f) => f,
|
||||
Err(err) => {
|
||||
tracing::error!(%err, "could not open file! make sure your upload path is valid");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// receive chunks and save them to file
|
||||
while let Some(chunk) = rx.recv().await {
|
||||
tracing::debug!(length = chunk.len(), "writing chunk to disk");
|
||||
if let Err(err) = file.write_all(&chunk).await {
|
||||
tracing::error!(%err, "error while writing file to disk");
|
||||
break;
|
||||
drop(rx);
|
||||
fail_callback(err).await;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// flush to disk
|
||||
// this should catch "no space left on device" i hope...
|
||||
if let Err(err) = file.flush().await {
|
||||
fail_callback(err).await;
|
||||
}
|
||||
});
|
||||
|
||||
tx
|
||||
|
|
|
|||
|
|
@ -90,7 +90,23 @@ pub struct Engine {
|
|||
cache: Arc<cache::Cache>,
|
||||
|
||||
/// An interface to the on-disk upload store
|
||||
disk: disk::Disk,
|
||||
disk: Arc<disk::Disk>,
|
||||
}
|
||||
|
||||
/// Wipe out an upload from all storage.
|
||||
/// * Intended for deletion URLs and failed uploads
|
||||
/// * Separated from [`Engine`] for use in [`disk::Disk`]
|
||||
async fn remove(cache: &cache::Cache, disk: &disk::Disk, saved_name: &str) -> eyre::Result<()> {
|
||||
info!(saved_name, "!! removing upload");
|
||||
|
||||
cache.remove(saved_name);
|
||||
disk.remove(saved_name)
|
||||
.await
|
||||
.wrap_err("failed to remove file from disk")?;
|
||||
|
||||
info!("!! successfully removed upload");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Try to parse a `Range` header into an easier format to work with
|
||||
|
|
@ -173,30 +189,26 @@ fn calculate_hash(len: u64, data_sample: Bytes) -> u128 {
|
|||
|
||||
impl Engine {
|
||||
/// Creates a new instance of the engine
|
||||
pub fn with_config(cfg: config::EngineConfig) -> Self {
|
||||
pub fn new(
|
||||
cfg: config::EngineConfig,
|
||||
cache: Arc<cache::Cache>,
|
||||
disk: disk::Disk,
|
||||
) -> std::io::Result<Self> {
|
||||
let deletion_hmac = cfg
|
||||
.deletion_secret
|
||||
.as_ref()
|
||||
.map(|s| HmacSha256::new_from_slice(s.as_bytes()).unwrap());
|
||||
|
||||
let cache = cache::Cache::with_config(cfg.cache.clone());
|
||||
let disk = disk::Disk::with_config(cfg.disk.clone());
|
||||
|
||||
let cache = Arc::new(cache);
|
||||
|
||||
let cache_scanner = cache.clone();
|
||||
tokio::spawn(async move { cache_scanner.scanner().await });
|
||||
|
||||
Self {
|
||||
Ok(Self {
|
||||
// initialise our cached upload count. this doesn't include temp uploads!
|
||||
upl_count: AtomicUsize::new(disk.count()),
|
||||
upl_count: AtomicUsize::new(disk.count()?),
|
||||
deletion_hmac,
|
||||
|
||||
cfg,
|
||||
|
||||
cache,
|
||||
disk,
|
||||
}
|
||||
disk: Arc::new(disk),
|
||||
})
|
||||
}
|
||||
|
||||
/// Fetch an upload.
|
||||
|
|
@ -378,17 +390,7 @@ impl Engine {
|
|||
///
|
||||
/// (Intended for deletion URLs and failed uploads)
|
||||
pub async fn remove(&self, saved_name: &str) -> eyre::Result<()> {
|
||||
info!(saved_name, "!! removing upload");
|
||||
|
||||
self.cache.remove(saved_name);
|
||||
self.disk
|
||||
.remove(saved_name)
|
||||
.await
|
||||
.wrap_err("failed to remove file from disk")?;
|
||||
|
||||
info!("!! successfully removed upload");
|
||||
|
||||
Ok(())
|
||||
remove(&self.cache, &self.disk, saved_name).await
|
||||
}
|
||||
|
||||
/// Save a file to disk, and optionally cache.
|
||||
|
|
@ -412,7 +414,19 @@ impl Engine {
|
|||
|
||||
// don't begin a disk save if we're using temporary lifetimes
|
||||
let tx = if lifetime.is_none() {
|
||||
Some(self.disk.start_save(saved_name))
|
||||
Some(self.disk.start_save(saved_name, {
|
||||
let cache = self.cache.clone();
|
||||
let disk = self.disk.clone();
|
||||
let saved_name = saved_name.to_string();
|
||||
|
||||
async move |err| {
|
||||
// try to delete the failed upload
|
||||
error!(%saved_name, %err, "error while saving file to disk");
|
||||
if let Err(err) = remove(&cache, &disk, &saved_name).await {
|
||||
error!(%saved_name, %err, "IO error callback failed to remove upload");
|
||||
}
|
||||
}
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
|
@ -445,6 +459,7 @@ impl Engine {
|
|||
if !coalesce_and_strip && let Some(ref tx) = tx {
|
||||
debug!("sending chunk to i/o task");
|
||||
tx.send(chunk.clone())
|
||||
.await
|
||||
.wrap_err("failed to send chunk to i/o task!")?;
|
||||
}
|
||||
|
||||
|
|
@ -499,6 +514,7 @@ impl Engine {
|
|||
if let Some(ref tx) = tx {
|
||||
debug!("sending filled buffer to i/o task");
|
||||
tx.send(data.clone())
|
||||
.await
|
||||
.wrap_err("failed to send coalesced buffer to i/o task!")?;
|
||||
}
|
||||
|
||||
|
|
|
|||
16
src/main.rs
16
src/main.rs
|
|
@ -23,6 +23,8 @@ mod view;
|
|||
#[cfg(not(target_env = "msvc"))]
|
||||
use tikv_jemallocator::Jemalloc;
|
||||
|
||||
use crate::{cache::Cache, disk::Disk};
|
||||
|
||||
#[cfg(not(target_env = "msvc"))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: Jemalloc = Jemalloc;
|
||||
|
|
@ -72,7 +74,7 @@ async fn main() -> eyre::Result<()> {
|
|||
|
||||
// Check config
|
||||
{
|
||||
let save_path = cfg.engine.disk.save_path.clone();
|
||||
let save_path = cfg.disk.save_path.clone();
|
||||
if !save_path.exists() || !save_path.is_dir() {
|
||||
bail!("the save path does not exist or is not a directory! this is invalid");
|
||||
}
|
||||
|
|
@ -81,8 +83,18 @@ async fn main() -> eyre::Result<()> {
|
|||
warn!("engine upload_key is empty! no key will be required for uploading new files");
|
||||
}
|
||||
|
||||
// Create backends
|
||||
let cache = Arc::new(Cache::with_config(cfg.cache)?);
|
||||
let disk = Disk::with_config(cfg.disk);
|
||||
|
||||
// Start cache scanner
|
||||
tokio::spawn({
|
||||
let cache = cache.clone();
|
||||
async move { cache.scanner().await }
|
||||
});
|
||||
|
||||
// Create engine
|
||||
let engine = Engine::with_config(cfg.engine);
|
||||
let engine = Engine::new(cfg.engine, cache, disk)?;
|
||||
|
||||
// Build main router
|
||||
let app = router(engine);
|
||||
|
|
|
|||
Loading…
Reference in New Issue