From 7cbdbde955548ffe107fc2b0fab750677cd65798 Mon Sep 17 00:00:00 2001 From: minish Date: Fri, 28 Feb 2025 18:54:43 -0500 Subject: [PATCH] v0.2.9 --- Cargo.lock | 149 ++++++++++++++++++-------------------------------- Cargo.toml | 17 ++++-- src/cache.rs | 17 +++--- src/disk.rs | 11 ++-- src/engine.rs | 32 +++++------ src/main.rs | 4 +- src/new.rs | 8 ++- src/view.rs | 17 ++++-- 8 files changed, 115 insertions(+), 140 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e77e7e0..9f9da1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,7 +224,7 @@ dependencies = [ [[package]] name = "breeze" -version = "0.2.8" +version = "0.2.9" dependencies = [ "anyhow", "argh", @@ -237,7 +237,6 @@ dependencies = [ "http", "img-parts", "rand", - "rayon", "serde", "serde_with", "tokio", @@ -320,25 +319,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-deque" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -402,7 +382,6 @@ dependencies = [ "lock_api", "once_cell", "parking_lot_core", - "rayon", ] [[package]] @@ -425,12 +404,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "either" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" - [[package]] name = "equivalent" version = "1.0.1" @@ -467,23 +440,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" -[[package]] -name = "futures-io" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" - -[[package]] -name = "futures-macro" -version = "0.3.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "futures-sink" version = "0.3.31" @@ -503,11 +459,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-core", - "futures-macro", "futures-task", "pin-project-lite", "pin-utils", - "slab", ] [[package]] @@ -522,13 +476,14 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.15" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +checksum = "43a49c392881ce6d5c3b8cb70f98717b7c07aabbdff06687b9030dbfbe2725f8" dependencies = [ "cfg-if", "libc", - "wasi", + "wasi 0.13.3+wasi-0.2.2", + "windows-targets", ] [[package]] @@ -826,7 +781,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", - "wasi", + "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -876,16 +831,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" -[[package]] -name = "parking_lot" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" -dependencies = [ - "lock_api", - "parking_lot_core", -] - [[package]] name = "parking_lot_core" version = "0.9.10" @@ -935,7 +880,7 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -958,20 +903,20 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.5" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" dependencies = [ - "libc", "rand_chacha", "rand_core", + "zerocopy 0.8.17", ] [[package]] name = "rand_chacha" -version = "0.3.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" dependencies = [ "ppv-lite86", "rand_core", @@ -979,31 +924,12 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.4" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +checksum = "b08f3c9802962f7e1b25113931d94f43ed9725bebc59db9d0c3e9a23b67e15ff" dependencies = [ "getrandom", -] - -[[package]] -name = "rayon" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", + "zerocopy 0.8.17", ] [[package]] @@ -1281,7 +1207,6 @@ dependencies = [ "bytes", "libc", "mio", - "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", @@ -1319,12 +1244,8 @@ checksum = "d7fcaa8d55a2bdd6b83ace262b016eca0d79ee02818c5c1bcdf0305114081078" dependencies = [ "bytes", "futures-core", - "futures-io", "futures-sink", - "futures-util", - "hashbrown 0.14.5", "pin-project-lite", - "slab", "tokio", ] @@ -1488,6 +1409,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasi" +version = "0.13.3+wasi-0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26816d2e1a4a36a2940b96c5296ce403917633dff8f3440e9b236ed6f6bacad2" +dependencies = [ + "wit-bindgen-rt", +] + [[package]] name = "wasm-bindgen" version = "0.2.99" @@ -1673,6 +1603,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen-rt" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3268f3d866458b787f390cf61f4bbb563b922d091359f9608842999eaee3943c" +dependencies = [ + "bitflags", +] + [[package]] name = "zerocopy" version = "0.7.35" @@ -1680,7 +1619,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa91407dacce3a68c56de03abe2760159582b846c6a4acd2f456618087f12713" +dependencies = [ + "zerocopy-derive 0.8.17", ] [[package]] @@ -1693,3 +1641,14 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerocopy-derive" +version = "0.8.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06718a168365cad3d5ff0bb133aad346959a2074bd4a85c121255a11304a8626" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 953ad85..b5badd6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "breeze" -version = "0.2.8" +version = "0.2.9" edition = "2021" [dependencies] @@ -12,20 +12,25 @@ axum = { version = "0.8.1", features = ["macros", "http2"] } tower = "0.5" http = "1.2" headers = "0.4" -tokio = { version = "1", features = ["full"] } -tokio-util = { version = "0.7", features = ["full"] } +tokio = { version = "1", features = [ + "rt-multi-thread", + "macros", + "net", + "fs", + "signal", +] } +tokio-util = { version = "0.7", features = ["io"] } tokio-stream = "0.1" tracing = "0.1" tracing-subscriber = "0.3" bytes = "1" -rand = "0.8.5" +rand = "0.9" walkdir = "2" anyhow = "1.0" serde = { version = "1.0", features = ["derive"] } serde_with = "3.12" toml = "0.8.2" argh = "0.1.12" -dashmap = { version = "6.1.0", features = ["rayon", "inline"] } -rayon = "1.8" +dashmap = { version = "6.1.0", features = ["inline"] } atomic-time = "0.1.4" img-parts = "0.3" diff --git a/src/cache.rs b/src/cache.rs index df896e3..89ae011 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -6,7 +6,6 @@ use std::{ use atomic_time::AtomicSystemTime; use bytes::Bytes; use dashmap::{mapref::one::Ref, DashMap}; -use rayon::prelude::*; use tokio::time; use crate::config; @@ -81,7 +80,7 @@ impl Cache { let mut sorted: Vec<_> = self.map.iter().collect(); // Sort by least recently used - sorted.par_sort_unstable_by(|e1, e2| e1.last_used().cmp(&e2.last_used())); + sorted.sort_unstable_by_key(|e| e.last_used()); // Total bytes we would be removing let mut total = 0; @@ -142,12 +141,12 @@ impl Cache { // How far we went above the limit let needed = new_total - self.cfg.mem_capacity; - self.next_out(needed).par_iter().for_each(|k| { + self.next_out(needed).iter().for_each(|k| { // Remove the element, and ignore the result // The only reason it should be failing is if it couldn't find it, // in which case it was already removed self.remove(k); - }) + }); } // Atomically add to total cached data length @@ -170,7 +169,7 @@ impl Cache { /// /// It exists so we can run the expiry check before /// actually working with any entries, so no weird bugs happen - fn _get(&self, key: &str) -> Option> { + fn get_(&self, key: &str) -> Option> { let e = self.map.get(key)?; // if the entry is expired get rid of it now @@ -190,7 +189,7 @@ impl Cache { /// Get an item from the cache, if it exists. pub fn get(&self, key: &str) -> Option { - let e = self._get(key)?; + let e = self.get_(key)?; if e.update_used { e.last_used.store(SystemTime::now(), Ordering::Relaxed); @@ -206,7 +205,7 @@ impl Cache { /// We don't use [`DashMap::contains_key`] here because it would just do /// the exact same thing I do here, but without running the expiry check logic pub fn has(&self, key: &str) -> bool { - self._get(key).is_some() + self.get_(key).is_some() } /// Returns if an upload is able to be cached @@ -235,7 +234,7 @@ impl Cache { // If we fail to compare the times, it gets added to the list anyways let expired: Vec<_> = self .map - .par_iter() + .iter() .filter_map(|e| { let elapsed = now.duration_since(e.last_used()).unwrap_or(Duration::MAX); let is_expired = elapsed >= e.lifetime; @@ -252,7 +251,7 @@ impl Cache { if !expired.is_empty() { // Use a retain call, should be less locks that way // (instead of many remove calls) - self.map.retain(|k, _| !expired.contains(k)) + self.map.retain(|k, _| !expired.contains(k)); } } } diff --git a/src/disk.rs b/src/disk.rs index 6923701..f07df38 100644 --- a/src/disk.rs +++ b/src/disk.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use bytes::Bytes; use tokio::{ @@ -32,8 +32,11 @@ impl Disk { /// Formats the path on disk for a `saved_name`. fn path_for(&self, saved_name: &str) -> PathBuf { - let mut p = self.cfg.save_path.clone(); - p.push(saved_name); + // try to prevent path traversal by ignoring everything except the file name + let name = Path::new(saved_name).file_name().unwrap_or_default(); + + let mut p: PathBuf = self.cfg.save_path.clone(); + p.push(name); p } @@ -65,7 +68,7 @@ impl Disk { } /// Create a background I/O task - pub async fn start_save(&self, saved_name: &str) -> mpsc::UnboundedSender { + pub fn start_save(&self, saved_name: &str) -> mpsc::UnboundedSender { // start a task that handles saving files to disk (we can save to cache/disk in parallel that way) let (tx, mut rx): (mpsc::UnboundedSender, mpsc::UnboundedReceiver) = mpsc::unbounded_channel(); diff --git a/src/engine.rs b/src/engine.rs index 57eea64..c83f36d 100644 --- a/src/engine.rs +++ b/src/engine.rs @@ -7,10 +7,11 @@ use std::{ time::Duration, }; +use anyhow::Context as _; use axum::body::BodyDataStream; use bytes::{BufMut, Bytes, BytesMut}; use img_parts::{DynImage, ImageEXIF}; -use rand::distributions::{Alphanumeric, DistString}; +use rand::distr::{Alphanumeric, SampleString}; use tokio::{ fs::File, io::{AsyncReadExt, AsyncSeekExt}, @@ -146,9 +147,7 @@ impl Engine { u } else { // now, check if we have it on disk - let mut f = if let Some(f) = self.disk.open(saved_name).await? { - f - } else { + let Some(mut f) = self.disk.open(saved_name).await? else { // file didn't exist return Ok(GetOutcome::NotFound); }; @@ -180,9 +179,7 @@ impl Engine { data } else { - let (start, end) = if let Some(range) = resolve_range(range, full_len) { - range - } else { + let Some((start, end)) = resolve_range(range, full_len) else { return Ok(GetOutcome::RangeNotSatisfiable); }; @@ -201,9 +198,7 @@ impl Engine { }; let full_len = data.len() as u64; - let (start, end) = if let Some(range) = resolve_range(range, full_len) { - range - } else { + let Some((start, end)) = resolve_range(range, full_len) else { return Ok(GetOutcome::RangeNotSatisfiable); }; @@ -243,7 +238,7 @@ impl Engine { pub async fn gen_saved_name(&self, ext: Option) -> String { loop { // generate a 6-character alphanumeric string - let mut saved_name: String = Alphanumeric.sample_string(&mut rand::thread_rng(), 6); + let mut saved_name: String = Alphanumeric.sample_string(&mut rand::rng(), 6); // if we have an extension, add it now if let Some(ref ext) = ext { @@ -267,7 +262,10 @@ impl Engine { info!("!! removing upload: {saved_name}"); self.cache.remove(saved_name); - self.disk.remove(saved_name).await?; + self.disk + .remove(saved_name) + .await + .context("failed to remove file from disk")?; info!("!! successfully removed upload"); @@ -295,12 +293,12 @@ impl Engine { // don't begin a disk save if we're using temporary lifetimes let tx = if lifetime.is_none() { - Some(self.disk.start_save(saved_name).await) + Some(self.disk.start_save(saved_name)) } else { None }; - // whether or not we're gonna coalesce the data + // whether or not we are going to coalesce the data // in order to strip the exif data at the end, // instead of just sending it off to the i/o task let coalesce_and_strip = use_cache @@ -323,7 +321,8 @@ impl Engine { if !coalesce_and_strip { if let Some(ref tx) = tx { debug!("sending chunk to i/o task"); - tx.send(chunk.clone())?; + tx.send(chunk.clone()) + .context("failed to send chunk to i/o task!")?; } } @@ -365,7 +364,8 @@ impl Engine { // send what we did over to the i/o task, all in one chunk if let Some(ref tx) = tx { debug!("sending filled buffer to i/o task"); - tx.send(data.clone())?; + tx.send(data.clone()) + .context("failed to send coalesced buffer to i/o task!")?; } data diff --git a/src/main.rs b/src/main.rs index b3275db..e8cba56 100644 --- a/src/main.rs +++ b/src/main.rs @@ -99,8 +99,8 @@ async fn shutdown_signal() { let terminate = std::future::pending::<()>(); tokio::select! { - _ = ctrl_c => {}, - _ = terminate => {}, + () = ctrl_c => {}, + () = terminate => {}, } info!("shutting down!"); diff --git a/src/new.rs b/src/new.rs index bfb1561..b6dd296 100644 --- a/src/new.rs +++ b/src/new.rs @@ -14,6 +14,7 @@ use headers::ContentLength; use http::StatusCode; use serde::Deserialize; use serde_with::{serde_as, DurationSeconds}; +use tracing::error; use crate::engine::ProcessOutcome; @@ -90,7 +91,7 @@ pub async fn new( // pass it off to the engine to be processed // -- - // also, error responses here don't get represented properly in ShareX most of the time + // also, error responses here don't get presented properly in ShareX most of the time // they don't expect the connection to close before they're done uploading, i think // so it will just present the user with a "connection closed" error match engine @@ -111,6 +112,9 @@ pub async fn new( }, // 500 Internal Server Error - Err(_) => Err(StatusCode::INTERNAL_SERVER_ERROR), + Err(err) => { + error!("failed to process upload!! {err:#}"); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } } } diff --git a/src/view.rs b/src/view.rs index 413ee50..6c7014a 100644 --- a/src/view.rs +++ b/src/view.rs @@ -10,6 +10,7 @@ use axum_extra::TypedHeader; use headers::Range; use http::{HeaderValue, StatusCode}; use tokio_util::io::ReaderStream; +use tracing::error; use crate::engine::{GetOutcome, UploadData, UploadResponse}; @@ -91,19 +92,23 @@ pub async fn view( Path(original_path): Path, range: Option>, ) -> Result { - let saved_name = if let Some(Some(n)) = original_path.file_name().map(OsStr::to_str) { - n - } else { - return Err(ViewError::NotFound); + // try to extract the file name (if it's the only component) + // this makes paths like `asdf%2fabcdef.png` invalid + let saved_name = match original_path.file_name().map(OsStr::to_str) { + Some(Some(n)) if original_path.components().count() == 1 => n, + _ => return Err(ViewError::NotFound), }; - let range = range.map(|th| th.0); + let range = range.map(|TypedHeader(range)| range); // get result from the engine match engine.get(saved_name, range).await { Ok(GetOutcome::Success(res)) => Ok(res), Ok(GetOutcome::NotFound) => Err(ViewError::NotFound), Ok(GetOutcome::RangeNotSatisfiable) => Err(ViewError::RangeNotSatisfiable), - Err(_) => Err(ViewError::InternalServerError), + Err(err) => { + error!("failed to get upload!! {err:#}"); + Err(ViewError::InternalServerError) + } } }