Compare commits
10 Commits
03de6f4283
...
9fdc2345f9
Author | SHA1 | Date |
---|---|---|
minish | 9fdc2345f9 | |
minish | b1bee30e23 | |
minish | 3dcea58419 | |
minish | 527e3c37ab | |
minish | 176cd813b9 | |
minish | d0a739386f | |
minish | 53b0b95554 | |
minish | 08826148bb | |
minish | 2662128bbb | |
minish | a3c69ef914 |
|
@ -129,7 +129,7 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
|||
|
||||
[[package]]
|
||||
name = "breeze"
|
||||
version = "0.1.1"
|
||||
version = "0.1.3"
|
||||
dependencies = [
|
||||
"archived",
|
||||
"async-recursion",
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
[package]
|
||||
name = "breeze"
|
||||
version = "0.1.1"
|
||||
version = "0.1.3"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
axum = { version = "0.6.1", features = ["macros"] }
|
||||
hyper = { version = "0.14", features = ["full"] }
|
||||
|
|
15
Dockerfile
15
Dockerfile
|
@ -1,10 +1,19 @@
|
|||
FROM rust:1.67.0 as builder
|
||||
# builder
|
||||
FROM rust:1.70 as builder
|
||||
|
||||
WORKDIR /usr/src/breeze
|
||||
COPY . .
|
||||
RUN [ "cargo", "install", "--path", "." ]
|
||||
RUN cargo install --path .
|
||||
|
||||
# runner
|
||||
FROM debian:bullseye-slim
|
||||
|
||||
RUN apt-get update && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /usr/local/cargo/bin/breeze /usr/local/bin/breeze
|
||||
|
||||
ENTRYPOINT [ "breeze" ]
|
||||
RUN useradd -m runner
|
||||
USER runner
|
||||
|
||||
EXPOSE 8000
|
||||
CMD [ "breeze" ]
|
||||
|
|
|
@ -0,0 +1,64 @@
|
|||
# breeze
|
||||
breeze is a simple, heavily optimised file upload server.
|
||||
|
||||
## Features
|
||||
Compared to the old Express.js backend, breeze has
|
||||
- Streamed uploading
|
||||
- Streamed downloading (on larger files)
|
||||
- Upload caching
|
||||
- Generally faster speeds overall
|
||||
|
||||
At this time, breeze does not support encrypted uploads on disk.
|
||||
|
||||
## Installation
|
||||
I wrote breeze with the intention of running it in a container, but it runs just fine outside of one.
|
||||
|
||||
Either way, you need to start off by cloning the Git repository.
|
||||
```bash
|
||||
git clone https://git.maple.vin/minish/breeze.git
|
||||
```
|
||||
|
||||
To run it in Docker, you need to build an image of it.
|
||||
```bash
|
||||
docker build -t breeze .
|
||||
```
|
||||
From there, you can make a `docker-compose.yaml` file with your configuration and run it using `docker-compose up`.
|
||||
|
||||
It can also be installed directly if you have the Rust toolchain installed
|
||||
```bash
|
||||
cargo install --path .
|
||||
```
|
||||
|
||||
## Usage
|
||||
### Hosting
|
||||
Configuration is read through environment variables, because I wanted to run this using `docker-compose`.
|
||||
```
|
||||
BRZ_BASE_URL - base url for upload urls (ex: http://127.0.0.1:8000 for http://127.0.0.1:8000/p/abcdef.png, http://picture.wtf for http://picture.wtf/p/abcdef.png)
|
||||
BRZ_SAVE_PATH - this should be a path where uploads are saved to disk (ex: /srv/uploads, C:\brzuploads)
|
||||
BRZ_UPLOAD_KEY (optional) - if not empty, the key you specify will be required to upload new files.
|
||||
BRZ_CACHE_UPL_MAX_LENGTH - this is the max length an upload can be in bytes before it won't be cached (ex: 80000000 for 80MB)
|
||||
BRZ_CACHE_UPL_LIFETIME - this indicates how long an upload will stay in cache (ex: 1800 for 30 minutes, 60 for 1 minute)
|
||||
BRZ_CACHE_SCAN_FREQ - this is the frequency of full cache scans, which scan for and remove expired uploads (ex: 60 for 1 minute)
|
||||
BRZ_CACHE_MEM_CAPACITY - this is the amount of memory the cache will hold before dropping entries
|
||||
```
|
||||
|
||||
### Uploading
|
||||
The HTTP API is fairly simple, and it's pretty easy to make a ShareX configuration for it.
|
||||
|
||||
Uploads should be sent to `/new?name={original filename}` as a POST request. If the server uses upload keys, it should be sent to `/new?name={original filename}&key={upload key}`. The uploaded file's content should be sent as raw binary in the request body.
|
||||
|
||||
Here's an example ShareX configuration for it (with a key):
|
||||
```json
|
||||
{
|
||||
"Version": "14.1.0",
|
||||
"Name": "breeze example",
|
||||
"DestinationType": "ImageUploader, TextUploader, FileUploader",
|
||||
"RequestMethod": "POST",
|
||||
"RequestURL": "http://127.0.0.1:8000/new",
|
||||
"Parameters": {
|
||||
"name": "{filename}",
|
||||
"key": "hiiiiiiii"
|
||||
},
|
||||
"Body": "Binary"
|
||||
}
|
||||
```
|
|
@ -8,7 +8,6 @@ use std::{
|
|||
use archived::Archive;
|
||||
use axum::extract::BodyStream;
|
||||
use bytes::{BufMut, Bytes, BytesMut};
|
||||
use hyper::StatusCode;
|
||||
use rand::Rng;
|
||||
use tokio::{
|
||||
fs::File,
|
||||
|
@ -21,16 +20,17 @@ use tokio::{
|
|||
use tokio_stream::StreamExt;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
use crate::view::ViewResponse;
|
||||
use crate::view::{ViewError, ViewSuccess};
|
||||
|
||||
pub struct Engine {
|
||||
// state
|
||||
cache: RwLock<Archive>, // in-memory cache. note/ i plan to lock the cache specifically only when needed rather than locking the whole struct
|
||||
cache: RwLock<Archive>, // in-memory cache
|
||||
pub upl_count: AtomicUsize, // cached count of uploaded files
|
||||
|
||||
// config
|
||||
pub base_url: String, // base url for formatting upload urls
|
||||
save_path: PathBuf, // where uploads are saved to disk
|
||||
pub base_url: String, // base url for formatting upload urls
|
||||
save_path: PathBuf, // where uploads are saved to disk
|
||||
pub upload_key: String, // authorisation key for uploading new files
|
||||
|
||||
cache_max_length: usize, // if an upload is bigger than this size, it won't be cached
|
||||
}
|
||||
|
@ -40,6 +40,7 @@ impl Engine {
|
|||
pub fn new(
|
||||
base_url: String,
|
||||
save_path: PathBuf,
|
||||
upload_key: String,
|
||||
cache_max_length: usize,
|
||||
cache_lifetime: Duration,
|
||||
cache_full_scan_freq: Duration, // how often the cache will be scanned for expired items
|
||||
|
@ -51,10 +52,11 @@ impl Engine {
|
|||
cache_lifetime,
|
||||
cache_mem_capacity,
|
||||
)),
|
||||
upl_count: AtomicUsize::new(WalkDir::new(&save_path).into_iter().count()), // count the amount of files in the save path and initialise our cached count with it
|
||||
upl_count: AtomicUsize::new(WalkDir::new(&save_path).min_depth(1).into_iter().count()), // count the amount of files in the save path and initialise our cached count with it
|
||||
|
||||
base_url,
|
||||
save_path,
|
||||
upload_key,
|
||||
|
||||
cache_max_length,
|
||||
}
|
||||
|
@ -142,11 +144,11 @@ impl Engine {
|
|||
// create file to save upload to
|
||||
let mut file = File::create(path)
|
||||
.await
|
||||
.expect("could not open file! make sure your upload path exists");
|
||||
.expect("could not open file! make sure your upload path is valid");
|
||||
|
||||
// receive chunks and save them to file
|
||||
while let Some(chunk) = rx.recv().await {
|
||||
debug!(target: "process_upload", "writing chunk to disk (length: {})", chunk.len());
|
||||
debug!("writing chunk to disk (length: {})", chunk.len());
|
||||
file.write_all(&chunk)
|
||||
.await
|
||||
.expect("error while writing file to disk");
|
||||
|
@ -158,15 +160,15 @@ impl Engine {
|
|||
let chunk = chunk.unwrap();
|
||||
|
||||
// send chunk to io task
|
||||
debug!(target: "process_upload", "sending data to io task");
|
||||
debug!("sending data to io task");
|
||||
tx.send(chunk.clone())
|
||||
.await
|
||||
.expect("failed to send data to io task");
|
||||
|
||||
if use_cache {
|
||||
debug!(target: "process_upload", "receiving data into buffer");
|
||||
debug!("receiving data into buffer");
|
||||
if data.len() + chunk.len() > data.capacity() {
|
||||
error!(target: "process_upload", "the amount of data sent exceeds the content-length provided by the client! caching will be cancelled for this upload.");
|
||||
error!("the amount of data sent exceeds the content-length provided by the client! caching will be cancelled for this upload.");
|
||||
|
||||
// if we receive too much data, drop the buffer and stop using cache (it is still okay to use disk, probably)
|
||||
data = BytesMut::new();
|
||||
|
@ -181,10 +183,12 @@ impl Engine {
|
|||
if use_cache {
|
||||
let mut cache = self.cache.write().await;
|
||||
|
||||
info!(target: "process_upload", "caching upload!");
|
||||
info!("caching upload!");
|
||||
cache.insert(name, data.freeze());
|
||||
}
|
||||
|
||||
info!("finished processing upload!!");
|
||||
|
||||
// if all goes well, increment the cached upload counter
|
||||
self.upl_count.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
@ -208,7 +212,7 @@ impl Engine {
|
|||
Some(data)
|
||||
}
|
||||
|
||||
pub async fn get_upload(&self, original_path: &PathBuf) -> Result<ViewResponse, StatusCode> {
|
||||
pub async fn get_upload(&self, original_path: &PathBuf) -> Result<ViewSuccess, ViewError> {
|
||||
// extract upload file name
|
||||
let name = original_path
|
||||
.file_name()
|
||||
|
@ -222,16 +226,16 @@ impl Engine {
|
|||
|
||||
// check if the upload exists, if not then 404
|
||||
if !self.upload_exists(&path).await {
|
||||
return Err(StatusCode::NOT_FOUND);
|
||||
return Err(ViewError::NotFound);
|
||||
}
|
||||
|
||||
// attempt to read upload from cache
|
||||
let cached_data = self.read_cached_upload(&name).await;
|
||||
|
||||
if let Some(data) = cached_data {
|
||||
info!(target: "get_upload", "got upload from cache!!");
|
||||
info!("got upload from cache!!");
|
||||
|
||||
return Ok(ViewResponse::FromCache(data));
|
||||
return Ok(ViewSuccess::FromCache(data));
|
||||
} else {
|
||||
let mut file = File::open(&path).await.unwrap();
|
||||
|
||||
|
@ -242,7 +246,7 @@ impl Engine {
|
|||
.expect("failed to read upload file metadata")
|
||||
.len() as usize;
|
||||
|
||||
debug!(target: "get_upload", "read upload from disk, size = {}", length);
|
||||
debug!("read upload from disk, size = {}", length);
|
||||
|
||||
// if the upload is okay to cache, recache it and send a fromcache response
|
||||
if self.will_use_cache(length) {
|
||||
|
@ -258,8 +262,8 @@ impl Engine {
|
|||
}
|
||||
}
|
||||
Err(_) => {
|
||||
return Err(StatusCode::INTERNAL_SERVER_ERROR);
|
||||
},
|
||||
return Err(ViewError::InternalServerError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -269,14 +273,14 @@ impl Engine {
|
|||
let mut cache = self.cache.write().await;
|
||||
cache.insert(name, data.clone());
|
||||
|
||||
info!(target: "get_upload", "recached upload from disk!");
|
||||
info!("recached upload from disk!");
|
||||
|
||||
return Ok(ViewResponse::FromCache(data));
|
||||
return Ok(ViewSuccess::FromCache(data));
|
||||
}
|
||||
|
||||
info!(target: "get_upload", "got upload from disk!");
|
||||
info!("got upload from disk!");
|
||||
|
||||
return Ok(ViewResponse::FromDisk(file));
|
||||
return Ok(ViewSuccess::FromDisk(file));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
11
src/index.rs
11
src/index.rs
|
@ -8,3 +8,14 @@ pub async fn index(State(engine): State<Arc<crate::engine::Engine>>) -> String {
|
|||
|
||||
format!("minish's image host, currently hosting {} files", count)
|
||||
}
|
||||
|
||||
// robots.txt that tells web crawlers not to list uploads
|
||||
const ROBOTS_TXT: &'static str = concat!(
|
||||
"User-Agent: *\n",
|
||||
"Disallow: /p/*\n",
|
||||
"Allow: /\n"
|
||||
);
|
||||
|
||||
pub async fn robots_txt() -> &'static str {
|
||||
ROBOTS_TXT
|
||||
}
|
12
src/main.rs
12
src/main.rs
|
@ -26,6 +26,7 @@ async fn main() {
|
|||
// read env vars
|
||||
let base_url = env::var("BRZ_BASE_URL").expect("missing BRZ_BASE_URL! base url for upload urls (ex: http://127.0.0.1:8000 for http://127.0.0.1:8000/p/abcdef.png, http://picture.wtf for http://picture.wtf/p/abcdef.png)");
|
||||
let save_path = env::var("BRZ_SAVE_PATH").expect("missing BRZ_SAVE_PATH! this should be a path where uploads are saved to disk (ex: /srv/uploads, C:\\brzuploads)");
|
||||
let upload_key = env::var("BRZ_UPLOAD_KEY").unwrap_or_default();
|
||||
let cache_max_length = env::var("BRZ_CACHE_UPL_MAX_LENGTH").expect("missing BRZ_CACHE_UPL_MAX_LENGTH! this is the max length an upload can be in bytes before it won't be cached (ex: 80000000 for 80MB)");
|
||||
let cache_upl_lifetime = env::var("BRZ_CACHE_UPL_LIFETIME").expect("missing BRZ_CACHE_UPL_LIFETIME! this indicates how long an upload will stay in cache (ex: 1800 for 30 minutes, 60 for 1 minute)");
|
||||
let cache_scan_freq = env::var("BRZ_CACHE_SCAN_FREQ").expect("missing BRZ_CACHE_SCAN_FREQ! this is the frequency of full cache scans, which scan for and remove expired uploads (ex: 60 for 1 minute)");
|
||||
|
@ -39,13 +40,19 @@ async fn main() {
|
|||
let cache_mem_capacity = usize::from_str_radix(&cache_mem_capacity, 10).expect("failed parsing BRZ_CACHE_MEM_CAPACITY! it should be a positive number without any separators");
|
||||
|
||||
if !save_path.exists() || !save_path.is_dir() {
|
||||
panic!("the save path does not exist or is not a directory. this is invalid");
|
||||
panic!("the save path does not exist or is not a directory! this is invalid");
|
||||
}
|
||||
|
||||
if upload_key.is_empty() {
|
||||
// i would prefer this to be a warning but the default log level hides those
|
||||
error!("upload key (BRZ_UPLOAD_KEY) is empty! no key will be required for uploading new files");
|
||||
}
|
||||
|
||||
// create engine
|
||||
let engine = Engine::new(
|
||||
base_url,
|
||||
save_path,
|
||||
upload_key,
|
||||
cache_max_length,
|
||||
cache_upl_lifetime,
|
||||
cache_scan_freq,
|
||||
|
@ -57,6 +64,7 @@ async fn main() {
|
|||
.route("/new", post(new::new))
|
||||
.route("/p/:name", get(view::view))
|
||||
.route("/", get(index::index))
|
||||
.route("/robots.txt", get(index::robots_txt))
|
||||
.with_state(Arc::new(engine));
|
||||
|
||||
// start web server
|
||||
|
@ -71,7 +79,7 @@ async fn shutdown_signal() {
|
|||
let ctrl_c = async {
|
||||
signal::ctrl_c()
|
||||
.await
|
||||
.expect("failed to add ctrl-c handler");
|
||||
.expect("failed to add SIGINT handler");
|
||||
};
|
||||
|
||||
#[cfg(unix)]
|
||||
|
|
19
src/new.rs
19
src/new.rs
|
@ -4,7 +4,7 @@ use axum::{
|
|||
extract::{BodyStream, Query, State},
|
||||
http::HeaderValue,
|
||||
};
|
||||
use hyper::{HeaderMap, StatusCode, header};
|
||||
use hyper::{header, HeaderMap, StatusCode};
|
||||
|
||||
#[axum::debug_handler]
|
||||
pub async fn new(
|
||||
|
@ -13,12 +13,21 @@ pub async fn new(
|
|||
Query(params): Query<HashMap<String, String>>,
|
||||
stream: BodyStream,
|
||||
) -> Result<String, StatusCode> {
|
||||
if !params.contains_key("name") {
|
||||
let key = params.get("key");
|
||||
|
||||
// check upload key, if i need to
|
||||
if !engine.upload_key.is_empty() && key.unwrap_or(&String::new()) != &engine.upload_key {
|
||||
return Err(StatusCode::FORBIDDEN);
|
||||
}
|
||||
|
||||
let original_name = params.get("name");
|
||||
|
||||
// the original file name wasn't given, so i can't work out what the extension should be
|
||||
if original_name.is_none() {
|
||||
return Err(StatusCode::BAD_REQUEST);
|
||||
}
|
||||
|
||||
let original_name = params.get("name").unwrap();
|
||||
let original_path = PathBuf::from(original_name);
|
||||
let original_path = PathBuf::from(original_name.unwrap());
|
||||
|
||||
let path = engine.gen_path(&original_path).await;
|
||||
let name = path
|
||||
|
@ -29,6 +38,7 @@ pub async fn new(
|
|||
|
||||
let url = format!("{}/p/{}", engine.base_url, name);
|
||||
|
||||
// read and parse content-length, and if it fails just assume it's really high so it doesn't cache
|
||||
let content_length = headers
|
||||
.get(header::CONTENT_LENGTH)
|
||||
.unwrap_or(&HeaderValue::from_static(""))
|
||||
|
@ -37,6 +47,7 @@ pub async fn new(
|
|||
.unwrap()
|
||||
.unwrap_or(usize::MAX);
|
||||
|
||||
// pass it off to the engine to be processed!
|
||||
engine
|
||||
.process_upload(path, name, content_length, stream)
|
||||
.await;
|
||||
|
|
41
src/view.rs
41
src/view.rs
|
@ -14,22 +14,27 @@ use hyper::StatusCode;
|
|||
use tokio::fs::File;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
pub enum ViewResponse {
|
||||
pub enum ViewSuccess {
|
||||
FromDisk(File),
|
||||
FromCache(Bytes),
|
||||
}
|
||||
|
||||
impl IntoResponse for ViewResponse {
|
||||
pub enum ViewError {
|
||||
NotFound, // 404
|
||||
InternalServerError, // 500
|
||||
}
|
||||
|
||||
impl IntoResponse for ViewSuccess {
|
||||
fn into_response(self) -> Response {
|
||||
match self {
|
||||
ViewResponse::FromDisk(file) => {
|
||||
ViewSuccess::FromDisk(file) => {
|
||||
// create a streamed body response (we want to stream larger files)
|
||||
let reader = ReaderStream::new(file);
|
||||
let stream = StreamBody::new(reader);
|
||||
|
||||
stream.into_response()
|
||||
}
|
||||
ViewResponse::FromCache(data) => {
|
||||
ViewSuccess::FromCache(data) => {
|
||||
// extract mutable headers from the response
|
||||
let mut res = data.into_response();
|
||||
let headers = res.headers_mut();
|
||||
|
@ -43,20 +48,42 @@ impl IntoResponse for ViewResponse {
|
|||
}
|
||||
}
|
||||
|
||||
impl IntoResponse for ViewError {
|
||||
fn into_response(self) -> Response {
|
||||
match self {
|
||||
ViewError::NotFound => {
|
||||
// convert string into response, change status code
|
||||
let mut res = "not found!".into_response();
|
||||
*res.status_mut() = StatusCode::NOT_FOUND;
|
||||
|
||||
res
|
||||
}
|
||||
ViewError::InternalServerError => {
|
||||
// convert string into response, change status code
|
||||
let mut res = "internal server error!".into_response();
|
||||
*res.status_mut() = StatusCode::NOT_FOUND;
|
||||
|
||||
res
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[axum::debug_handler]
|
||||
pub async fn view(
|
||||
State(engine): State<Arc<crate::engine::Engine>>,
|
||||
Path(original_path): Path<PathBuf>,
|
||||
) -> Result<ViewResponse, StatusCode> {
|
||||
) -> Result<ViewSuccess, ViewError> {
|
||||
// (hopefully) prevent path traversal, just check for any non-file components
|
||||
if original_path
|
||||
.components()
|
||||
.into_iter()
|
||||
.any(|x| !matches!(x, Component::Normal(_)))
|
||||
{
|
||||
warn!(target: "view", "a request attempted path traversal");
|
||||
return Err(StatusCode::NOT_FOUND);
|
||||
warn!("a request attempted path traversal");
|
||||
return Err(ViewError::NotFound);
|
||||
}
|
||||
|
||||
// get result from the engine!
|
||||
engine.get_upload(&original_path).await
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue