Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,19 @@ RELAY_URL=ws://localhost:3000
# BUZZ_GIT_PACK_CACHE_MAX_BYTES=5368709120
# BUZZ_GIT_PACK_CACHE_MAX_CONCURRENT_POPULATIONS=2

# -----------------------------------------------------------------------------
# S3-Compatible Object Storage (media + Git/CAS)
# -----------------------------------------------------------------------------
# The local MinIO container is reachable from host processes at localhost:9000.
# Path style keeps the bucket in the URL path and is required by this local DNS
# setup. Use `virtual` only when the provider requires bucket-as-subdomain URLs.
BUZZ_S3_ENDPOINT=http://localhost:9000
BUZZ_S3_ACCESS_KEY=buzz_dev
BUZZ_S3_SECRET_KEY=buzz_dev_secret
BUZZ_S3_BUCKET=buzz-media
BUZZ_S3_REGION=us-east-1
BUZZ_S3_ADDRESSING_STYLE=path

# -----------------------------------------------------------------------------
# Media Upload Admission
# -----------------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

69 changes: 68 additions & 1 deletion crates/buzz-media/src/config.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,38 @@
//! Media storage configuration.

use std::str::FromStr;

/// S3 URL addressing style shared by media and Git/CAS storage.
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum S3AddressingStyle {
/// Put the bucket in the request path (`https://endpoint/bucket/key`).
///
/// This preserves compatibility with the bundled MinIO deployments, whose
/// internal DNS only resolves the endpoint hostname.
#[default]
Path,
/// Put the bucket in the hostname (`https://bucket.endpoint/key`).
///
/// This is the standard S3 form and is required by providers such as new
/// Railway Storage Buckets.
Virtual,
}

impl FromStr for S3AddressingStyle {
type Err = String;

fn from_str(value: &str) -> Result<Self, Self::Err> {
match value {
"path" => Ok(Self::Path),
"virtual" => Ok(Self::Virtual),
_ => Err(format!(
"BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}"
)),
}
}
}

fn default_max_video_bytes() -> u64 {
524_288_000 // 500 MB
}
Expand Down Expand Up @@ -31,6 +64,9 @@ pub struct MediaConfig {
/// the value is not meaningfully checked.
#[serde(default = "default_s3_region")]
pub s3_region: String,
/// S3 URL addressing style. Defaults to path style for MinIO compatibility.
#[serde(default)]
pub s3_addressing_style: S3AddressingStyle,
/// Maximum upload size for images (bytes). Default: 50 MB.
pub max_image_bytes: u64,
/// Maximum upload size for animated GIFs (bytes). Default: 10 MB.
Expand Down Expand Up @@ -123,7 +159,8 @@ impl MediaConfig {

#[cfg(test)]
mod tests {
use super::MediaConfig;
use super::{MediaConfig, S3AddressingStyle};
use std::str::FromStr;

fn valid_config() -> MediaConfig {
MediaConfig {
Expand All @@ -132,6 +169,7 @@ mod tests {
s3_secret_key: "s".to_string(),
s3_bucket: "buzz-media".to_string(),
s3_region: "us-east-1".to_string(),
s3_addressing_style: S3AddressingStyle::Path,
max_image_bytes: 1,
max_gif_bytes: 1,
max_video_bytes: 1,
Expand All @@ -143,6 +181,35 @@ mod tests {
}
}

#[test]
fn addressing_style_parses_supported_values() {
assert_eq!(
S3AddressingStyle::from_str("path"),
Ok(S3AddressingStyle::Path)
);
assert_eq!(
S3AddressingStyle::from_str("virtual"),
Ok(S3AddressingStyle::Virtual)
);
}

#[test]
fn addressing_style_defaults_to_path() {
assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path);
}

#[test]
fn addressing_style_rejects_unknown_or_ambiguous_values() {
for invalid in ["", "auto", "PATH", "virtual-hosted"] {
let error =
S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style");
assert!(
error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"),
"unexpected error for {invalid:?}: {error}"
);
}
}

#[test]
fn upload_record_knobs_default_off_and_validate() {
assert!(valid_config().validate().is_ok());
Expand Down
2 changes: 1 addition & 1 deletion crates/buzz-media/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ pub use bucket_index::{
classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass,
Page, SweepError,
};
pub use config::MediaConfig;
pub use config::{MediaConfig, S3AddressingStyle};
pub use error::MediaError;
pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage};
pub use types::BlobDescriptor;
Expand Down
27 changes: 24 additions & 3 deletions crates/buzz-media/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::pin::Pin;

use buzz_core::tenant::{CommunityId, TenantContext};

use crate::config::MediaConfig;
use crate::config::{MediaConfig, S3AddressingStyle};
use crate::error::MediaError;
use bytes::Bytes;
use s3::creds::Credentials;
Expand Down Expand Up @@ -61,8 +61,11 @@ impl MediaStorage {
}
.map_err(|e| MediaError::StorageError(e.to_string()))?;
let bucket = Bucket::new(&config.s3_bucket, region, creds)
.map_err(|e| MediaError::StorageError(e.to_string()))?
.with_path_style();
.map_err(|e| MediaError::StorageError(e.to_string()))?;
let bucket = match config.s3_addressing_style {
S3AddressingStyle::Path => bucket.with_path_style(),
S3AddressingStyle::Virtual => bucket,
};
Ok(Self { bucket })
}

Expand Down Expand Up @@ -285,6 +288,7 @@ mod tests {
s3_secret_key: secret.to_string(),
s3_bucket: "buzz-media".to_string(),
s3_region: "us-west-2".to_string(),
s3_addressing_style: S3AddressingStyle::Path,
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
Expand All @@ -309,6 +313,23 @@ mod tests {
}
}

#[test]
fn client_constructor_applies_both_addressing_styles() {
let path = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret"))
.expect("path-style client");
assert!(path.bucket.is_path_style());
assert_eq!(path.bucket.url(), "http://localhost:9000/buzz-media");

let mut virtual_config = storage_config("buzz_dev", "buzz_dev_secret");
virtual_config.s3_addressing_style = S3AddressingStyle::Virtual;
let virtual_hosted = MediaStorage::new(&virtual_config).expect("virtual-hosted client");
assert!(virtual_hosted.bucket.is_subdomain_style());
assert_eq!(
virtual_hosted.bucket.url(),
"http://buzz-media.localhost:9000"
);
}

#[test]
fn partial_static_keys_are_rejected() {
let err = match MediaStorage::new(&storage_config("buzz_dev", "")) {
Expand Down
1 change: 1 addition & 0 deletions crates/buzz-media/src/upload.rs
Original file line number Diff line number Diff line change
Expand Up @@ -570,6 +570,7 @@ mod tests {
s3_secret_key: String::new(),
s3_bucket: String::new(),
s3_region: "us-east-1".to_string(),
s3_addressing_style: crate::config::S3AddressingStyle::Path,
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
Expand Down
1 change: 1 addition & 0 deletions crates/buzz-media/src/validation.rs
Original file line number Diff line number Diff line change
Expand Up @@ -949,6 +949,7 @@ mod tests {
s3_secret_key: String::new(),
s3_bucket: String::new(),
s3_region: "us-east-1".to_string(),
s3_addressing_style: crate::config::S3AddressingStyle::Path,
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
Expand Down
13 changes: 9 additions & 4 deletions crates/buzz-media/tests/static_creds_minio.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
//! Live round-trip test for the **static-credentials** S3 path against a local
//! MinIO, guarded by `#[ignore]`.
//! Live round-trip test for the **static-credentials** S3 path against an
//! S3-compatible service. It is guarded by `#[ignore]`.
//!
//! This is the path local/dev and any static-key deployment uses
//! (`s3_access_key`/`s3_secret_key` both non-empty -> `Credentials::new`). It
Expand All @@ -15,7 +15,8 @@
//! ```
//!
//! Overridable via `BUZZ_S3_ENDPOINT` / `BUZZ_S3_ACCESS_KEY` /
//! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET`.
//! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET` / `BUZZ_S3_REGION` /
//! `BUZZ_S3_ADDRESSING_STYLE`. The default remains `path` for MinIO.

use buzz_media::config::MediaConfig;
use buzz_media::storage::MediaStorage;
Expand All @@ -29,7 +30,11 @@ fn minio_config() -> MediaConfig {
s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY")
.unwrap_or_else(|_| "buzz_dev_secret".to_string()),
s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()),
s3_region: "us-east-1".to_string(),
s3_region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()),
s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE")
.unwrap_or_else(|_| "path".to_string())
.parse()
.expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"),
max_image_bytes: 50 * 1024 * 1024,
max_gif_bytes: 10 * 1024 * 1024,
max_video_bytes: 524_288_000,
Expand Down
25 changes: 9 additions & 16 deletions crates/buzz-relay/src/api/git/cas_publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1583,22 +1583,15 @@ mod tests {
}

fn live_store() -> GitStore {
let endpoint = std::env::var("BUZZ_GIT_S3_ENDPOINT")
.or_else(|_| std::env::var("BUZZ_S3_ENDPOINT"))
.unwrap_or_else(|_| "http://localhost:9000".into());
let access_key = std::env::var("BUZZ_GIT_S3_ACCESS_KEY")
.or_else(|_| std::env::var("BUZZ_S3_ACCESS_KEY"))
.unwrap_or_else(|_| "buzz_dev".into());
let secret_key = std::env::var("BUZZ_GIT_S3_SECRET_KEY")
.or_else(|_| std::env::var("BUZZ_S3_SECRET_KEY"))
.unwrap_or_else(|_| "buzz_dev_secret".into());
let bucket = std::env::var("BUZZ_GIT_S3_BUCKET")
.or_else(|_| std::env::var("BUZZ_S3_BUCKET"))
.unwrap_or_else(|_| "buzz-media".into());
let region = std::env::var("BUZZ_GIT_S3_REGION")
.or_else(|_| std::env::var("BUZZ_S3_REGION"))
.unwrap_or_else(|_| "us-east-1".into());
GitStore::new(&endpoint, &access_key, &secret_key, &bucket, &region).expect("connect minio")
GitStore::new(
"http://localhost:9000",
"buzz_dev",
"buzz_dev_secret",
"buzz-media",
"us-east-1",
buzz_media::config::S3AddressingStyle::Path,
)
.expect("connect local MinIO")
}

fn tenant() -> TenantContext {
Expand Down
14 changes: 11 additions & 3 deletions crates/buzz-relay/src/api/git/hydrate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -543,8 +543,15 @@ mod tests {
#[tokio::test]
async fn materialized_repo_is_created_under_configured_scratch_dir() {
let scratch = TempDir::new().unwrap();
let store = GitStore::new("http://localhost:9000", "x", "x", "x", "us-east-1")
.expect("construct store");
let store = GitStore::new(
"http://localhost:9000",
"x",
"x",
"x",
"us-east-1",
buzz_media::config::S3AddressingStyle::Path,
)
.expect("construct store");
let manifest = Manifest {
version: 1,
head: "refs/heads/main".into(),
Expand Down Expand Up @@ -587,8 +594,9 @@ mod tests {
"buzz_dev_secret",
"buzz-git",
"us-east-1",
buzz_media::config::S3AddressingStyle::Path,
)
.expect("connect minio")
.expect("connect local MinIO")
}

/// Build a tiny on-disk repo, return (pack bytes, head_oid).
Expand Down
Loading
Loading