matteo/serves3
1
0
Fork 1
Code Issues Pull requests Projects Releases Packages Wiki Activity

chore: migrate to object_store

Browse source 
Move to the more modern and maintained object_store
library to access generic buckets.

We should be able also to do streaming of files now,
and proceed by implementing paginated results for listings
if we want.

Fixes #1.
This commit is contained in:
Matteo Settenvini 2025-08-03 20:47:12 +02:00
parent 996be0f6df
commit 32e2a5ea4a
Signed by: matteo
GPG key ID: 1C1B12600D81DE05
11 changed files with 765 additions and 558 deletions
Download patch file Download diff file Expand all files Collapse all files

2
tests/common/minio.rs
View file

@ -11,7 +11,7 @@ use {
testcontainers_modules::minio,
};
const MINIO_IMAGE_TAG: &'static str = "RELEASE.2025-06-13T11-33-47Z";
const MINIO_IMAGE_TAG: &'static str = "RELEASE.2025-07-23T15-54-02Z";
pub struct MinIO {
inner: minio::MinIO,

36
tests/common/mod.rs
View file

@ -4,7 +4,11 @@
mod minio;
use {
::minio::s3::{
client::Client as MinIOClient, creds::StaticProvider, http::BaseUrl, types::S3Api,
},
anyhow::{Result, anyhow},
object_store::{ObjectStore, aws::AmazonS3Builder},
reqwest::Url,
std::{ptr::null_mut, str::FromStr},
testcontainers::{ContainerAsync, runners::AsyncRunner},
@ -13,7 +17,7 @@ use {
pub struct Test {
pub base_url: Url,
pub bucket: Box<s3::Bucket>,
pub bucket: Box<dyn ObjectStore>,
pub serves3: tokio::process::Child,
pub _minio: ContainerAsync<minio::MinIO>,
}
@ -41,24 +45,22 @@ impl Test {
port = container.get_host_port_ipv4(9000).await?
);
let credentials = s3::creds::Credentials::new(
Some(&ACCESS_KEY),
Some(&SECRET_KEY),
// We need to create the bucket
let minio_client = MinIOClient::new(
BaseUrl::from_str(&endpoint).unwrap(),
Some(Box::new(StaticProvider::new(ACCESS_KEY, SECRET_KEY, None))),
None,
None,
Some("test"),
)?;
let bucket = s3::Bucket::create_with_path_style(
&BUCKET_NAME,
s3::Region::Custom {
region: REGION.into(),
endpoint: endpoint.clone(),
},
credentials,
s3::BucketConfiguration::private(),
)
.await?
.bucket;
minio_client.create_bucket(BUCKET_NAME).send().await?;
let bucket = AmazonS3Builder::new()
.with_endpoint(&endpoint)
.with_access_key_id(ACCESS_KEY)
.with_secret_access_key(SECRET_KEY)
.with_bucket_name(BUCKET_NAME)
.with_allow_http(true)
.build()?;
let bin = std::env!("CARGO_BIN_EXE_serves3");
let mut child = tokio::process::Command::new(bin)
@ -104,7 +106,7 @@ impl Test {
Ok(Self {
base_url,
bucket,
bucket: Box::new(bucket),
serves3: child,
_minio: container,
})

60
tests/integration.rs
View file

@ -3,17 +3,27 @@
mod common;
use scraper::{Html, Selector};
use {
object_store::{PutPayload, path::Path as ObjectStorePath},
scraper::{Html, Selector},
};
#[test_log::test(tokio::test)]
#[test_log::test(tokio::test(flavor = "multi_thread"))]
async fn serves_files() -> anyhow::Result<()> {
let test = common::Test::new().await?;
test.bucket
.put_object("file.txt", "I am a file".as_bytes())
.put(
&ObjectStorePath::from("file.txt"),
PutPayload::from_static("I am a file".as_bytes()),
)
.await?;
test.bucket
.put_object("folder/file.txt", "I am a file in a folder".as_bytes())
.put(
&ObjectStorePath::from("folder/file.txt"),
PutPayload::from_static("I am a file in a folder".as_bytes()),
)
.await?;
let resp = reqwest::get(test.base_url.join("file.txt")?).await?;
@ -25,15 +35,21 @@ async fn serves_files() -> anyhow::Result<()> {
Ok(())
}
#[test_log::test(tokio::test)]
#[test_log::test(tokio::test(flavor = "multi_thread"))]
async fn serves_top_level_folder() -> anyhow::Result<()> {
let test = common::Test::new().await?;
test.bucket
.put_object("file.txt", "I am a file".as_bytes())
.put(
&ObjectStorePath::from("file.txt"),
PutPayload::from_static("I am a file".as_bytes()),
)
.await?;
test.bucket
.put_object("folder/file.txt", "I am a file in a folder".as_bytes())
.put(
&ObjectStorePath::from("folder/file.txt"),
PutPayload::from_static("I am a file in a folder".as_bytes()),
)
.await?;
// Check that a file in the toplevel is listed:
@ -43,6 +59,7 @@ async fn serves_top_level_folder() -> anyhow::Result<()> {
"Request failed with {}",
resp.status()
);
let text = resp.text().await?;
println!("{}", &text);
let document = Html::parse_document(&text);
@ -55,8 +72,8 @@ async fn serves_top_level_folder() -> anyhow::Result<()> {
let selector =
Selector::parse(r#"table > tbody > tr:nth-child(1) > td:first-child > a"#).unwrap();
for item in document.select(&selector) {
assert_eq!(item.attr("href"), Some("folder/"));
assert_eq!(item.text().next(), Some("folder/"));
assert_eq!(item.attr("href"), Some("folder"));
assert_eq!(item.text().next(), Some("folder"));
}
let selector =
@ -69,15 +86,22 @@ async fn serves_top_level_folder() -> anyhow::Result<()> {
Ok(())
}
#[test_log::test(tokio::test)]
#[test_log::test(tokio::test(flavor = "multi_thread"))]
async fn serves_second_level_folder() -> anyhow::Result<()> {
let test = common::Test::new().await?;
test.bucket
.put_object("file.txt", "I am a file".as_bytes())
.put(
&ObjectStorePath::from("file.txt"),
PutPayload::from_static("I am a file".as_bytes()),
)
.await?;
test.bucket
.put_object("folder/file.txt", "I am a file in a folder".as_bytes())
.put(
&ObjectStorePath::from("folder/file.txt"),
PutPayload::from_static("I am a file in a folder".as_bytes()),
)
.await?;
// Check that a file in the second level is listed:
@ -113,15 +137,21 @@ async fn serves_second_level_folder() -> anyhow::Result<()> {
Ok(())
}
#[test_log::test(tokio::test)]
#[test_log::test(tokio::test(flavor = "multi_thread"))]
async fn serves_second_level_folder_without_ending_slash() -> anyhow::Result<()> {
let test = common::Test::new().await?;
test.bucket
.put_object("file.txt", "I am a file".as_bytes())
.put(
&ObjectStorePath::from("file.txt"),
PutPayload::from_static("I am a file".as_bytes()),
)
.await?;
test.bucket
.put_object("folder/file.txt", "I am a file in a folder".as_bytes())
.put(
&ObjectStorePath::from("folder/file.txt"),
PutPayload::from_static("I am a file in a folder".as_bytes()),
)
.await?;
// Check that a file in the second level is listed even without an ending slash: