added storage library
This commit is contained in:
2
backend/storage/.gitignore
vendored
Normal file
2
backend/storage/.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
target/
|
||||
.env
|
3589
backend/storage/Cargo.lock
generated
Normal file
3589
backend/storage/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
15
backend/storage/Cargo.toml
Normal file
15
backend/storage/Cargo.toml
Normal file
@ -0,0 +1,15 @@
|
||||
[package]
|
||||
name = "storage"
|
||||
description = "Internal object storage library"
|
||||
version = "0.1.0"
|
||||
edition = "2024"
|
||||
|
||||
[dependencies]
|
||||
aws-sdk-s3 = "1.94.0"
|
||||
aws-config = "1.8"
|
||||
azure_core = "0.25.0"
|
||||
azure_storage = "0.21.0"
|
||||
azure_storage_blobs = "0.21.0"
|
||||
async-trait = "0.1"
|
||||
tokio = { version = "1.0", features = ["full"] }
|
||||
thiserror = "2.0.12"
|
11
backend/storage/src/error.rs
Normal file
11
backend/storage/src/error.rs
Normal file
@ -0,0 +1,11 @@
|
||||
use azure_core::error::HttpError;
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum AdapterError {
|
||||
#[error("Azure error: {0}")]
|
||||
Azure(#[from] azure_core::Error),
|
||||
#[error("HTTP error: {0}")]
|
||||
Http(#[from] HttpError),
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(String),
|
||||
}
|
2
backend/storage/src/lib.rs
Normal file
2
backend/storage/src/lib.rs
Normal file
@ -0,0 +1,2 @@
|
||||
pub mod error;
|
||||
pub mod services;
|
101
backend/storage/src/services/aws.rs
Normal file
101
backend/storage/src/services/aws.rs
Normal file
@ -0,0 +1,101 @@
|
||||
use crate::{error::AdapterError, services::ObjectStorageClient};
|
||||
use async_trait::async_trait;
|
||||
use aws_config::{BehaviorVersion, Region};
|
||||
use aws_sdk_s3::{Client, Config, config::Credentials};
|
||||
use std::env;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct S3ClientConfig {
|
||||
pub access_key: String,
|
||||
secret_key: String,
|
||||
endpoint: String,
|
||||
pub bucket: String,
|
||||
region: String,
|
||||
}
|
||||
|
||||
pub struct S3Client {
|
||||
client: Client,
|
||||
}
|
||||
|
||||
impl S3ClientConfig {
|
||||
pub fn from_env() -> Result<Self, Box<dyn std::error::Error>> {
|
||||
Ok(S3ClientConfig {
|
||||
access_key: env::var("S3_ACCESS_KEY")
|
||||
.map_err(|_| "S3_ACCESS_KEY environment variable not set")?,
|
||||
secret_key: env::var("S3_SECRET_KEY")
|
||||
.map_err(|_| "S3_SECRET_KEY environment variable not set")?,
|
||||
endpoint: env::var("S3_ENDPOINT")
|
||||
.unwrap_or_else(|_| "us-ord-1.linodeobjects.com".to_string()),
|
||||
bucket: env::var("S3_BUCKET").map_err(|_| "S3_BUCKET environment variable not set")?,
|
||||
region: env::var("S3_REGION").unwrap_or_else(|_| "us-ord".to_string()),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Client {
|
||||
pub fn new(config: &S3ClientConfig) -> Self {
|
||||
let credentials = Credentials::new(
|
||||
&config.access_key,
|
||||
&config.secret_key,
|
||||
None,
|
||||
None,
|
||||
"linode-object-storage",
|
||||
);
|
||||
|
||||
let s3_config = Config::builder()
|
||||
.behavior_version(BehaviorVersion::latest())
|
||||
.region(Region::new(config.region.clone()))
|
||||
.endpoint_url(format!("https://{}", config.endpoint))
|
||||
.credentials_provider(credentials)
|
||||
.force_path_style(false)
|
||||
.build();
|
||||
|
||||
Self {
|
||||
client: Client::from_conf(s3_config),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ObjectStorageClient for S3Client {
|
||||
type Error = AdapterError;
|
||||
|
||||
async fn put_object(&self, bucket: &str, key: &str, data: Vec<u8>) -> Result<(), Self::Error> {
|
||||
println!("Uploading to S3 (or S3 like) Object Storage...");
|
||||
println!("Bucket: {}", bucket);
|
||||
|
||||
let _ = self
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(key)
|
||||
.body(data.into())
|
||||
.acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead)
|
||||
.content_type("application/xml")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_object(&self, _bucket: &str, _key: &str) -> Result<Vec<u8>, Self::Error> {
|
||||
todo!("not impl")
|
||||
}
|
||||
|
||||
async fn delete_object(&self, _bucket: &str, _key: &str) -> Result<(), Self::Error> {
|
||||
todo!("not impl")
|
||||
}
|
||||
|
||||
async fn list_objects(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: Option<&str>,
|
||||
) -> Result<Vec<String>, Self::Error> {
|
||||
todo!("not impl")
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _bucket: &str, _key: &str) -> Result<bool, Self::Error> {
|
||||
todo!("not impl")
|
||||
}
|
||||
}
|
71
backend/storage/src/services/azure.rs
Normal file
71
backend/storage/src/services/azure.rs
Normal file
@ -0,0 +1,71 @@
|
||||
use crate::error::AdapterError;
|
||||
use async_trait::async_trait;
|
||||
use azure_storage::prelude::*;
|
||||
use azure_storage_blobs::prelude::*;
|
||||
|
||||
use super::ObjectStorageClient;
|
||||
|
||||
pub struct AzureBlobClient {
|
||||
client: BlobServiceClient,
|
||||
}
|
||||
|
||||
impl AzureBlobClient {
|
||||
pub fn new(account_name: &str, account_key: String) -> Self {
|
||||
let storage_credentials = StorageCredentials::access_key(account_name, account_key);
|
||||
let client = BlobServiceClient::new(account_name, storage_credentials);
|
||||
|
||||
Self { client }
|
||||
}
|
||||
|
||||
// Helper method to get container client
|
||||
fn get_container_client(&self, container_name: &str) -> ContainerClient {
|
||||
self.client.container_client(container_name)
|
||||
}
|
||||
|
||||
// Helper method to get blob client
|
||||
fn get_blob_client(&self, container_name: &str, blob_name: &str) -> BlobClient {
|
||||
self.get_container_client(container_name)
|
||||
.blob_client(blob_name)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ObjectStorageClient for AzureBlobClient {
|
||||
type Error = AdapterError;
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
bucket: &str, // container name
|
||||
key: &str, // blob name
|
||||
data: Vec<u8>,
|
||||
) -> Result<(), Self::Error> {
|
||||
let blob_client = self.get_blob_client(bucket, key);
|
||||
let _request = blob_client.put_block_blob(data).await.unwrap();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_object(&self, bucket: &str, key: &str) -> Result<Vec<u8>, Self::Error> {
|
||||
let blob_client = self.get_blob_client(bucket, key);
|
||||
|
||||
let response = blob_client.get_content().await.unwrap();
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
async fn delete_object(&self, bucket: &str, key: &str) -> Result<(), Self::Error> {
|
||||
let blob_client = self.get_blob_client(bucket, key);
|
||||
blob_client.delete().await.unwrap();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn list_objects(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_prefix: Option<&str>,
|
||||
) -> Result<Vec<String>, Self::Error> {
|
||||
todo!("not impl")
|
||||
}
|
||||
|
||||
async fn object_exists(&self, _bucket: &str, _key: &str) -> Result<bool, Self::Error> {
|
||||
todo!("not impl")
|
||||
}
|
||||
}
|
23
backend/storage/src/services/mod.rs
Normal file
23
backend/storage/src/services/mod.rs
Normal file
@ -0,0 +1,23 @@
|
||||
pub mod aws;
|
||||
pub mod azure;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
#[async_trait]
|
||||
pub trait ObjectStorageClient {
|
||||
type Error;
|
||||
|
||||
async fn put_object(&self, bucket: &str, key: &str, data: Vec<u8>) -> Result<(), Self::Error>;
|
||||
|
||||
async fn get_object(&self, bucket: &str, key: &str) -> Result<Vec<u8>, Self::Error>;
|
||||
|
||||
async fn delete_object(&self, bucket: &str, key: &str) -> Result<(), Self::Error>;
|
||||
|
||||
async fn list_objects(
|
||||
&self,
|
||||
bucket: &str,
|
||||
prefix: Option<&str>,
|
||||
) -> Result<Vec<String>, Self::Error>;
|
||||
|
||||
async fn object_exists(&self, bucket: &str, key: &str) -> Result<bool, Self::Error>;
|
||||
}
|
Reference in New Issue
Block a user