added cache, s3 to taskmanager, ask cache if result is the same, among others

This commit is contained in:
2025-07-14 23:30:29 -04:00
parent 57952ec41d
commit d53f3da4c6
11 changed files with 241 additions and 53 deletions

View File

@@ -2,19 +2,24 @@ use crate::utils::{
request::{Request, Response},
task_log,
};
use storage::services::{
aws::{S3Client, S3ClientConfig},
ObjectStorageClient,
};
use cache::KeysInterface;
use storage::services::{aws::S3Client, ObjectStorageClient};
pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) {
pub fn register(pool: &sqlx::Pool<sqlx::Postgres>, cache: &cache::Pool, s3_client: &S3Client) {
let p = pool.clone();
let c = cache.clone();
let s3 = s3_client.to_owned();
tokio::spawn(async move {
let _ = upload_rss(&p).await;
let _ = upload_rss(&p, &c, s3).await;
});
}
async fn upload_rss(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), Box<dyn std::error::Error>> {
async fn upload_rss(
pool: &sqlx::Pool<sqlx::Postgres>,
cache: &cache::Pool,
s3_client: S3Client,
) -> Result<(), Box<dyn std::error::Error>> {
// start task logging
task_log::start(2, pool).await?;
@@ -25,15 +30,37 @@ async fn upload_rss(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), Box<dyn std
// upload the sucker to obj storage
if let Response::Xml(rss) = rss_result {
let client_config = S3ClientConfig::from_env().unwrap();
let s3_client = S3Client::new(&client_config);
let cached: &Option<String> = &cache.get(String::from("rss")).await.unwrap_or(None);
let cache_clone = cache.clone();
if let Some(cached_value) = cached {
if *cached_value == rss {
println!("Response is the same in the cache, exiting");
return Ok(());
}
}
let r = rss.clone();
let _ = s3_client
.put_object(
client_config.bucket.as_str(),
s3_client.client_config.bucket.as_str(),
"feed.xml",
rss.as_bytes().to_vec(),
)
.await;
.await?;
tokio::spawn(async move {
cache_clone
.set::<String, String, &String>(
String::from("rss"),
&r,
Some(cache::Expiration::EX(3600)),
None,
false,
)
.await
.unwrap();
});
println!("Finished uploading RSS feed");
}