added cache, s3 to taskmanager, ask cache if result is the same, among others
This commit is contained in:
@@ -2,19 +2,24 @@ use crate::utils::{
|
||||
request::{Request, Response},
|
||||
task_log,
|
||||
};
|
||||
use storage::services::{
|
||||
aws::{S3Client, S3ClientConfig},
|
||||
ObjectStorageClient,
|
||||
};
|
||||
use cache::KeysInterface;
|
||||
use storage::services::{aws::S3Client, ObjectStorageClient};
|
||||
|
||||
pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) {
|
||||
pub fn register(pool: &sqlx::Pool<sqlx::Postgres>, cache: &cache::Pool, s3_client: &S3Client) {
|
||||
let p = pool.clone();
|
||||
let c = cache.clone();
|
||||
let s3 = s3_client.to_owned();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let _ = upload_rss(&p).await;
|
||||
let _ = upload_rss(&p, &c, s3).await;
|
||||
});
|
||||
}
|
||||
|
||||
async fn upload_rss(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn upload_rss(
|
||||
pool: &sqlx::Pool<sqlx::Postgres>,
|
||||
cache: &cache::Pool,
|
||||
s3_client: S3Client,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
// start task logging
|
||||
task_log::start(2, pool).await?;
|
||||
|
||||
@@ -25,15 +30,37 @@ async fn upload_rss(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), Box<dyn std
|
||||
|
||||
// upload the sucker to obj storage
|
||||
if let Response::Xml(rss) = rss_result {
|
||||
let client_config = S3ClientConfig::from_env().unwrap();
|
||||
let s3_client = S3Client::new(&client_config);
|
||||
let cached: &Option<String> = &cache.get(String::from("rss")).await.unwrap_or(None);
|
||||
let cache_clone = cache.clone();
|
||||
if let Some(cached_value) = cached {
|
||||
if *cached_value == rss {
|
||||
println!("Response is the same in the cache, exiting");
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let r = rss.clone();
|
||||
|
||||
let _ = s3_client
|
||||
.put_object(
|
||||
client_config.bucket.as_str(),
|
||||
s3_client.client_config.bucket.as_str(),
|
||||
"feed.xml",
|
||||
rss.as_bytes().to_vec(),
|
||||
)
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
tokio::spawn(async move {
|
||||
cache_clone
|
||||
.set::<String, String, &String>(
|
||||
String::from("rss"),
|
||||
&r,
|
||||
Some(cache::Expiration::EX(3600)),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
});
|
||||
|
||||
println!("Finished uploading RSS feed");
|
||||
}
|
||||
|
||||
|
Reference in New Issue
Block a user