From 328dacb675dc0ef959861cf05f9f2ec353a9c2cb Mon Sep 17 00:00:00 2001 From: "Wyatt J. Miller" Date: Sat, 15 Mar 2025 17:27:32 -0400 Subject: [PATCH 1/3] wip: broken state --- backend/public/Cargo.lock | 103 +++++++++++++++++++++++++++++++++++- backend/public/Cargo.toml | 1 + backend/public/src/main.rs | 5 +- backend/public/src/state.rs | 73 +++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 backend/public/src/state.rs diff --git a/backend/public/Cargo.lock b/backend/public/Cargo.lock index a313c98..51f31a9 100644 --- a/backend/public/Cargo.lock +++ b/backend/public/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "addr2line" @@ -59,6 +59,12 @@ dependencies = [ "libc", ] +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "async-trait" version = "0.1.82" @@ -209,6 +215,16 @@ version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "428d9aa8fbc0670b7b8d6030a7fadd0f86151cae55e4dbbece15f3780a3dfaf3" +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + [[package]] name = "cc" version = "1.1.21" @@ -253,6 +269,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "cookie-factory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -283,6 +305,12 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + [[package]] name = "crossbeam-queue" version = "0.3.11" @@ -403,6 +431,15 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "flume" version = "0.11.0" @@ -439,6 +476,43 @@ dependencies = [ "thiserror", ] +[[package]] +name = "fred" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a7b2fd0f08b23315c13b6156f971aeedb6f75fb16a29ac1872d2eabccc1490e" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "bytes-utils", + "float-cmp", + "fred-macros", + "futures", + "log", + "parking_lot", + "rand", + "redis-protocol", + "semver", + "socket2", + "tokio", + "tokio-stream", + "tokio-util", + "url", + "urlencoding", +] + +[[package]] +name = "fred-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1458c6e22d36d61507034d5afecc64f105c1d39712b7ac6ec3b352c423f715cc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures" version = "0.3.30" @@ -1180,6 +1254,7 @@ dependencies = [ "axum", "chrono", "dotenvy", + "fred", "serde", "serde_json", "sqlx", @@ -1254,6 +1329,20 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redis-protocol" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdba59219406899220fc4cdfd17a95191ba9c9afb719b5fa5a083d63109a9f1" +dependencies = [ + "bytes", + "bytes-utils", + "cookie-factory", + "crc16", + "log", + "nom", +] + [[package]] name = "redox_syscall" version = "0.5.4" @@ -1420,6 +1509,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + [[package]] name = "serde" version = "1.0.210" @@ -2142,6 +2237,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "valuable" version = "0.1.0" diff --git a/backend/public/Cargo.toml b/backend/public/Cargo.toml index f4f7b7c..b626412 100644 --- a/backend/public/Cargo.toml +++ b/backend/public/Cargo.toml @@ -24,3 +24,4 @@ serde = "1.0.210" serde_json = "1.0.128" chrono = "0.4.38" xml = "0.8.20" +fred = "10.1.0" diff --git a/backend/public/src/main.rs b/backend/public/src/main.rs index 9f62690..c516d01 100644 --- a/backend/public/src/main.rs +++ b/backend/public/src/main.rs @@ -16,12 +16,9 @@ use tracing_subscriber::{filter, layer::SubscriberExt, prelude::*, util::Subscri mod config; mod datasources; mod routes; +mod state; mod utils; -pub struct AppState { - db: PgPool, -} - #[tokio::main] async fn main() { // setting up configuration diff --git a/backend/public/src/state.rs b/backend/public/src/state.rs new file mode 100644 index 0000000..ab662a9 --- /dev/null +++ b/backend/public/src/state.rs @@ -0,0 +1,73 @@ +use fred::interfaces::KeysInterface; +use fred::{clients::Pool, prelude::*}; +use serde_json::Value; +use sqlx::PgPool; + +pub type AppState = std::sync::Arc>; + +pub struct AppInternalState { + pub database: sqlx::postgres::PgPool, + pub cache: Cache, +} + +pub struct Cache { + pub inmem: Pool, +} + +impl AppInternalState { + pub fn new(database: PgPool, cache: Pool) -> Self { + AppInternalState { + database, + cache: Cache { inmem: cache }, + } + } +} + +impl Cache { + pub async fn get(&mut self, key: String) -> Result, Box> + where + T: for<'de> serde::Deserialize<'de>, + { + if !self.inmem.is_connected() { + return Err(Box::new("Are you connected to the cache?".into())); + } + + let value: Option = self.inmem.get(key).await?; + + let result = match value { + Some(x) => match serde_json::from_value(x) { + Ok(x) => Some(x), + Err(_) => None, + }, + None => None, + }; + Ok(result) + } + + pub async fn set( + &mut self, + key: String, + contents: &T, + expiration: Option, + set_opts: Option, + get: bool, + ) -> Result<(), Box> + where + T: for<'de> serde::Deserialize<'de>, + { + if !self.inmem.is_connected() { + return Err(Box::new("Are you connected to the cache?".into())); + } + + let value: Value = serde_json::to_value(contents)?; + self.inmem + .set(key, value.to_string(), expiration, set_opts, get) + .await?; + Ok(()) + } + + pub async fn del(&mut self, key: String) -> Result<(), Box> { + self.inmem.del(key).await?; + Ok(()) + } +} From d126fed2bd993d3c3319a544469dfaae98e6b4a0 Mon Sep 17 00:00:00 2001 From: "Wyatt J. Miller" Date: Sun, 16 Mar 2025 02:56:54 -0400 Subject: [PATCH 2/3] implemented working cache some testing still has to be done --- backend/public/src/main.rs | 30 ++- backend/public/src/routes/authors.rs | 96 ++++++-- backend/public/src/routes/comments.rs | 60 +++-- backend/public/src/routes/posts.rs | 310 +++++++++++++++++++++++--- backend/public/src/state.rs | 32 +-- 5 files changed, 457 insertions(+), 71 deletions(-) diff --git a/backend/public/src/main.rs b/backend/public/src/main.rs index c516d01..68d407b 100644 --- a/backend/public/src/main.rs +++ b/backend/public/src/main.rs @@ -1,11 +1,13 @@ -use axum::{http::Method, Router}; +use axum::Router; use config::config; -use sqlx::{postgres::PgPoolOptions, PgPool}; +use fred::prelude::*; +use sqlx::postgres::PgPoolOptions; use std::fs::File; use std::sync::Arc; use std::time::Duration; use tokio::net::TcpListener; use tokio::signal; +use tokio::sync::Mutex; use tower_governor::{governor::GovernorConfigBuilder, GovernorLayer}; use tower_http::{ cors::{Any, CorsLayer}, @@ -84,6 +86,11 @@ async fn main() { // grabbing the database url from our env variables let db_connection_str = std::env::var("DATABASE_URL") .unwrap_or_else(|_| "postgres://postgres:password@localhost".to_string()); + let redis_url = match std::env::var("REDIS_URL").unwrap().as_str() { + // TODO: fix the unwrap ^ + "" => "redis://localhost:6379".to_string(), + x => x.to_string(), + }; // set up connection pool let pool = PgPoolOptions::new() @@ -93,7 +100,24 @@ async fn main() { .await .expect("Failed to connect to database"); - let app_state = AppState { db: pool.clone() }; + let pool_size = 8; + let config = Config::from_url(&redis_url).unwrap(); // TODO: fix the unwrap <<< + + let redis_pool = Builder::from_config(config) + .with_performance_config(|config| { + config.default_command_timeout = Duration::from_secs(60); + }) + .set_policy(ReconnectPolicy::new_exponential(0, 100, 30_000, 2)) + .build_pool(pool_size) + .expect("Failed to create cache pool"); + + if std::env::var("REDIS_URL").unwrap() != "" { + // TODO: fix the unwrap ^ + redis_pool.init().await.expect("Failed to connect to cache"); + let _ = redis_pool.flushall::(false).await; + } + + let app_state = Arc::new(Mutex::new(state::AppInternalState::new(pool, redis_pool))); // build our application with some routes let app = Router::new() diff --git a/backend/public/src/routes/authors.rs b/backend/public/src/routes/authors.rs index 903211d..96dd1c1 100644 --- a/backend/public/src/routes/authors.rs +++ b/backend/public/src/routes/authors.rs @@ -5,14 +5,14 @@ use axum::{ routing::get, Json, }; +use fred::types::Expiration; use serde::{Deserialize, Serialize}; -use sqlx::{Pool, Postgres}; -use crate::{datasources::authors::AuthorsDatasource, AppState}; +use crate::{datasources::authors::AuthorsDatasource, state::AppState}; use super::comments::Pagination; -#[derive(Serialize)] +#[derive(Deserialize, Serialize, Clone)] pub struct Author { pub author_id: i32, pub first_name: String, @@ -21,7 +21,7 @@ pub struct Author { pub image: Option, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize, Clone)] pub struct AuthorGetOneParams { pub id: i32, } @@ -33,34 +33,104 @@ impl AuthorsRoute { .route("/", get(AuthorsRoute::get_all)) .route("/:id", get(AuthorsRoute::get_one)) .route("/:id/posts", get(AuthorsRoute::get_authors_posts)) - .with_state(app_state.db.clone()) + .with_state(app_state.clone()) } async fn get_all( - State(pool): State>, + State(app_state): State, Json(pagination): Json, ) -> impl IntoResponse { - match AuthorsDatasource::get_all(&pool, pagination).await { - Ok(a) => Ok(Json(a)), + let mut state = app_state.lock().await; + let cached: Option> = state + .cache + .get(String::from("authors:all")) + .await + .unwrap_or(None); + + if let Some(authors) = cached { + tracing::info!("grabbing all authors from cache"); + return Ok(Json(authors)); + } + + match AuthorsDatasource::get_all(&state.database, pagination).await { + Ok(authors) => { + tracing::info!("grabbing all authors from the database"); + if let a = &authors { + let author_cloned = a.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + String::from("authors:all"), + &author_cloned, + Some(Expiration::EX(5)), + None, + false, + ) + .await; + }); + } + Ok(Json(authors)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } async fn get_one( - State(pool): State>, + State(app_state): State, Path(params): Path, ) -> impl IntoResponse { - match AuthorsDatasource::get_one(&pool, params.id).await { - Ok(a) => Ok(Json(a)), + let mut state = app_state.lock().await; + let cached: Option = state + .cache + .get(format!("authors:{}", params.id)) + .await + .unwrap_or(None); + + if let Some(author) = cached { + tracing::info!("grabbing one author from cache"); + return Ok(Json(author)); + } + + match AuthorsDatasource::get_one(&state.database, params.id).await { + Ok(author) => { + tracing::info!("grabbing all authors from the database"); + if let a = &author { + let author_cloned = a.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + format!("authors:{}", author_cloned.author_id), + &author_cloned, + Some(Expiration::EX(5)), + None, + false, + ) + .await; + }); + } + Ok(Json(author)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } async fn get_authors_posts( - State(pool): State>, + State(app_state): State, Path(params): Path, ) -> impl IntoResponse { - match AuthorsDatasource::get_authors_posts(&pool, params.id).await { + let state = app_state.lock().await; + + match AuthorsDatasource::get_authors_posts(&state.database, params.id).await { Ok(p) => Ok(Json(p)), Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } diff --git a/backend/public/src/routes/comments.rs b/backend/public/src/routes/comments.rs index 53b26df..4a71457 100644 --- a/backend/public/src/routes/comments.rs +++ b/backend/public/src/routes/comments.rs @@ -1,5 +1,5 @@ -use super::posts::serialize_datetime; -use crate::{datasources::comments::CommentsDatasource, AppState}; +use super::posts::{deserialize_datetime, serialize_datetime}; +use crate::{datasources::comments::CommentsDatasource, state::AppState}; use axum::{ extract::{Path, State}, http::StatusCode, @@ -8,33 +8,34 @@ use axum::{ Json, }; use chrono::Utc; +use fred::types::{Expiration, SetOptions}; use serde::{Deserialize, Serialize}; -use sqlx::{Pool, Postgres}; -#[derive(Deserialize, Debug)] +#[derive(Deserialize, Serialize, Debug)] pub struct CommentInputPayload { pub name: String, pub body: String, pub post_id: i32, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize)] pub struct CommentPathParams { id: i32, } -#[derive(Deserialize)] +#[derive(Deserialize, Serialize)] pub struct Pagination { pub page_number: i64, pub page_size: i64, } -#[derive(sqlx::FromRow, Serialize, Debug)] +#[derive(sqlx::FromRow, Deserialize, Serialize, Debug, Clone)] pub struct Comment { pub comment_id: i32, pub name: String, pub body: String, #[serde(serialize_with = "serialize_datetime")] + #[serde(deserialize_with = "deserialize_datetime")] pub created_at: Option>, } @@ -46,34 +47,61 @@ impl CommentsRoute { .route("/post/:id", get(CommentsRoute::get_post_comments)) .route("/add", post(CommentsRoute::insert_comment)) .route("/index", get(CommentsRoute::get_comments_index)) - .with_state(app_state.db.clone()) + .with_state(app_state.clone()) } async fn get_post_comments( - State(pool): State>, + State(app_state): State, Path(params): Path, ) -> impl IntoResponse { - match CommentsDatasource::get_posts_comments(&pool, params.id).await { + let state = app_state.lock().await; + + match CommentsDatasource::get_posts_comments(&state.database, params.id).await { Ok(c) => Ok(Json(c)), Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } - // + async fn insert_comment( - State(pool): State>, + State(app_state): State, Json(input): Json, ) -> impl IntoResponse { - match CommentsDatasource::insert_comment(&pool, input).await { - Ok(c) => Ok((StatusCode::CREATED, Json(c))), + let state = app_state.lock().await; + + match CommentsDatasource::insert_comment(&state.database, input).await { + Ok(c) => { + if let co = &c { + let co = c.clone(); + let state = app_state.clone(); + + tokio::spawn(async move { + tracing::info!("update cache if key already exists!"); + let mut s = state.lock().await; + let _ = s + .cache + .set( + format!("comments:{}", co.comment_id), + &co, + Some(Expiration::EX(60 * 15)), + Some(SetOptions::XX), + false, + ) + .await; + }); + } + Ok((StatusCode::CREATED, Json(c))) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } async fn get_comments_index( - State(pool): State>, + State(app_state): State, Json(pagination): Json, ) -> impl IntoResponse { - match CommentsDatasource::get_index_comments(&pool, pagination).await { + let state = app_state.lock().await; + + match CommentsDatasource::get_index_comments(&state.database, pagination).await { Ok(c) => Ok(Json(c)), Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } diff --git a/backend/public/src/routes/posts.rs b/backend/public/src/routes/posts.rs index f5678d6..5dbb94f 100644 --- a/backend/public/src/routes/posts.rs +++ b/backend/public/src/routes/posts.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use crate::utils::rss; -use crate::{datasources::posts::PostsDatasource, AppState}; +use crate::{datasources::posts::PostsDatasource, state::AppState}; use axum::http::{HeaderMap, HeaderValue}; use axum::{ extract::{Path, State}, @@ -11,10 +11,11 @@ use axum::{ Json, Router, }; use chrono::Utc; -use serde::{Deserialize, Serialize, Serializer}; -use sqlx::{Pool, Postgres}; +use fred::types::Expiration; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; -#[derive(sqlx::FromRow, Serialize, Debug, Clone)] +#[derive(sqlx::FromRow, Deserialize, Serialize, Debug, Clone)] pub struct Post { pub post_id: i32, pub author_id: Option, @@ -23,10 +24,11 @@ pub struct Post { pub title: String, pub body: String, #[serde(serialize_with = "serialize_datetime")] + #[serde(deserialize_with = "deserialize_datetime")] pub created_at: Option>, } -#[derive(sqlx::FromRow, Serialize, Debug)] +#[derive(sqlx::FromRow, Deserialize, Serialize, Debug, Clone)] pub struct PostFeaturedVariant { pub post_id: i32, pub author_id: Option, @@ -35,6 +37,7 @@ pub struct PostFeaturedVariant { pub title: String, pub body: String, #[serde(serialize_with = "serialize_datetime")] + #[serde(deserialize_with = "deserialize_datetime")] pub created_at: Option>, pub is_featured: Option, } @@ -56,63 +59,275 @@ impl PostsRoute { .route("/hot", get(PostsRoute::get_hot_posts)) .route("/featured", get(PostsRoute::get_featured_posts)) .route("/rss", get(PostsRoute::get_rss_posts)) - .with_state(app_state.db.clone()) + .with_state(app_state.clone()) } // get all posts - async fn get_all(State(pool): State>) -> impl IntoResponse { - match PostsDatasource::get_all(&pool).await { - Ok(posts) => Ok(Json(posts)), + async fn get_all(State(app_state): State) -> impl IntoResponse { + let mut state = app_state.lock().await; + let cached: Option> = state + .cache + .get(String::from("posts:all")) + .await + .unwrap_or(None); + + if let Some(posts) = cached { + tracing::info!("grabbing all posts from cache"); + return Ok(Json(posts)); + }; + + match PostsDatasource::get_all(&state.database).await { + Ok(posts) => { + tracing::info!("grabbing all posts from database"); + if let p = &posts { + let posts = p.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + String::from("posts:all"), + &posts, + Some(Expiration::EX(10)), + None, + false, + ) + .await; + }); + }; + + Ok(Json(posts)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } // get one post async fn get_one( - State(pool): State>, + State(app_state): State, Path(params): Path, ) -> impl IntoResponse { - match PostsDatasource::get_one(&pool, params.id).await { - Ok(post) => Ok(Json(post)), + let mut state = app_state.lock().await; + let cached: Option = state + .cache + .get(format!("posts:{}", params.id)) + .await + .unwrap_or(None); + + if let Some(post) = cached { + tracing::info!("grabbing one post from cache"); + return Ok(Json(post)); + }; + + match PostsDatasource::get_one(&state.database, params.id).await { + Ok(post) => { + tracing::info!("grabbing one post from database"); + if let p = &post { + let post = p.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + format!("posts:{}", params.id), + &post, + Some(Expiration::EX(10)), + None, + false, + ) + .await; + }); + }; + + Ok(Json(post)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } // get recent posts - async fn get_recent_posts(State(pool): State>) -> impl IntoResponse { - match PostsDatasource::get_recent(&pool).await { - Ok(posts) => Ok(Json(posts)), + async fn get_recent_posts(State(app_state): State) -> impl IntoResponse { + let mut state = app_state.lock().await; + let cached: Option> = state + .cache + .get(String::from("posts:recent")) + .await + .unwrap_or(None); + + if let Some(posts) = cached { + tracing::info!("grabbing recent posts from cache"); + return Ok(Json(posts)); + }; + + match PostsDatasource::get_recent(&state.database).await { + Ok(posts) => { + tracing::info!("grabbing recent posts from database"); + if let p = &posts { + let posts = p.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + String::from("posts:recent"), + &posts, + Some(Expiration::EX(5)), + None, + false, + ) + .await; + }); + }; + + Ok(Json(posts)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } // get the top three posts with the highest view count - async fn get_popular_posts(State(pool): State>) -> impl IntoResponse { - match PostsDatasource::get_popular(&pool).await { - Ok(posts) => Ok(Json(posts)), + async fn get_popular_posts(State(app_state): State) -> impl IntoResponse { + let mut state = app_state.lock().await; + let cached: Option> = state + .cache + .get(String::from("posts:popular")) + .await + .unwrap_or(None); + + if let Some(posts) = cached { + tracing::info!("grabbing popular posts from cache"); + return Ok(Json(posts)); + }; + + match PostsDatasource::get_popular(&state.database).await { + Ok(posts) => { + tracing::info!("grabbing popular posts from database"); + if let p = &posts { + let posts = p.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + String::from("posts:popular"), + &posts, + Some(Expiration::EX(5)), + None, + false, + ) + .await; + }); + }; + + Ok(Json(posts)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } // get the top three posts with the most comments - async fn get_hot_posts(State(pool): State>) -> impl IntoResponse { - match PostsDatasource::get_hot(&pool).await { - Ok(posts) => Ok(Json(posts)), + async fn get_hot_posts(State(app_state): State) -> impl IntoResponse { + let mut state = app_state.lock().await; + let cached: Option> = state + .cache + .get(String::from("posts:hot")) + .await + .unwrap_or(None); + + if let Some(posts) = cached { + tracing::info!("grabbing hot posts from cache"); + return Ok(Json(posts)); + }; + + match PostsDatasource::get_hot(&state.database).await { + Ok(posts) => { + tracing::info!("grabbing hot posts from database"); + if let p = &posts { + let posts = p.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + String::from("posts:hot"), + &posts, + Some(Expiration::EX(5)), + None, + false, + ) + .await; + }); + }; + + Ok(Json(posts)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } // get posts that are featured - async fn get_featured_posts(State(pool): State>) -> impl IntoResponse { - match PostsDatasource::get_featured(&pool).await { - Ok(posts) => Ok(Json(posts)), + async fn get_featured_posts(State(app_state): State) -> impl IntoResponse { + let mut state = app_state.lock().await; + let cached: Option> = state + .cache + .get(String::from("posts:featured")) + .await + .unwrap_or(None); + + if let Some(posts) = cached { + tracing::info!("grabbing featured posts from cache"); + return Ok(Json(posts)); + }; + + match PostsDatasource::get_featured(&state.database).await { + Ok(posts) => { + tracing::info!("grabbing featured posts from database"); + if let p = &posts { + let posts = p.clone(); + let state = app_state.clone(); + + tracing::info!("storing database data in cache"); + tokio::spawn(async move { + let mut s = state.lock().await; + let _ = s + .cache + .set( + String::from("posts:featured"), + &posts, + Some(Expiration::EX(5)), + None, + false, + ) + .await; + }); + }; + + Ok(Json(posts)) + } Err(e) => Err((StatusCode::INTERNAL_SERVER_ERROR, e.to_string())), } } // get rss posts - async fn get_rss_posts(State(pool): State>) -> impl IntoResponse { - match PostsDatasource::get_all(&pool).await { + async fn get_rss_posts(State(app_state): State) -> impl IntoResponse { + let state = app_state.lock().await; + + match PostsDatasource::get_all(&state.database).await { Ok(posts) => { let web_url = std::env::var("BASE_URI_WEB").expect("No environment variable found"); let mapped_posts: HashMap = posts @@ -154,3 +369,46 @@ where { serializer.serialize_str(&date.unwrap().to_rfc3339()) } + +pub fn deserialize_datetime<'de, D>( + deserializer: D, +) -> Result>, D::Error> +where + D: Deserializer<'de>, +{ + struct DateTimeVisitor; + + impl<'de> serde::de::Visitor<'de> for DateTimeVisitor { + type Value = Option>; + + fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { + formatter.write_str("an ISO 8601 formatted date string or null") + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + match chrono::DateTime::parse_from_rfc3339(value) { + Ok(dt) => Ok(Some(dt.with_timezone(&Utc))), + Err(e) => Err(E::custom(format!("Failed to parse datetime: {}", e))), + } + } + + fn visit_none(self) -> Result + where + E: serde::de::Error, + { + Ok(None) + } + + fn visit_some(self, deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_str(self) + } + } + + deserializer.deserialize_option(DateTimeVisitor) +} diff --git a/backend/public/src/state.rs b/backend/public/src/state.rs index ab662a9..cb8140e 100644 --- a/backend/public/src/state.rs +++ b/backend/public/src/state.rs @@ -29,19 +29,21 @@ impl Cache { T: for<'de> serde::Deserialize<'de>, { if !self.inmem.is_connected() { - return Err(Box::new("Are you connected to the cache?".into())); + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Not connected to cache".to_string(), + ))); } - let value: Option = self.inmem.get(key).await?; + let value: Option = self.inmem.get(&key).await?; - let result = match value { - Some(x) => match serde_json::from_value(x) { - Ok(x) => Some(x), - Err(_) => None, + match value { + Some(json_str) => match serde_json::from_str::(&json_str) { + Ok(deserialized) => Ok(Some(deserialized)), + Err(_) => Ok(None), }, - None => None, - }; - Ok(result) + None => Ok(None), + } } pub async fn set( @@ -53,16 +55,20 @@ impl Cache { get: bool, ) -> Result<(), Box> where - T: for<'de> serde::Deserialize<'de>, + T: for<'de> serde::Deserialize<'de> + serde::Serialize, { if !self.inmem.is_connected() { - return Err(Box::new("Are you connected to the cache?".into())); + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Not connected to cache".to_string(), + ))); } - let value: Value = serde_json::to_value(contents)?; + let json_string = serde_json::to_string(contents)?; self.inmem - .set(key, value.to_string(), expiration, set_opts, get) + .set(key, json_string, expiration, set_opts, get) .await?; + Ok(()) } From 075c73bb3ac180110195c86a0622eda9071b8e2f Mon Sep 17 00:00:00 2001 From: "Wyatt J. Miller" Date: Sun, 16 Mar 2025 14:35:50 -0400 Subject: [PATCH 3/3] cleanup, added some boilerplate readmes for backend projects --- .gitignore | 1 + backend/README.md | 9 ++++++++- backend/public/README.md | 20 ++++++++++++++++++++ backend/public/src/routes/posts.rs | 2 +- backend/public/src/state.rs | 1 - backend/task/README.md | 7 +++++++ database/README.md | 7 ++++++- 7 files changed, 43 insertions(+), 4 deletions(-) create mode 100644 backend/public/README.md create mode 100644 backend/task/README.md diff --git a/.gitignore b/.gitignore index 57db87f..5741ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ debug.log .vscode/ .idea/ +dump.rdb diff --git a/backend/README.md b/backend/README.md index b6a4bdb..62e2fa7 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,3 +1,10 @@ # Backend -TODO +## What is this? + +This is just an orginizational way of keeping the backend services together (so I don't lose my mind). + +## Projects + +- [`public`](./public/README.md) - a RESTful API service +- [`task`](./task/README.md) - a task scheduler service diff --git a/backend/public/README.md b/backend/public/README.md new file mode 100644 index 0000000..4aa3b72 --- /dev/null +++ b/backend/public/README.md @@ -0,0 +1,20 @@ +# Backend API + +also known as `public` + +## What is this? + +This is a RESTful API service. Most of the data retrival, requesting, and processing happens here. + +## Things you should know + +### Environment variables + +`public` uses a `.env` file at the root of the project. The file takes standard environment variables (like enviroment variables you would put into a `.bashrc` or ad-hoc into your shell). + +For `public` to work properly, please make sure to first create the `.env` file, then fill out the following environment variables: + +- `RUST_ENV` - needed for letting the service that we are working in either `development` or `production` +- `DATABASE_URL` - needed for connecting to Postgres +- `REDIS_URL` - needed for connecting to the cache (Redis or Valkey) +- `BASE_URI_WEB` - needed for connecting to the frontend user interface of the system to this service diff --git a/backend/public/src/routes/posts.rs b/backend/public/src/routes/posts.rs index 5dbb94f..987bf7b 100644 --- a/backend/public/src/routes/posts.rs +++ b/backend/public/src/routes/posts.rs @@ -1,4 +1,5 @@ use std::collections::HashMap; +use std::fmt; use crate::utils::rss; use crate::{datasources::posts::PostsDatasource, state::AppState}; @@ -13,7 +14,6 @@ use axum::{ use chrono::Utc; use fred::types::Expiration; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use std::fmt; #[derive(sqlx::FromRow, Deserialize, Serialize, Debug, Clone)] pub struct Post { diff --git a/backend/public/src/state.rs b/backend/public/src/state.rs index cb8140e..9f7d25d 100644 --- a/backend/public/src/state.rs +++ b/backend/public/src/state.rs @@ -1,6 +1,5 @@ use fred::interfaces::KeysInterface; use fred::{clients::Pool, prelude::*}; -use serde_json::Value; use sqlx::PgPool; pub type AppState = std::sync::Arc>; diff --git a/backend/task/README.md b/backend/task/README.md new file mode 100644 index 0000000..e8e215c --- /dev/null +++ b/backend/task/README.md @@ -0,0 +1,7 @@ +# Task scheduler + +also known as `task` + +## What is this? + +I don't know yet - hopefully this will be filled out soon. diff --git a/database/README.md b/database/README.md index 052ddde..3c994ab 100644 --- a/database/README.md +++ b/database/README.md @@ -1,6 +1,11 @@ # Database -You can set environment variables either through the command line, the Nix flake (if you are running nix/NixOS), _or_ the `.env` file +You can set environment variables either through the command line, the Nix flake (if you are running nix/NixOS), _or_ the `.env` file. + +Uses the following data storing services: + +- PostgreSQL 16 +- Valkey 8.0.2 (or Redis, haven't tested) ## Create migration database