diff --git a/backend/README.md b/backend/README.md index 62e2fa7..6d5791e 100644 --- a/backend/README.md +++ b/backend/README.md @@ -8,3 +8,5 @@ This is just an orginizational way of keeping the backend services together (so - [`public`](./public/README.md) - a RESTful API service - [`task`](./task/README.md) - a task scheduler service +- [`storage`](./storage/README.md) - an internal storage library +- [`cache`](./cache/README.md) - an internal caching library diff --git a/backend/cache/.gitignore b/backend/cache/.gitignore new file mode 100644 index 0000000..14ee500 --- /dev/null +++ b/backend/cache/.gitignore @@ -0,0 +1,2 @@ +target/ +.env diff --git a/backend/cache/Cargo.lock b/backend/cache/Cargo.lock new file mode 100644 index 0000000..9f2fd44 --- /dev/null +++ b/backend/cache/Cargo.lock @@ -0,0 +1,1014 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "addr2line" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "backtrace" +version = "0.3.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", + "windows-targets", +] + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "bytes" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" + +[[package]] +name = "bytes-utils" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" +dependencies = [ + "bytes", + "either", +] + +[[package]] +name = "cache" +version = "0.1.0" +dependencies = [ + "fred", + "serde", + "serde_json", +] + +[[package]] +name = "cfg-if" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" + +[[package]] +name = "cookie-factory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" + +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fred" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a7b2fd0f08b23315c13b6156f971aeedb6f75fb16a29ac1872d2eabccc1490e" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "bytes-utils", + "float-cmp", + "fred-macros", + "futures", + "log", + "parking_lot", + "rand", + "redis-protocol", + "semver", + "socket2", + "tokio", + "tokio-stream", + "tokio-util", + "url", + "urlencoding", +] + +[[package]] +name = "fred-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1458c6e22d36d61507034d5afecc64f105c1d39712b7ac6ec3b352c423f715cc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.31.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" + +[[package]] +name = "icu_collections" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" + +[[package]] +name = "icu_properties" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "potential_utf", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" + +[[package]] +name = "icu_provider" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +dependencies = [ + "displaydoc", + "icu_locale_core", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "io-uring" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b86e202f00093dcba4275d4636b93ef9dd75d025ae560d2521b45ea28ab49013" +dependencies = [ + "bitflags", + "cfg-if", + "libc", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "libc" +version = "0.2.174" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" + +[[package]] +name = "litemap" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" + +[[package]] +name = "lock_api" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "mio" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.59.0", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.36.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +dependencies = [ + "memchr", +] + +[[package]] +name = "parking_lot" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "potential_utf" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +dependencies = [ + "zerovec", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "redis-protocol" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdba59219406899220fc4cdfd17a95191ba9c9afb719b5fa5a083d63109a9f1" +dependencies = [ + "bytes", + "bytes-utils", + "cookie-factory", + "crc16", + "log", + "nom", +] + +[[package]] +name = "redox_syscall" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d04b7d0ee6b4a0207a0a7adb104d23ecb0b47d6beae7152d0fa34b692b29fd6" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989e6739f80c4ad5b13e0fd7fe89531180375b18520cc8c82080e4dc4035b84f" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "semver" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.140" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "slab" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" + +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "syn" +version = "2.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.46.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +dependencies = [ + "backtrace", + "io-uring", + "libc", + "mio", + "pin-project-lite", + "slab", + "socket2", + "tokio-macros", + "windows-sys 0.52.0", +] + +[[package]] +name = "tokio-macros" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "writeable" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" + +[[package]] +name = "yoke" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/backend/cache/Cargo.toml b/backend/cache/Cargo.toml new file mode 100644 index 0000000..9f859a0 --- /dev/null +++ b/backend/cache/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "cache" +version = "0.1.0" +edition = "2024" + +[dependencies] +fred = "10.1.0" +serde = "1.0.219" +serde_json = "1.0.140" diff --git a/backend/cache/README.md b/backend/cache/README.md new file mode 100644 index 0000000..ee78d89 --- /dev/null +++ b/backend/cache/README.md @@ -0,0 +1,7 @@ +# Caching library + +also known as `cache` + +## What is this? + +An internal caching library that houses functionality needed for a key-value database like Redis or Valkey. This was turned into a library because both `public` and `task` needed functionality within. diff --git a/backend/cache/src/lib.rs b/backend/cache/src/lib.rs new file mode 100644 index 0000000..73bb6b9 --- /dev/null +++ b/backend/cache/src/lib.rs @@ -0,0 +1,70 @@ +pub use fred::{ + clients::Pool, + interfaces::{ClientLike, KeysInterface}, + prelude::*, + types::{Expiration, SetOptions}, +}; + +pub struct Cache { + pub inmem: Pool, +} + +impl Cache { + pub async fn get(&mut self, key: String) -> Result, Box> + where + T: for<'de> serde::Deserialize<'de>, + { + self.is_connected()?; + let value: Option = self.inmem.get(&key).await?; + + match value { + Some(json_str) => match serde_json::from_str::(&json_str) { + Ok(deserialized) => Ok(Some(deserialized)), + Err(_) => Ok(None), + }, + None => Ok(None), + } + } + + pub async fn set( + &mut self, + key: String, + contents: &T, + expiration: Option, + set_opts: Option, + get: bool, + ) -> Result<(), Box> + where + T: for<'de> serde::Deserialize<'de> + serde::Serialize, + { + self.is_connected()?; + let json_string = match serde_json::to_string::(contents) { + Ok(s) => s, + Err(_) => { + return Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Unable to deserialize contents passed to cache".to_string(), + ))); + } + }; + + Ok(self + .inmem + .set(key, json_string, expiration, set_opts, get) + .await?) + } + + pub async fn del(&mut self, key: String) -> Result<(), Box> { + Ok(self.inmem.del(key).await?) + } + + fn is_connected(&mut self) -> Result<(), Box> { + match self.inmem.is_connected() { + true => Ok(()), + false => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Not connected to cache".to_string(), + ))), + } + } +} diff --git a/backend/public/Cargo.lock b/backend/public/Cargo.lock index ef632b9..d1f04b4 100644 --- a/backend/public/Cargo.lock +++ b/backend/public/Cargo.lock @@ -279,6 +279,15 @@ dependencies = [ "either", ] +[[package]] +name = "cache" +version = "0.1.0" +dependencies = [ + "fred", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.1.21" @@ -1322,7 +1331,12 @@ dependencies = [ name = "public" version = "0.1.0" dependencies = [ +<<<<<<< HEAD "axum 0.8.4", +======= + "axum", + "cache", +>>>>>>> origin "chrono", "dotenvy", "fred", @@ -1588,18 +1602,18 @@ checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", @@ -1608,9 +1622,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -1982,9 +1996,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.77" +version = "2.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f35bcdf61fd8e7be6caf75f429fdca8beb3ed76584befb503b1569faee373ed" +checksum = "25aa4ce346d03a6dcd68dd8b4010bcb74e54e62c90c573f394c46eae99aba32d" dependencies = [ "proc-macro2", "quote", diff --git a/backend/public/Cargo.toml b/backend/public/Cargo.toml index bbbacbf..d1cbf32 100644 --- a/backend/public/Cargo.toml +++ b/backend/public/Cargo.toml @@ -25,3 +25,4 @@ serde_json = "1.0.128" chrono = "0.4.38" xml = "0.8.20" fred = "10.1.0" +cache = { version = "*", path = "../cache" } diff --git a/backend/public/src/main.rs b/backend/public/src/main.rs index d3c4ac9..31f86d4 100644 --- a/backend/public/src/main.rs +++ b/backend/public/src/main.rs @@ -1,6 +1,6 @@ use axum::Router; +use cache::ClientLike; use config::config; -use fred::prelude::*; use sqlx::postgres::PgPoolOptions; use std::fs::File; use std::sync::Arc; @@ -8,7 +8,7 @@ use std::time::Duration; use tokio::net::TcpListener; use tokio::signal; use tokio::sync::Mutex; -use tower_governor::{governor::GovernorConfigBuilder, GovernorLayer}; +// use tower_governor::{governor::GovernorConfigBuilder, GovernorLayer}; use tower_http::{ cors::{Any, CorsLayer}, trace::{self, TraceLayer}, @@ -101,13 +101,13 @@ async fn main() { .expect("Failed to connect to database"); let pool_size = 8; - let config = Config::from_url(&redis_url).unwrap(); // TODO: fix the unwrap <<< + let config = cache::Config::from_url(&redis_url).unwrap(); // TODO: fix the unwrap <<< - let redis_pool = Builder::from_config(config) + let redis_pool = cache::Builder::from_config(config) .with_performance_config(|config| { config.default_command_timeout = Duration::from_secs(60); }) - .set_policy(ReconnectPolicy::new_exponential(0, 100, 30_000, 2)) + .set_policy(cache::ReconnectPolicy::new_exponential(0, 100, 30_000, 2)) .build_pool(pool_size) .expect("Failed to create cache pool"); diff --git a/backend/public/src/routes/authors.rs b/backend/public/src/routes/authors.rs index 57f7225..1d6acba 100644 --- a/backend/public/src/routes/authors.rs +++ b/backend/public/src/routes/authors.rs @@ -5,7 +5,7 @@ use axum::{ routing::get, Json, }; -use fred::types::Expiration; +use cache::Expiration; use serde::{Deserialize, Serialize}; use crate::{ diff --git a/backend/public/src/routes/comments.rs b/backend/public/src/routes/comments.rs index 8f8f419..4cde964 100644 --- a/backend/public/src/routes/comments.rs +++ b/backend/public/src/routes/comments.rs @@ -13,8 +13,8 @@ use axum::{ routing::{get, post}, Json, }; +use cache::{Expiration, SetOptions}; use chrono::Utc; -use fred::types::{Expiration, SetOptions}; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug)] diff --git a/backend/public/src/routes/posts.rs b/backend/public/src/routes/posts.rs index feea25b..83a9160 100644 --- a/backend/public/src/routes/posts.rs +++ b/backend/public/src/routes/posts.rs @@ -15,8 +15,8 @@ use axum::{ routing::get, Json, Router, }; +use cache::Expiration; use chrono::Utc; -use fred::types::Expiration; use serde::{Deserialize, Serialize}; use std::collections::HashMap; diff --git a/backend/public/src/routes/projects.rs b/backend/public/src/routes/projects.rs index b3db630..5eb78f1 100644 --- a/backend/public/src/routes/projects.rs +++ b/backend/public/src/routes/projects.rs @@ -1,6 +1,6 @@ use crate::{datasources::projects::ProjectsDatasource, state::AppState, utils::datetime::*}; use axum::{extract::State, http::StatusCode, response::IntoResponse, routing::get, Json, Router}; -use fred::types::Expiration; +use cache::Expiration; use serde::{Deserialize, Serialize}; #[derive(sqlx::FromRow, Deserialize, Serialize, Debug, Clone)] diff --git a/backend/public/src/state.rs b/backend/public/src/state.rs index 751bfc6..77e920e 100644 --- a/backend/public/src/state.rs +++ b/backend/public/src/state.rs @@ -1,83 +1,17 @@ -use fred::interfaces::KeysInterface; -use fred::{clients::Pool, prelude::*}; use sqlx::PgPool; pub type AppState = std::sync::Arc>; pub struct AppInternalState { pub database: sqlx::postgres::PgPool, - pub cache: Cache, -} - -pub struct Cache { - pub inmem: Pool, + pub cache: cache::Cache, } impl AppInternalState { - pub fn new(database: PgPool, cache: Pool) -> Self { + pub fn new(database: PgPool, cache: cache::Pool) -> Self { AppInternalState { database, - cache: Cache { inmem: cache }, - } - } -} - -impl Cache { - pub async fn get(&mut self, key: String) -> Result, Box> - where - T: for<'de> serde::Deserialize<'de>, - { - self.is_connected()?; - let value: Option = self.inmem.get(&key).await?; - - match value { - Some(json_str) => match serde_json::from_str::(&json_str) { - Ok(deserialized) => Ok(Some(deserialized)), - Err(_) => Ok(None), - }, - None => Ok(None), - } - } - - pub async fn set( - &mut self, - key: String, - contents: &T, - expiration: Option, - set_opts: Option, - get: bool, - ) -> Result<(), Box> - where - T: for<'de> serde::Deserialize<'de> + serde::Serialize, - { - self.is_connected()?; - let json_string = match serde_json::to_string::(contents) { - Ok(s) => s, - Err(_) => { - return Err(Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "Unable to deserialize contents passed to cache".to_string(), - ))) - } - }; - - Ok(self - .inmem - .set(key, json_string, expiration, set_opts, get) - .await?) - } - - pub async fn del(&mut self, key: String) -> Result<(), Box> { - Ok(self.inmem.del(key).await?) - } - - fn is_connected(&mut self) -> Result<(), Box> { - match self.inmem.is_connected() { - true => Ok(()), - false => Err(Box::new(std::io::Error::new( - std::io::ErrorKind::Other, - "Not connected to cache".to_string(), - ))), + cache: cache::Cache { inmem: cache }, } } } diff --git a/backend/public/src/utils/rss.rs b/backend/public/src/utils/rss.rs index 851e100..133b57a 100644 --- a/backend/public/src/utils/rss.rs +++ b/backend/public/src/utils/rss.rs @@ -73,17 +73,18 @@ pub fn generate_rss( format!( r#" - - {safe_title} - {safe_description} - {link} - en-us - 60 - Kyouma 1.0.0-SE - - {} - - "#, + + {safe_title} + {safe_description} + {link} + en-us + 60 + Kyouma 1.0.0-SE + + {} + + + "#, rss_entries ) } diff --git a/backend/public/src/utils/sitemap.rs b/backend/public/src/utils/sitemap.rs index 00f735a..628b8b4 100644 --- a/backend/public/src/utils/sitemap.rs +++ b/backend/public/src/utils/sitemap.rs @@ -23,7 +23,6 @@ impl SitemapEntry { pub fn generate_sitemap(entries: &HashMap) -> String { let urls = entries .values() - .into_iter() .map(|entry| entry.to_item()) .collect::(); format!( @@ -39,21 +38,21 @@ pub fn generate_sitemap(entries: &HashMap) -> String { pub fn get_static_pages(entries: &mut HashMap, web_url: &String) { entries.insert( - (entries.len() + 1).to_string(), + "10000".to_string(), SitemapEntry { location: web_url.clone(), lastmod: chrono::Utc::now(), }, ); entries.insert( - (entries.len() + 1).to_string(), + "10001".to_string(), SitemapEntry { location: format!("{}/posts", web_url), lastmod: chrono::Utc::now(), }, ); entries.insert( - (entries.len() + 1).to_string(), + "10002".to_string(), SitemapEntry { location: format!("{}/projects", web_url), lastmod: chrono::Utc::now(), diff --git a/backend/storage/README.md b/backend/storage/README.md new file mode 100644 index 0000000..7cb4034 --- /dev/null +++ b/backend/storage/README.md @@ -0,0 +1,7 @@ +# Storage library + +also known as `storage` + +## What is this? + +An internal storage library. This was needed because both `public` and `task` needed storage functionality. Additionally, this helps maintainability and avoids duplicate code. diff --git a/backend/storage/src/services/aws.rs b/backend/storage/src/services/aws.rs index aae82d0..a87306c 100644 --- a/backend/storage/src/services/aws.rs +++ b/backend/storage/src/services/aws.rs @@ -4,7 +4,7 @@ use aws_config::{BehaviorVersion, Region}; use aws_sdk_s3::{Client, Config, config::Credentials}; use std::env; -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct S3ClientConfig { pub access_key: String, secret_key: String, @@ -13,11 +13,29 @@ pub struct S3ClientConfig { region: String, } +#[derive(Clone)] pub struct S3Client { client: Client, + pub client_config: S3ClientConfig, } impl S3ClientConfig { + pub fn new( + access_key: &str, + secret_key: &str, + endpoint: &str, + bucket: &str, + region: &str, + ) -> Result> { + Ok(S3ClientConfig { + access_key: access_key.to_owned(), + secret_key: secret_key.to_owned(), + endpoint: endpoint.to_owned(), + bucket: bucket.to_owned(), + region: region.to_owned(), + }) + } + pub fn from_env() -> Result> { Ok(S3ClientConfig { access_key: env::var("S3_ACCESS_KEY") @@ -52,6 +70,7 @@ impl S3Client { Self { client: Client::from_conf(s3_config), + client_config: config.clone(), } } } @@ -99,3 +118,9 @@ impl ObjectStorageClient for S3Client { todo!("not impl") } } + +impl Default for S3ClientConfig { + fn default() -> Self { + S3ClientConfig::from_env().unwrap() + } +} diff --git a/backend/task/Cargo.lock b/backend/task/Cargo.lock index 48ac821..4a55da8 100644 --- a/backend/task/Cargo.lock +++ b/backend/task/Cargo.lock @@ -71,6 +71,12 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "arc-swap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" + [[package]] name = "async-channel" version = "1.9.0" @@ -796,6 +802,15 @@ dependencies = [ "either", ] +[[package]] +name = "cache" +version = "0.1.0" +dependencies = [ + "fred", + "serde", + "serde_json", +] + [[package]] name = "cc" version = "1.1.21" @@ -877,6 +892,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "cookie-factory" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "396de984970346b0d9e93d1415082923c679e5ae5c3ee3dcbd104f5610af126b" + [[package]] name = "core-foundation" version = "0.9.4" @@ -940,6 +961,12 @@ dependencies = [ "regex", ] +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + [[package]] name = "crc32fast" version = "1.4.2" @@ -1227,6 +1254,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "flume" version = "0.11.0" @@ -1274,6 +1310,43 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fred" +version = "10.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a7b2fd0f08b23315c13b6156f971aeedb6f75fb16a29ac1872d2eabccc1490e" +dependencies = [ + "arc-swap", + "async-trait", + "bytes", + "bytes-utils", + "float-cmp", + "fred-macros", + "futures", + "log", + "parking_lot", + "rand 0.8.5", + "redis-protocol", + "semver", + "socket2", + "tokio", + "tokio-stream", + "tokio-util", + "url", + "urlencoding", +] + +[[package]] +name = "fred-macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1458c6e22d36d61507034d5afecc64f105c1d39712b7ac6ec3b352c423f715cc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2695,6 +2768,20 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "redis-protocol" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdba59219406899220fc4cdfd17a95191ba9c9afb719b5fa5a083d63109a9f1" +dependencies = [ + "bytes", + "bytes-utils", + "cookie-factory", + "crc16", + "log", + "nom 7.1.3", +] + [[package]] name = "redox_syscall" version = "0.5.4" @@ -3085,18 +3172,18 @@ checksum = "f79dfe2d285b0488816f30e700a7438c5a73d816b5b7d3ac72fbc48b0d185e03" [[package]] name = "serde" -version = "1.0.210" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.210" +version = "1.0.219" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" dependencies = [ "proc-macro2", "quote", @@ -3105,9 +3192,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.128" +version = "1.0.140" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" +checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" dependencies = [ "itoa", "memchr", @@ -3580,6 +3667,7 @@ dependencies = [ name = "task" version = "0.1.0" dependencies = [ + "cache", "chrono", "dotenvy", "futures", diff --git a/backend/task/Cargo.toml b/backend/task/Cargo.toml index ca3c96d..099ea6c 100644 --- a/backend/task/Cargo.toml +++ b/backend/task/Cargo.toml @@ -7,6 +7,7 @@ edition = "2021" [dependencies] storage = { version = "0.1.0", path = "../storage" } +cache = { version = "0.1.0", path = "../cache" } tokio = { version = "1.19.2", features = ["full"] } reqwest = { version = "0.12.20", features = ["json", "rustls-tls"] } job_scheduler = "1.2.1" diff --git a/backend/task/README.md b/backend/task/README.md index dd34a6e..0baadc7 100644 --- a/backend/task/README.md +++ b/backend/task/README.md @@ -12,4 +12,9 @@ This is a task runner/scheduler programs that will fire off various tasks. These For `task` to work properly, please make sure to first create the `.env` file, then fill out the following environment variables: +- `BASE_URI_API` - needed for communicating with `public` - `DATABASE_URL` - needed for communicating to Postgres +- `REDIS_URL` - needed for communicating with the cache (Redis or Valkey) +- `S3_ACCESS_KEY` - needed for Amazon S3 (or compatible services) storage +- `S3_SECRET_KEY` - needed for Amazon S3 (or compatible services) storage +- `S3_BUCKET` - needed for Amazon S3 (or compatible services) storage diff --git a/backend/task/src/main.rs b/backend/task/src/main.rs index 89eea65..0fa7fbd 100644 --- a/backend/task/src/main.rs +++ b/backend/task/src/main.rs @@ -1,8 +1,10 @@ +use cache::ClientLike; use chrono::Utc; use sqlx::{postgres::PgPoolOptions, Pool, Postgres}; use std::env; use std::sync::Arc; use std::time::Duration; +use storage::services::aws; use tasks::*; //mod config; @@ -11,6 +13,8 @@ mod utils; pub struct TaskManager<'a> { pool: Pool, + cache: cache::Pool, + s3_client: aws::S3Client, jobs: Vec, last_activated: Option>, last_job: Option, @@ -49,6 +53,8 @@ async fn main() { println!("Hello, world!"); dotenvy::dotenv().unwrap(); + + // setup database let database_url = env::var("DATABASE_URL").expect("Environment variable DATABASE_URL is not found"); let pool = PgPoolOptions::new() @@ -58,7 +64,35 @@ async fn main() { .await .expect("Failed to connect to the database"); - let mut manager = TaskManager::new(pool); + // setup redis/valkey + let redis_url = match std::env::var("REDIS_URL").unwrap().as_str() { + // TODO: fix the unwrap ^ + "" => "redis://localhost:6379".to_string(), + x => x.to_string(), + }; + + let pool_size = 8; + let config = cache::Config::from_url(&redis_url).unwrap(); // TODO: fix the unwrap <<< + + let redis_pool = cache::Builder::from_config(config) + .with_performance_config(|config| { + config.default_command_timeout = Duration::from_secs(60); + }) + .set_policy(cache::ReconnectPolicy::new_exponential(0, 100, 30_000, 2)) + .build_pool(pool_size) + .expect("Failed to create cache pool"); + + if std::env::var("REDIS_URL").unwrap() != "" { + // TODO: fix the unwrap ^ + redis_pool.init().await.expect("Failed to connect to cache"); + let _ = redis_pool.flushall::(false).await; + } + + // setup storage + let s3_client_config = aws::S3ClientConfig::from_env().unwrap(); + let s3_client = aws::S3Client::new(&s3_client_config); + + let mut manager = TaskManager::new(pool, redis_pool, s3_client); manager.register_jobs().await.unwrap(); loop { @@ -68,9 +102,11 @@ async fn main() { } impl<'a> TaskManager<'a> { - fn new(pool: Pool) -> Self { + fn new(pool: Pool, cache: cache::Pool, s3_client: aws::S3Client) -> Self { TaskManager { pool, + cache, + s3_client, jobs: Vec::new(), last_activated: None, last_job: None, @@ -100,11 +136,15 @@ impl<'a> TaskManager<'a> { } 2 => { let pool = Arc::new(self.pool.clone()); - Box::new(move || upload_rss::register(&pool)) + let cache = Arc::new(self.cache.clone()); + let s3_client = Arc::new(self.s3_client.clone()); + Box::new(move || upload_rss::register(&pool, &cache, &s3_client)) } 3 => { let pool = Arc::new(self.pool.clone()); - Box::new(move || upload_sitemap::register(&pool)) + let cache = Arc::new(self.cache.clone()); + let s3_client = Arc::new(self.s3_client.clone()); + Box::new(move || upload_sitemap::register(&pool, &cache, &s3_client)) } id => return Err(format!("Unknown task_id: {}", id).into()), }; diff --git a/backend/task/src/tasks/upload_rss.rs b/backend/task/src/tasks/upload_rss.rs index af6c6aa..7affefd 100644 --- a/backend/task/src/tasks/upload_rss.rs +++ b/backend/task/src/tasks/upload_rss.rs @@ -2,19 +2,24 @@ use crate::utils::{ request::{Request, Response}, task_log, }; -use storage::services::{ - aws::{S3Client, S3ClientConfig}, - ObjectStorageClient, -}; +use cache::KeysInterface; +use storage::services::{aws::S3Client, ObjectStorageClient}; -pub fn register(pool: &sqlx::Pool) { +pub fn register(pool: &sqlx::Pool, cache: &cache::Pool, s3_client: &S3Client) { let p = pool.clone(); + let c = cache.clone(); + let s3 = s3_client.to_owned(); + tokio::spawn(async move { - let _ = upload_rss(&p).await; + let _ = upload_rss(&p, &c, s3).await; }); } -async fn upload_rss(pool: &sqlx::Pool) -> Result<(), Box> { +async fn upload_rss( + pool: &sqlx::Pool, + cache: &cache::Pool, + s3_client: S3Client, +) -> Result<(), Box> { // start task logging task_log::start(2, pool).await?; @@ -25,15 +30,37 @@ async fn upload_rss(pool: &sqlx::Pool) -> Result<(), Box = &cache.get(String::from("rss")).await.unwrap_or(None); + let cache_clone = cache.clone(); + if let Some(cached_value) = cached { + if *cached_value == rss { + println!("Response is the same in the cache, exiting"); + return Ok(()); + } + } + let r = rss.clone(); + let _ = s3_client .put_object( - client_config.bucket.as_str(), + s3_client.client_config.bucket.as_str(), "feed.xml", rss.as_bytes().to_vec(), ) - .await; + .await?; + + tokio::spawn(async move { + cache_clone + .set::( + String::from("rss"), + &r, + Some(cache::Expiration::EX(3600)), + None, + false, + ) + .await + .unwrap(); + }); + println!("Finished uploading RSS feed"); } diff --git a/backend/task/src/tasks/upload_sitemap.rs b/backend/task/src/tasks/upload_sitemap.rs index 6f5a760..8ceee5e 100644 --- a/backend/task/src/tasks/upload_sitemap.rs +++ b/backend/task/src/tasks/upload_sitemap.rs @@ -2,20 +2,23 @@ use crate::utils::{ request::{Request, Response}, task_log, }; -use storage::services::{ - aws::{S3Client, S3ClientConfig}, - ObjectStorageClient, -}; +use cache::KeysInterface; +use storage::services::{aws::S3Client, ObjectStorageClient}; -pub fn register(pool: &sqlx::Pool) { +pub fn register(pool: &sqlx::Pool, cache: &cache::Pool, s3_client: &S3Client) { let p = pool.clone(); + let c = cache.clone(); + let s3 = s3_client.to_owned(); + tokio::spawn(async move { - let _ = upload_sitemap(&p).await; + let _ = upload_sitemap(&p, &c, s3).await; }); } async fn upload_sitemap( pool: &sqlx::Pool, + cache: &cache::Pool, + s3_client: S3Client, ) -> Result<(), Box> { // start task logging task_log::start(3, pool).await?; @@ -27,15 +30,36 @@ async fn upload_sitemap( // upload the sucker to obj storage if let Response::Xml(sitemap) = sitemap_result { - let client_config = S3ClientConfig::from_env().unwrap(); - let s3_client = S3Client::new(&client_config); + let cached: &Option = &cache.get(String::from("sitemap")).await.unwrap_or(None); + let cache_clone = cache.clone(); + if let Some(cached_value) = cached { + if *cached_value == sitemap { + println!("Response is the same in the cache, exiting"); + return Ok(()); + } + } + let s = sitemap.clone(); + let _ = s3_client .put_object( - client_config.bucket.as_str(), + s3_client.client_config.bucket.as_str(), "sitemap.xml", sitemap.as_bytes().to_vec(), ) - .await; + .await?; + + tokio::spawn(async move { + cache_clone + .set::( + String::from("sitemap"), + &s, + Some(cache::Expiration::EX(3600)), + None, + false, + ) + .await + .unwrap(); + }); println!("Finished uploading sitemap!"); } diff --git a/frontend/components/Footer.tsx b/frontend/components/Footer.tsx index efb8e5c..a6fa7a3 100644 --- a/frontend/components/Footer.tsx +++ b/frontend/components/Footer.tsx @@ -2,7 +2,7 @@ import * as hi from "jsr:@preact-icons/hi2"; export default function Footer() { return ( -