From a64b8fdceb175dcf1adbd2813ab19c402ca6e4ab Mon Sep 17 00:00:00 2001 From: "Wyatt J. Miller" Date: Mon, 30 Jun 2025 22:58:52 -0400 Subject: [PATCH] impl storage library --- backend/task/Cargo.lock | 469 ++++++++++++++++++++++- backend/task/Cargo.toml | 3 +- backend/task/src/main.rs | 1 + backend/task/src/tasks/upload_rss.rs | 21 +- backend/task/src/tasks/upload_sitemap.rs | 23 +- backend/task/src/utils/upload.rs | 73 ---- 6 files changed, 485 insertions(+), 105 deletions(-) diff --git a/backend/task/Cargo.lock b/backend/task/Cargo.lock index 592b337..48ac821 100644 --- a/backend/task/Cargo.lock +++ b/backend/task/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "RustyXML" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5" + [[package]] name = "addr2line" version = "0.24.1" @@ -65,6 +71,52 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-compression" +version = "0.4.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4" +dependencies = [ + "flate2", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "async-lock" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +dependencies = [ + "event-listener 5.3.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-trait" +version = "0.1.88" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atoi" version = "2.0.0" @@ -105,7 +157,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "hex", "http 1.2.0", "ring", @@ -167,7 +219,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "http-body 0.4.6", "percent-encoding", @@ -196,7 +248,7 @@ dependencies = [ "aws-smithy-xml", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "hex", "hmac", "http 0.2.12", @@ -226,7 +278,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "regex-lite", "tracing", @@ -248,7 +300,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "regex-lite", "tracing", @@ -271,7 +323,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-xml", "aws-types", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "regex-lite", "tracing", @@ -438,7 +490,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand", + "fastrand 2.3.0", "http 0.2.12", "http 1.2.0", "http-body 0.4.6", @@ -515,6 +567,111 @@ dependencies = [ "tracing", ] +[[package]] +name = "azure_core" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.15", + "hmac", + "http-types", + "once_cell", + "paste", + "pin-project", + "quick-xml", + "rand 0.8.5", + "reqwest", + "rustc_version", + "serde", + "serde_json", + "sha2", + "time", + "tracing", + "url", + "uuid 1.15.1", +] + +[[package]] +name = "azure_core" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82c33c072c9d87777262f35abfe2a64b609437076551d4dac8373e60f0e3fde9" +dependencies = [ + "async-lock", + "async-trait", + "bytes", + "futures", + "pin-project", + "rustc_version", + "serde", + "serde_json", + "tracing", + "typespec", + "typespec_client_core", +] + +[[package]] +name = "azure_storage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126" +dependencies = [ + "RustyXML", + "async-lock", + "async-trait", + "azure_core 0.21.0", + "bytes", + "serde", + "serde_derive", + "time", + "tracing", + "url", + "uuid 1.15.1", +] + +[[package]] +name = "azure_storage_blobs" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4" +dependencies = [ + "RustyXML", + "azure_core 0.21.0", + "azure_storage", + "azure_svc_blobstorage", + "bytes", + "futures", + "serde", + "serde_derive", + "serde_json", + "time", + "tracing", + "url", + "uuid 1.15.1", +] + +[[package]] +name = "azure_svc_blobstorage" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6" +dependencies = [ + "azure_core 0.21.0", + "bytes", + "futures", + "log", + "once_cell", + "serde", + "serde_json", + "time", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -536,6 +693,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.7" @@ -872,6 +1035,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" dependencies = [ "powerfmt", + "serde", ] [[package]] @@ -909,6 +1073,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" +[[package]] +name = "dyn-clone" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" + [[package]] name = "ecdsa" version = "0.14.8" @@ -995,6 +1165,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + [[package]] name = "event-listener" version = "5.3.1" @@ -1006,6 +1182,25 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.3.1", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" +dependencies = [ + "instant", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -1022,6 +1217,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.0" @@ -1134,6 +1339,21 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +[[package]] +name = "futures-lite" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce" +dependencies = [ + "fastrand 1.9.0", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.30" @@ -1185,6 +1405,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "getrandom" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" +dependencies = [ + "cfg-if", + "libc", + "wasi 0.9.0+wasi-snapshot-preview1", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -1404,6 +1635,26 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-types" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" +dependencies = [ + "anyhow", + "async-channel", + "base64 0.13.1", + "futures-lite", + "infer", + "pin-project-lite", + "rand 0.7.3", + "serde", + "serde_json", + "serde_qs", + "serde_urlencoded", + "url", +] + [[package]] name = "httparse" version = "1.10.0" @@ -1676,6 +1927,21 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "infer" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" + +[[package]] +name = "instant" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" +dependencies = [ + "cfg-if", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -2137,6 +2403,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -2229,6 +2515,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" version = "0.11.8" @@ -2299,6 +2595,19 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom 0.1.16", + "libc", + "rand_chacha 0.2.2", + "rand_core 0.5.1", + "rand_hc", +] + [[package]] name = "rand" version = "0.8.5" @@ -2320,6 +2629,16 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core 0.5.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -2340,6 +2659,15 @@ dependencies = [ "rand_core 0.9.3", ] +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom 0.1.16", +] + [[package]] name = "rand_core" version = "0.6.4" @@ -2358,6 +2686,15 @@ dependencies = [ "getrandom 0.3.3", ] +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "redox_syscall" version = "0.5.4" @@ -2423,10 +2760,12 @@ version = "0.12.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "encoding_rs", "futures-core", + "futures-util", "h2 0.4.10", "http 1.2.0", "http-body 1.0.1", @@ -2451,12 +2790,14 @@ dependencies = [ "tokio", "tokio-native-tls", "tokio-rustls 0.26.2", + "tokio-util", "tower", "tower-http", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", "webpki-roots 1.0.1", ] @@ -2774,6 +3115,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_qs" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6" +dependencies = [ + "percent-encoding", + "serde", + "thiserror 1.0.63", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -2960,7 +3312,7 @@ dependencies = [ "crc", "crossbeam-queue", "either", - "event-listener", + "event-listener 5.3.1", "futures-channel", "futures-core", "futures-intrusive", @@ -3141,6 +3493,20 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "storage" +version = "0.1.0" +dependencies = [ + "async-trait", + "aws-config", + "aws-sdk-s3", + "azure_core 0.25.0", + "azure_storage", + "azure_storage_blobs", + "thiserror 2.0.12", + "tokio", +] + [[package]] name = "stringprep" version = "0.1.5" @@ -3214,8 +3580,6 @@ dependencies = [ name = "task" version = "0.1.0" dependencies = [ - "aws-config", - "aws-sdk-s3", "chrono", "dotenvy", "futures", @@ -3226,6 +3590,7 @@ dependencies = [ "serde", "serde_yml", "sqlx", + "storage", "tokio", "tracing", "tracing-subscriber", @@ -3238,7 +3603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" dependencies = [ "cfg-if", - "fastrand", + "fastrand 2.3.0", "once_cell", "rustix", "windows-sys 0.59.0", @@ -3301,6 +3666,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" dependencies = [ "deranged", + "itoa", + "js-sys", "num-conv", "powerfmt", "serde", @@ -3551,6 +3918,56 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" +[[package]] +name = "typespec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c7a952f1f34257f945fc727b20defe7a3c01c05ddd42925977626cfa6e62ab" +dependencies = [ + "base64 0.22.1", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "typespec_client_core" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5879ce67ba9e525fe088c882ede1337c32c3f80e83e72d9fd3cc6c8e05bcb3d7" +dependencies = [ + "async-trait", + "base64 0.22.1", + "bytes", + "dyn-clone", + "futures", + "getrandom 0.2.15", + "pin-project", + "rand 0.8.5", + "reqwest", + "serde", + "serde_json", + "time", + "tokio", + "tracing", + "typespec", + "typespec_macros", + "url", + "uuid 1.15.1", +] + +[[package]] +name = "typespec_macros" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cbccdbe531c8d553812a609bdb70c0d1002ad91333498e18df42c98744b15cc" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "syn", +] + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -3605,6 +4022,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3633,6 +4051,10 @@ name = "uuid" version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" +dependencies = [ + "getrandom 0.3.3", + "serde", +] [[package]] name = "valuable" @@ -3658,6 +4080,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "waker-fn" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7" + [[package]] name = "want" version = "0.3.1" @@ -3667,6 +4095,12 @@ dependencies = [ "try-lock", ] +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -3759,6 +4193,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-streams" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "web-sys" version = "0.3.77" diff --git a/backend/task/Cargo.toml b/backend/task/Cargo.toml index 71e253e..ca3c96d 100644 --- a/backend/task/Cargo.toml +++ b/backend/task/Cargo.toml @@ -6,6 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +storage = { version = "0.1.0", path = "../storage" } tokio = { version = "1.19.2", features = ["full"] } reqwest = { version = "0.12.20", features = ["json", "rustls-tls"] } job_scheduler = "1.2.1" @@ -21,7 +22,5 @@ futures = "0.3.30" markdown = "1.0.0-alpha.20" serde = { version = "*", features = ["derive"] } serde_yml = "*" -aws-sdk-s3 = "1.94.0" -aws-config = "1.8" tracing = "0.1" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } diff --git a/backend/task/src/main.rs b/backend/task/src/main.rs index f96f0cb..89eea65 100644 --- a/backend/task/src/main.rs +++ b/backend/task/src/main.rs @@ -29,6 +29,7 @@ pub struct TaskLog { #[derive(Debug)] pub enum TaskStatus { Pending(String), + Running(String), Completed(String), Failed(String), } diff --git a/backend/task/src/tasks/upload_rss.rs b/backend/task/src/tasks/upload_rss.rs index fd41055..af6c6aa 100644 --- a/backend/task/src/tasks/upload_rss.rs +++ b/backend/task/src/tasks/upload_rss.rs @@ -1,7 +1,10 @@ use crate::utils::{ request::{Request, Response}, task_log, - {upload::S3ClientConfig, *}, +}; +use storage::services::{ + aws::{S3Client, S3ClientConfig}, + ObjectStorageClient, }; pub fn register(pool: &sqlx::Pool) { @@ -23,14 +26,14 @@ async fn upload_rss(pool: &sqlx::Pool) -> Result<(), Box) { @@ -14,7 +17,7 @@ pub fn register(pool: &sqlx::Pool) { async fn upload_sitemap( pool: &sqlx::Pool, ) -> Result<(), Box> { - // TODO:: get sitemap and upload it to bucket?? + // start task logging task_log::start(3, pool).await?; // get request and request the things @@ -25,14 +28,14 @@ async fn upload_sitemap( // upload the sucker to obj storage if let Response::Xml(sitemap) = sitemap_result { let client_config = S3ClientConfig::from_env().unwrap(); - let s3_client = upload::create_s3_client(&client_config).await.unwrap(); - let _ = upload::upload( - &s3_client, - client_config.bucket.as_str(), - "sitemap.xml", - sitemap.as_str(), - ) - .await; + let s3_client = S3Client::new(&client_config); + let _ = s3_client + .put_object( + client_config.bucket.as_str(), + "sitemap.xml", + sitemap.as_bytes().to_vec(), + ) + .await; println!("Finished uploading sitemap!"); } diff --git a/backend/task/src/utils/upload.rs b/backend/task/src/utils/upload.rs index 9dbb46d..e69de29 100644 --- a/backend/task/src/utils/upload.rs +++ b/backend/task/src/utils/upload.rs @@ -1,73 +0,0 @@ -use aws_config::{BehaviorVersion, Region}; -use aws_sdk_s3::{config::Credentials, Client, Config}; -use std::env; - -#[derive(Debug)] -pub struct S3ClientConfig { - pub access_key: String, - secret_key: String, - endpoint: String, - pub bucket: String, - region: String, -} - -impl S3ClientConfig { - pub fn from_env() -> Result> { - Ok(S3ClientConfig { - access_key: env::var("LINODE_ACCESS_KEY") - .map_err(|_| "LINODE_ACCESS_KEY environment variable not set")?, - secret_key: env::var("LINODE_SECRET_KEY") - .map_err(|_| "LINODE_SECRET_KEY environment variable not set")?, - endpoint: env::var("LINODE_ENDPOINT") - .unwrap_or_else(|_| "us-ord-1.linodeobjects.com".to_string()), - bucket: env::var("LINODE_BUCKET") - .map_err(|_| "LINODE_BUCKET environment variable not set")?, - region: env::var("LINODE_REGION").unwrap_or_else(|_| "us-ord".to_string()), - }) - } -} - -pub async fn create_s3_client( - config: &S3ClientConfig, -) -> Result> { - let credentials = Credentials::new( - &config.access_key, - &config.secret_key, - None, - None, - "linode-object-storage", - ); - - let s3_config = Config::builder() - .behavior_version(BehaviorVersion::latest()) - .region(Region::new(config.region.clone())) - .endpoint_url(format!("https://{}", config.endpoint)) - .credentials_provider(credentials) - .force_path_style(false) - .build(); - - Ok(Client::from_conf(s3_config)) -} - -pub async fn upload( - client: &Client, - bucket: &str, - key: &str, - content: &str, -) -> Result<(), Box> { - println!("Uploading to Linode Object Storage..."); - println!("Bucket: {}", bucket); - - let put_object_req = client - .put_object() - .bucket(bucket) - .key(key) - .body(content.as_bytes().to_vec().into()) - .acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead) - .content_type("application/xml") - .send() - .await?; - - println!("Upload successful! ETag: {:?}", put_object_req.e_tag()); - Ok(()) -}