wymiller/my-website-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

impl storage library

Browse Source
This commit is contained in:
Wyatt J. Miller
2025-06-30 22:58:52 -04:00
parent a6b4f6917b
commit a64b8fdceb
6 changed files with 485 additions and 105 deletions
Download Patch File Download Diff File Expand all files Collapse all files

469
backend/task/Cargo.lock generated
View File

@ -2,6 +2,12 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 4 version = 4
[[package]]
name = "RustyXML"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b5ace29ee3216de37c0546865ad08edef58b0f9e76838ed8959a84a990e58c5"
[[package]] [[package]]
name = "addr2line" name = "addr2line"
version = "0.24.1" version = "0.24.1"
@ -65,6 +71,52 @@ version = "1.0.98"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
[[package]]
name = "async-channel"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35"
dependencies = [
"concurrent-queue",
"event-listener 2.5.3",
"futures-core",
]
[[package]]
name = "async-compression"
version = "0.4.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40f6024f3f856663b45fd0c9b6f2024034a702f453549449e0d84a305900dad4"
dependencies = [
"flate2",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
]
[[package]]
name = "async-lock"
version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18"
dependencies = [
"event-listener 5.3.1",
"event-listener-strategy",
"pin-project-lite",
]
[[package]]
name = "async-trait"
version = "0.1.88"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "atoi" name = "atoi"
version = "2.0.0" version = "2.0.0"
@ -105,7 +157,7 @@ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"aws-types", "aws-types",
"bytes", "bytes",
"fastrand", "fastrand 2.3.0",
"hex", "hex",
"http 1.2.0", "http 1.2.0",
"ring", "ring",
@ -167,7 +219,7 @@ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"aws-types", "aws-types",
"bytes", "bytes",
"fastrand", "fastrand 2.3.0",
"http 0.2.12", "http 0.2.12",
"http-body 0.4.6", "http-body 0.4.6",
"percent-encoding", "percent-encoding",
@ -196,7 +248,7 @@ dependencies = [
"aws-smithy-xml", "aws-smithy-xml",
"aws-types", "aws-types",
"bytes", "bytes",
"fastrand", "fastrand 2.3.0",
"hex", "hex",
"hmac", "hmac",
"http 0.2.12", "http 0.2.12",
@ -226,7 +278,7 @@ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"aws-types", "aws-types",
"bytes", "bytes",
"fastrand", "fastrand 2.3.0",
"http 0.2.12", "http 0.2.12",
"regex-lite", "regex-lite",
"tracing", "tracing",
@ -248,7 +300,7 @@ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"aws-types", "aws-types",
"bytes", "bytes",
"fastrand", "fastrand 2.3.0",
"http 0.2.12", "http 0.2.12",
"regex-lite", "regex-lite",
"tracing", "tracing",
@ -271,7 +323,7 @@ dependencies = [
"aws-smithy-types", "aws-smithy-types",
"aws-smithy-xml", "aws-smithy-xml",
"aws-types", "aws-types",
"fastrand", "fastrand 2.3.0",
"http 0.2.12", "http 0.2.12",
"regex-lite", "regex-lite",
"tracing", "tracing",
@ -438,7 +490,7 @@ dependencies = [
"aws-smithy-runtime-api", "aws-smithy-runtime-api",
"aws-smithy-types", "aws-smithy-types",
"bytes", "bytes",
"fastrand", "fastrand 2.3.0",
"http 0.2.12", "http 0.2.12",
"http 1.2.0", "http 1.2.0",
"http-body 0.4.6", "http-body 0.4.6",
@ -515,6 +567,111 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "azure_core"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b552ad43a45a746461ec3d3a51dfb6466b4759209414b439c165eb6a6b7729e"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"dyn-clone",
"futures",
"getrandom 0.2.15",
"hmac",
"http-types",
"once_cell",
"paste",
"pin-project",
"quick-xml",
"rand 0.8.5",
"reqwest",
"rustc_version",
"serde",
"serde_json",
"sha2",
"time",
"tracing",
"url",
"uuid 1.15.1",
]
[[package]]
name = "azure_core"
version = "0.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82c33c072c9d87777262f35abfe2a64b609437076551d4dac8373e60f0e3fde9"
dependencies = [
"async-lock",
"async-trait",
"bytes",
"futures",
"pin-project",
"rustc_version",
"serde",
"serde_json",
"tracing",
"typespec",
"typespec_client_core",
]
[[package]]
name = "azure_storage"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59f838159f4d29cb400a14d9d757578ba495ae64feb07a7516bf9e4415127126"
dependencies = [
"RustyXML",
"async-lock",
"async-trait",
"azure_core 0.21.0",
"bytes",
"serde",
"serde_derive",
"time",
"tracing",
"url",
"uuid 1.15.1",
]
[[package]]
name = "azure_storage_blobs"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "97e83c3636ae86d9a6a7962b2112e3b19eb3903915c50ce06ff54ff0a2e6a7e4"
dependencies = [
"RustyXML",
"azure_core 0.21.0",
"azure_storage",
"azure_svc_blobstorage",
"bytes",
"futures",
"serde",
"serde_derive",
"serde_json",
"time",
"tracing",
"url",
"uuid 1.15.1",
]
[[package]]
name = "azure_svc_blobstorage"
version = "0.21.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e6c6f20c5611b885ba94c7bae5e02849a267381aecb8aee577e8c35ff4064c6"
dependencies = [
"azure_core 0.21.0",
"bytes",
"futures",
"log",
"once_cell",
"serde",
"serde_json",
"time",
]
[[package]] [[package]]
name = "backtrace" name = "backtrace"
version = "0.3.74" version = "0.3.74"
@ -536,6 +693,12 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce" checksum = "349a06037c7bf932dd7e7d1f653678b2038b9ad46a74102f1fc7bd7872678cce"
[[package]]
name = "base64"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.21.7" version = "0.21.7"
@ -872,6 +1035,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4"
dependencies = [ dependencies = [
"powerfmt", "powerfmt",
"serde",
] ]
[[package]] [[package]]
@ -909,6 +1073,12 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813"
[[package]]
name = "dyn-clone"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005"
[[package]] [[package]]
name = "ecdsa" name = "ecdsa"
version = "0.14.8" version = "0.14.8"
@ -995,6 +1165,12 @@ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
[[package]]
name = "event-listener"
version = "2.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
[[package]] [[package]]
name = "event-listener" name = "event-listener"
version = "5.3.1" version = "5.3.1"
@ -1006,6 +1182,25 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "event-listener-strategy"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93"
dependencies = [
"event-listener 5.3.1",
"pin-project-lite",
]
[[package]]
name = "fastrand"
version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
dependencies = [
"instant",
]
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "2.3.0" version = "2.3.0"
@ -1022,6 +1217,16 @@ dependencies = [
"subtle", "subtle",
] ]
[[package]]
name = "flate2"
version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c"
dependencies = [
"crc32fast",
"miniz_oxide",
]
[[package]] [[package]]
name = "flume" name = "flume"
version = "0.11.0" version = "0.11.0"
@ -1134,6 +1339,21 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-lite"
version = "1.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49a9d51ce47660b1e808d3c990b4709f2f415d928835a17dfd16991515c46bce"
dependencies = [
"fastrand 1.9.0",
"futures-core",
"futures-io",
"memchr",
"parking",
"pin-project-lite",
"waker-fn",
]
[[package]] [[package]]
name = "futures-macro" name = "futures-macro"
version = "0.3.30" version = "0.3.30"
@ -1185,6 +1405,17 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "getrandom"
version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
"cfg-if",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]] [[package]]
name = "getrandom" name = "getrandom"
version = "0.2.15" version = "0.2.15"
@ -1404,6 +1635,26 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "http-types"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad"
dependencies = [
"anyhow",
"async-channel",
"base64 0.13.1",
"futures-lite",
"infer",
"pin-project-lite",
"rand 0.7.3",
"serde",
"serde_json",
"serde_qs",
"serde_urlencoded",
"url",
]
[[package]] [[package]]
name = "httparse" name = "httparse"
version = "1.10.0" version = "1.10.0"
@ -1676,6 +1927,21 @@ dependencies = [
"hashbrown 0.14.5", "hashbrown 0.14.5",
] ]
[[package]]
name = "infer"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac"
[[package]]
name = "instant"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222"
dependencies = [
"cfg-if",
]
[[package]] [[package]]
name = "ipnet" name = "ipnet"
version = "2.11.0" version = "2.11.0"
@ -2137,6 +2403,26 @@ version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
[[package]]
name = "pin-project"
version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
dependencies = [
"pin-project-internal",
]
[[package]]
name = "pin-project-internal"
version = "1.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]] [[package]]
name = "pin-project-lite" name = "pin-project-lite"
version = "0.2.14" version = "0.2.14"
@ -2229,6 +2515,16 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "quick-xml"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33"
dependencies = [
"memchr",
"serde",
]
[[package]] [[package]]
name = "quinn" name = "quinn"
version = "0.11.8" version = "0.11.8"
@ -2299,6 +2595,19 @@ version = "5.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
[[package]]
name = "rand"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom 0.1.16",
"libc",
"rand_chacha 0.2.2",
"rand_core 0.5.1",
"rand_hc",
]
[[package]] [[package]]
name = "rand" name = "rand"
version = "0.8.5" version = "0.8.5"
@ -2320,6 +2629,16 @@ dependencies = [
"rand_core 0.9.3", "rand_core 0.9.3",
] ]
[[package]]
name = "rand_chacha"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
dependencies = [
"ppv-lite86",
"rand_core 0.5.1",
]
[[package]] [[package]]
name = "rand_chacha" name = "rand_chacha"
version = "0.3.1" version = "0.3.1"
@ -2340,6 +2659,15 @@ dependencies = [
"rand_core 0.9.3", "rand_core 0.9.3",
] ]
[[package]]
name = "rand_core"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom 0.1.16",
]
[[package]] [[package]]
name = "rand_core" name = "rand_core"
version = "0.6.4" version = "0.6.4"
@ -2358,6 +2686,15 @@ dependencies = [
"getrandom 0.3.3", "getrandom 0.3.3",
] ]
[[package]]
name = "rand_hc"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
dependencies = [
"rand_core 0.5.1",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.4" version = "0.5.4"
@ -2423,10 +2760,12 @@ version = "0.12.20"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813" checksum = "eabf4c97d9130e2bf606614eb937e86edac8292eaa6f422f995d7e8de1eb1813"
dependencies = [ dependencies = [
"async-compression",
"base64 0.22.1", "base64 0.22.1",
"bytes", "bytes",
"encoding_rs", "encoding_rs",
"futures-core", "futures-core",
"futures-util",
"h2 0.4.10", "h2 0.4.10",
"http 1.2.0", "http 1.2.0",
"http-body 1.0.1", "http-body 1.0.1",
@ -2451,12 +2790,14 @@ dependencies = [
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-rustls 0.26.2", "tokio-rustls 0.26.2",
"tokio-util",
"tower", "tower",
"tower-http", "tower-http",
"tower-service", "tower-service",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"wasm-streams",
"web-sys", "web-sys",
"webpki-roots 1.0.1", "webpki-roots 1.0.1",
] ]
@ -2774,6 +3115,17 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "serde_qs"
version = "0.8.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7715380eec75f029a4ef7de39a9200e0a63823176b759d055b613f5a87df6a6"
dependencies = [
"percent-encoding",
"serde",
"thiserror 1.0.63",
]
[[package]] [[package]]
name = "serde_urlencoded" name = "serde_urlencoded"
version = "0.7.1" version = "0.7.1"
@ -2960,7 +3312,7 @@ dependencies = [
"crc", "crc",
"crossbeam-queue", "crossbeam-queue",
"either", "either",
"event-listener", "event-listener 5.3.1",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-intrusive", "futures-intrusive",
@ -3141,6 +3493,20 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "storage"
version = "0.1.0"
dependencies = [
"async-trait",
"aws-config",
"aws-sdk-s3",
"azure_core 0.25.0",
"azure_storage",
"azure_storage_blobs",
"thiserror 2.0.12",
"tokio",
]
[[package]] [[package]]
name = "stringprep" name = "stringprep"
version = "0.1.5" version = "0.1.5"
@ -3214,8 +3580,6 @@ dependencies = [
name = "task" name = "task"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"aws-config",
"aws-sdk-s3",
"chrono", "chrono",
"dotenvy", "dotenvy",
"futures", "futures",
@ -3226,6 +3590,7 @@ dependencies = [
"serde", "serde",
"serde_yml", "serde_yml",
"sqlx", "sqlx",
"storage",
"tokio", "tokio",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
@ -3238,7 +3603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64" checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"fastrand", "fastrand 2.3.0",
"once_cell", "once_cell",
"rustix", "rustix",
"windows-sys 0.59.0", "windows-sys 0.59.0",
@ -3301,6 +3666,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21"
dependencies = [ dependencies = [
"deranged", "deranged",
"itoa",
"js-sys",
"num-conv", "num-conv",
"powerfmt", "powerfmt",
"serde", "serde",
@ -3551,6 +3918,56 @@ version = "1.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
[[package]]
name = "typespec"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04c7a952f1f34257f945fc727b20defe7a3c01c05ddd42925977626cfa6e62ab"
dependencies = [
"base64 0.22.1",
"serde",
"serde_json",
"url",
]
[[package]]
name = "typespec_client_core"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5879ce67ba9e525fe088c882ede1337c32c3f80e83e72d9fd3cc6c8e05bcb3d7"
dependencies = [
"async-trait",
"base64 0.22.1",
"bytes",
"dyn-clone",
"futures",
"getrandom 0.2.15",
"pin-project",
"rand 0.8.5",
"reqwest",
"serde",
"serde_json",
"time",
"tokio",
"tracing",
"typespec",
"typespec_macros",
"url",
"uuid 1.15.1",
]
[[package]]
name = "typespec_macros"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cbccdbe531c8d553812a609bdb70c0d1002ad91333498e18df42c98744b15cc"
dependencies = [
"proc-macro2",
"quote",
"rustc_version",
"syn",
]
[[package]] [[package]]
name = "unicode-bidi" name = "unicode-bidi"
version = "0.3.15" version = "0.3.15"
@ -3605,6 +4022,7 @@ dependencies = [
"form_urlencoded", "form_urlencoded",
"idna", "idna",
"percent-encoding", "percent-encoding",
"serde",
] ]
[[package]] [[package]]
@ -3633,6 +4051,10 @@ name = "uuid"
version = "1.15.1" version = "1.15.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587" checksum = "e0f540e3240398cce6128b64ba83fdbdd86129c16a3aa1a3a252efd66eb3d587"
dependencies = [
"getrandom 0.3.3",
"serde",
]
[[package]] [[package]]
name = "valuable" name = "valuable"
@ -3658,6 +4080,12 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
[[package]]
name = "waker-fn"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "317211a0dc0ceedd78fb2ca9a44aed3d7b9b26f81870d485c07122b4350673b7"
[[package]] [[package]]
name = "want" name = "want"
version = "0.3.1" version = "0.3.1"
@ -3667,6 +4095,12 @@ dependencies = [
"try-lock", "try-lock",
] ]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.11.0+wasi-snapshot-preview1" version = "0.11.0+wasi-snapshot-preview1"
@ -3759,6 +4193,19 @@ dependencies = [
"unicode-ident", "unicode-ident",
] ]
[[package]]
name = "wasm-streams"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e072d4e72f700fb3443d8fe94a39315df013eef1104903cdb0a2abd322bbecd"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.77" version = "0.3.77"

3
backend/task/Cargo.toml
View File

@ -6,6 +6,7 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies] [dependencies]
storage = { version = "0.1.0", path = "../storage" }
tokio = { version = "1.19.2", features = ["full"] } tokio = { version = "1.19.2", features = ["full"] }
reqwest = { version = "0.12.20", features = ["json", "rustls-tls"] } reqwest = { version = "0.12.20", features = ["json", "rustls-tls"] }
job_scheduler = "1.2.1" job_scheduler = "1.2.1"
@ -21,7 +22,5 @@ futures = "0.3.30"
markdown = "1.0.0-alpha.20" markdown = "1.0.0-alpha.20"
serde = { version = "*", features = ["derive"] } serde = { version = "*", features = ["derive"] }
serde_yml = "*" serde_yml = "*"
aws-sdk-s3 = "1.94.0"
aws-config = "1.8"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }

1
backend/task/src/main.rs
View File

@ -29,6 +29,7 @@ pub struct TaskLog {
#[derive(Debug)] #[derive(Debug)]
pub enum TaskStatus { pub enum TaskStatus {
Pending(String), Pending(String),
Running(String),
Completed(String), Completed(String),
Failed(String), Failed(String),
} }

21
backend/task/src/tasks/upload_rss.rs
View File

@ -1,7 +1,10 @@
use crate::utils::{ use crate::utils::{
request::{Request, Response}, request::{Request, Response},
task_log, task_log,
{upload::S3ClientConfig, *}, };
use storage::services::{
aws::{S3Client, S3ClientConfig},
ObjectStorageClient,
}; };
pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) { pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) {
@ -23,14 +26,14 @@ async fn upload_rss(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), Box<dyn std
// upload the sucker to obj storage // upload the sucker to obj storage
if let Response::Xml(rss) = rss_result { if let Response::Xml(rss) = rss_result {
let client_config = S3ClientConfig::from_env().unwrap(); let client_config = S3ClientConfig::from_env().unwrap();
let s3_client = upload::create_s3_client(&client_config).await.unwrap(); let s3_client = S3Client::new(&client_config);
let _ = upload::upload( let _ = s3_client
&s3_client, .put_object(
client_config.bucket.as_str(), client_config.bucket.as_str(),
"feed.xml", "feed.xml",
rss.as_str(), rss.as_bytes().to_vec(),
) )
.await; .await;
println!("Finished uploading RSS feed"); println!("Finished uploading RSS feed");
} }

23
backend/task/src/tasks/upload_sitemap.rs
View File

@ -1,7 +1,10 @@
use crate::utils::{ use crate::utils::{
request::{Request, Response}, request::{Request, Response},
task_log, task_log,
{upload::S3ClientConfig, *}, };
use storage::services::{
aws::{S3Client, S3ClientConfig},
ObjectStorageClient,
}; };
pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) { pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) {
@ -14,7 +17,7 @@ pub fn register(pool: &sqlx::Pool<sqlx::Postgres>) {
async fn upload_sitemap( async fn upload_sitemap(
pool: &sqlx::Pool<sqlx::Postgres>, pool: &sqlx::Pool<sqlx::Postgres>,
) -> Result<(), Box<dyn std::error::Error>> { ) -> Result<(), Box<dyn std::error::Error>> {
// TODO:: get sitemap and upload it to bucket?? // start task logging
task_log::start(3, pool).await?; task_log::start(3, pool).await?;
// get request and request the things // get request and request the things
@ -25,14 +28,14 @@ async fn upload_sitemap(
// upload the sucker to obj storage // upload the sucker to obj storage
if let Response::Xml(sitemap) = sitemap_result { if let Response::Xml(sitemap) = sitemap_result {
let client_config = S3ClientConfig::from_env().unwrap(); let client_config = S3ClientConfig::from_env().unwrap();
let s3_client = upload::create_s3_client(&client_config).await.unwrap(); let s3_client = S3Client::new(&client_config);
let _ = upload::upload( let _ = s3_client
&s3_client, .put_object(
client_config.bucket.as_str(), client_config.bucket.as_str(),
"sitemap.xml", "sitemap.xml",
sitemap.as_str(), sitemap.as_bytes().to_vec(),
) )
.await; .await;
println!("Finished uploading sitemap!"); println!("Finished uploading sitemap!");
} }

73
backend/task/src/utils/upload.rs
View File

@ -1,73 +0,0 @@
use aws_config::{BehaviorVersion, Region};
use aws_sdk_s3::{config::Credentials, Client, Config};
use std::env;
#[derive(Debug)]
pub struct S3ClientConfig {
pub access_key: String,
secret_key: String,
endpoint: String,
pub bucket: String,
region: String,
}
impl S3ClientConfig {
pub fn from_env() -> Result<Self, Box<dyn std::error::Error>> {
Ok(S3ClientConfig {
access_key: env::var("LINODE_ACCESS_KEY")
.map_err(|_| "LINODE_ACCESS_KEY environment variable not set")?,
secret_key: env::var("LINODE_SECRET_KEY")
.map_err(|_| "LINODE_SECRET_KEY environment variable not set")?,
endpoint: env::var("LINODE_ENDPOINT")
.unwrap_or_else(|_| "us-ord-1.linodeobjects.com".to_string()),
bucket: env::var("LINODE_BUCKET")
.map_err(|_| "LINODE_BUCKET environment variable not set")?,
region: env::var("LINODE_REGION").unwrap_or_else(|_| "us-ord".to_string()),
})
}
}
pub async fn create_s3_client(
config: &S3ClientConfig,
) -> Result<Client, Box<dyn std::error::Error>> {
let credentials = Credentials::new(
&config.access_key,
&config.secret_key,
None,
None,
"linode-object-storage",
);
let s3_config = Config::builder()
.behavior_version(BehaviorVersion::latest())
.region(Region::new(config.region.clone()))
.endpoint_url(format!("https://{}", config.endpoint))
.credentials_provider(credentials)
.force_path_style(false)
.build();
Ok(Client::from_conf(s3_config))
}
pub async fn upload(
client: &Client,
bucket: &str,
key: &str,
content: &str,
) -> Result<(), Box<dyn std::error::Error>> {
println!("Uploading to Linode Object Storage...");
println!("Bucket: {}", bucket);
let put_object_req = client
.put_object()
.bucket(bucket)
.key(key)
.body(content.as_bytes().to_vec().into())
.acl(aws_sdk_s3::types::ObjectCannedAcl::PublicRead)
.content_type("application/xml")
.send()
.await?;
println!("Upload successful! ETag: {:?}", put_object_req.e_tag());
Ok(())
}