diff --git a/.env.example b/.env.example index bcea61db5..61cfcde65 100644 --- a/.env.example +++ b/.env.example @@ -23,3 +23,5 @@ ARK_COLLECTIONS_TABLE_NAME= ARK_TOKENS_TABLE_NAME= ARK_BLOCKS_TABLE_NAME= ARK_TOKENS_OWNERS_TABLE_NAME= + +IPFS_GATEWAY_URI= \ No newline at end of file diff --git a/.gitignore b/.gitignore index 053de4f6a..6e26162c5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ /target + +crates/ark-metadata/**/tmp/ +crates/ark-metadata/images/**/* + .env -.aws \ No newline at end of file +.aws diff --git a/.vscode/settings.json b/.vscode/settings.json index 08a446a71..0ee06669b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,5 +1,3 @@ { - "rust-analyzer.linkedProjects": [ - "./Cargo.toml", - ] + "rust-analyzer.linkedProjects": ["./Cargo.toml"] } diff --git a/Cargo.lock b/Cargo.lock index c3886c8e2..7cf21275e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,12 +122,22 @@ dependencies = [ name = "ark-metadata" version = "0.1.0" dependencies = [ - "aws-sdk-dynamodb", + "anyhow", + "ark-starknet", + "async-trait", + "aws-config", + "aws-sdk-s3", + "base64 0.21.4", "dotenv", "log", + "mockall", "reqwest", "serde", + "serde_derive", "serde_json", + "starknet", + "tokio", + "urlencoding", ] [[package]] @@ -139,6 +149,9 @@ dependencies = [ "ark-metadata", "ark-starknet", "ark-storage", + "async-trait", + "mockall", + "starknet", "tokio", ] @@ -159,6 +172,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "mockall", + "num-bigint", "regex", "starknet", "url", @@ -179,8 +194,10 @@ name = "ark-storage" version = "0.1.0" dependencies = [ "log", + "mockall", "num-bigint", "serde", + "serde_json", "starknet", ] @@ -198,7 +215,7 @@ checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -219,6 +236,36 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "aws-config" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc6b3804dca60326e07205179847f17a4fce45af3a1106939177ad41ac08a6de" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-sdk-sso", + "aws-sdk-sts", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-http-tower", + "aws-smithy-json", + "aws-smithy-types", + "aws-types", + "bytes", + "fastrand", + "hex", + "http", + "hyper", + "ring", + "time", + "tokio", + "tower", + "tracing", + "zeroize", +] + [[package]] name = "aws-credential-types" version = "0.56.1" @@ -262,6 +309,7 @@ dependencies = [ "aws-http", "aws-sigv4", "aws-smithy-async", + "aws-smithy-eventstream", "aws-smithy-http", "aws-smithy-runtime-api", "aws-smithy-types", @@ -274,10 +322,42 @@ dependencies = [ ] [[package]] -name = "aws-sdk-dynamodb" -version = "0.29.0" +name = "aws-sdk-s3" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3b4df8750310555fa980f020f152e91013cf83d01036dab992cb64286e11431" +checksum = "a531d010f9f556bf65eb3bcd8d24f1937600ab6940fede4d454cd9b1f031fb34" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-sigv4", + "aws-smithy-async", + "aws-smithy-checksums", + "aws-smithy-client", + "aws-smithy-eventstream", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "bytes", + "http", + "http-body", + "once_cell", + "percent-encoding", + "regex", + "tokio-stream", + "tracing", + "url", +] + +[[package]] +name = "aws-sdk-sso" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "903f888ff190e64f6f5c83fb0f8d54f9c20481f1dc26359bb8896f5d99908949" dependencies = [ "aws-credential-types", "aws-http", @@ -291,20 +371,45 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand", "http", "regex", "tokio-stream", "tracing", ] +[[package]] +name = "aws-sdk-sts" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a47ad6bf01afc00423d781d464220bf69fb6a674ad6629cbbcb06d88cdc2be82" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-client", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "http", + "regex", + "tracing", +] + [[package]] name = "aws-sigv4" version = "0.56.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7b28f4910bb956b7ab320b62e98096402354eca976c587d1eeccd523d9bac03" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-http", + "bytes", "form_urlencoded", "hex", "hmac", @@ -329,6 +434,27 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "aws-smithy-checksums" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afb15946af1b8d3beeff53ad991d9bff68ac22426b6d40372b958a75fa61eaed" +dependencies = [ + "aws-smithy-http", + "aws-smithy-types", + "bytes", + "crc32c", + "crc32fast", + "hex", + "http", + "http-body", + "md-5", + "pin-project-lite", + "sha1", + "sha2", + "tracing", +] + [[package]] name = "aws-smithy-client" version = "0.56.1" @@ -353,12 +479,24 @@ dependencies = [ "tracing", ] +[[package]] +name = "aws-smithy-eventstream" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "850233feab37b591b7377fd52063aa37af615687f5896807abe7f49bd4e1d25b" +dependencies = [ + "aws-smithy-types", + "bytes", + "crc32fast", +] + [[package]] name = "aws-smithy-http" version = "0.56.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54cdcf365d8eee60686885f750a34c190e513677db58bbc466c44c588abf4199" dependencies = [ + "aws-smithy-eventstream", "aws-smithy-types", "bytes", "bytes-utils", @@ -400,6 +538,16 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-query" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28556a3902091c1f768a34f6c998028921bdab8d47d92586f363f14a4a32d047" +dependencies = [ + "aws-smithy-types", + "urlencoding", +] + [[package]] name = "aws-smithy-runtime" version = "0.56.1" @@ -451,6 +599,15 @@ dependencies = [ "time", ] +[[package]] +name = "aws-smithy-xml" +version = "0.56.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01d2dedcdd8023043716cfeeb3c6c59f2d447fce365d8e194838891794b23b6" +dependencies = [ + "xmlparser", +] + [[package]] name = "aws-types" version = "0.56.1" @@ -551,9 +708,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.13.0" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3e2c3daef883ecc1b5d58c15adae93470a91d425f3532ba1695849656af3fc1" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" [[package]] name = "byte-slice-cast" @@ -646,6 +803,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.3.2" @@ -712,7 +878,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -723,7 +889,7 @@ checksum = "836a9bbc7ad63342d6d6e7b815ccab164bc77a2d95d84bc3117a8c0d5c98e2d5" dependencies = [ "darling_core", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -746,6 +912,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.10.7" @@ -763,6 +935,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "either" version = "1.9.0" @@ -882,6 +1060,15 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" @@ -912,6 +1099,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "funty" version = "2.0.0" @@ -941,7 +1134,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -1292,9 +1485,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.147" +version = "0.2.148" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +checksum = "9cdc71e17332e86d2e1d38c1f99edcb6288ee11b815fb1a4b049eaa2114d369b" [[package]] name = "linux-raw-sys" @@ -1327,6 +1520,15 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +dependencies = [ + "digest", +] + [[package]] name = "memchr" version = "2.6.3" @@ -1359,6 +1561,33 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -1377,6 +1606,12 @@ dependencies = [ "tempfile", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -1465,7 +1700,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -1595,7 +1830,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -1622,6 +1857,36 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "primitive-types" version = "0.12.1" @@ -1671,9 +1936,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.66" +version = "1.0.67" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328" dependencies = [ "unicode-ident", ] @@ -2028,14 +2293,14 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] name = "serde_json" -version = "1.0.106" +version = "1.0.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cc66a619ed80bf7a0f6b17dd063a84b88f6dea1813737cf469aef1d081142c2" +checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" dependencies = [ "itoa", "ryu", @@ -2090,7 +2355,18 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", +] + +[[package]] +name = "sha1" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", ] [[package]] @@ -2281,7 +2557,7 @@ checksum = "af6527b845423542c8a16e060ea1bc43f67229848e7cd4c4d80be994a84220ce" dependencies = [ "starknet-curve", "starknet-ff", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2315,7 +2591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef846b6bb48fc8c3e9a2aa9b5b037414f04a908d9db56493a3ae69a857eb2506" dependencies = [ "starknet-core 0.6.0", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2385,9 +2661,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.32" +version = "2.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "239814284fd6f1a4ffe4ca893952cdd93c224b6a1571c9a9eadd670295c0c9e2" +checksum = "9caece70c63bfba29ec2fed841a09851b14a235c60010fa4de58089b6c025668" dependencies = [ "proc-macro2", "quote", @@ -2413,6 +2689,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "thiserror" version = "1.0.48" @@ -2430,7 +2712,7 @@ checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2522,7 +2804,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2636,7 +2918,7 @@ checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] [[package]] @@ -2710,9 +2992,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -2740,6 +3022,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + [[package]] name = "uuid" version = "0.8.2" @@ -2816,7 +3104,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", "wasm-bindgen-shared", ] @@ -2850,7 +3138,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3002,6 +3290,12 @@ dependencies = [ "tap", ] +[[package]] +name = "xmlparser" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d25c75bf9ea12c4040a97f829154768bbbce366287e2dc044af160cd79a13fd" + [[package]] name = "zeroize" version = "1.6.0" @@ -3019,5 +3313,5 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.32", + "syn 2.0.33", ] diff --git a/Cargo.toml b/Cargo.toml index f541397ae..61f85331a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,4 +38,8 @@ ark-starknet.workspace = true ark-metadata.workspace = true ark-storage.workspace = true ark-indexer.workspace = true +starknet.workspace = true +async-trait.workspace = true +[dev-dependencies] +mockall = "0.11.4" \ No newline at end of file diff --git a/README.md b/README.md index 0739e2027..5e2e3e035 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ to then identify the contract and tokens associated to the event. The `indexer` crate provide a `main_loop` with this logic, for an efficient indexation per blocks. -### - Metadata +### - [Metadata](/crates/ark-metadata/README.md) Even if the metadata are not at the core of the indexing process, they are vital for any NFT ecosystem. diff --git a/crates/ark-metadata/Cargo.toml b/crates/ark-metadata/Cargo.toml index fc6b33aa3..ddd55daa7 100644 --- a/crates/ark-metadata/Cargo.toml +++ b/crates/ark-metadata/Cargo.toml @@ -5,8 +5,21 @@ edition = "2021" [dependencies] reqwest = { version = "0.11", features = ["json", "native-tls-vendored"] } -serde_json = "1.0" dotenv = "0.15.0" -serde = "1.0.164" -aws-sdk-dynamodb = "0.29.0" log = "0.4" +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +anyhow = "1.0" +starknet = "0.5.0" +tokio = { version = "1", features = ["full"] } +ark-starknet.workspace = true +urlencoding = "2.1.2" +base64 = "0.21.0" +aws-config = "0.56.1" +aws-sdk-s3 = "0.30.0" +async-trait.workspace = true + +[dev-dependencies] +ark-starknet = { path = "../ark-starknet", features = ["mock"] } +mockall = "0.11.4" diff --git a/crates/ark-metadata/README.md b/crates/ark-metadata/README.md new file mode 100644 index 000000000..45afa847d --- /dev/null +++ b/crates/ark-metadata/README.md @@ -0,0 +1,34 @@ +# ark_metadata 🖼️ + +## Overview + +`ark_metadata` is a library designed to fetch all NFT metadata on Starknet. It allows for efficient refreshing of specific token metadata or capturing metadata for an entire collection in a streamlined manner. + +## Features +### MetadataManager + +- `refresh_token_metadata()`: Refresh metadata for a specific token, and caches images if available. +- `refresh_collection_token_metadata()`: Refresh metadata for all tokens in a collection. + +## Getting Started + +To integrate `ark_metadata`, include ```ark-rs``` in your `Cargo.toml`. +*Check out the provided example to see its usage in various scenarios.* + +To instantiate a new `MetadataManager`, you'll need some implementations: + +- **Storage**: Implements the data access layer. +- **StarknetClient**: Facilitates interactions with Starknet and contract calls. +- **FileManager**: Handles file storage. Within `ark_metadata`, two defaults are available: + - **LocalFileManager**: For local file storage. + - **AWSFileManager**: For AWS S3 cloud storage. + +## Dependencies + +- `ReqwestClient`: Used for making HTTP requests, crucial for fetching metadata from URIs. + +## Testing + +Run tests with: +```bash +cargo test --workspace diff --git a/crates/ark-metadata/src/cairo_string_parser.rs b/crates/ark-metadata/src/cairo_string_parser.rs new file mode 100644 index 000000000..3b31ffdca --- /dev/null +++ b/crates/ark-metadata/src/cairo_string_parser.rs @@ -0,0 +1,237 @@ +use anyhow::Result; +use starknet::core::{types::FieldElement, utils::parse_cairo_short_string}; + +#[derive(Debug)] +pub enum ParseError { + NoValueFound, + ShortStringError, +} + +/// Parse a Cairo "long string" represented as a Vec of FieldElements into a Rust String. +/// +/// # Arguments +/// * `field_elements`: A vector of FieldElements representing the Cairo long string. +/// +/// # Returns +/// * A `Result` which is either the parsed Rust string or an error. +pub fn parse_cairo_long_string(field_elements: Vec) -> Result { + match field_elements.len() { + 0 => Err(ParseError::NoValueFound), + // If the long_string contains only one FieldElement, try to parse it using the short string parser. + 1 => match field_elements.first() { + Some(first_string_field_element) => { + match parse_cairo_short_string(first_string_field_element) { + Ok(value) => Ok(value), + Err(_) => Err(ParseError::ShortStringError), + } + } + None => Err(ParseError::NoValueFound), + }, + // If the long_string has more than one FieldElement, parse each FieldElement sequentially + // and concatenate their results. + _ => { + let mut result = String::new(); + for field_element in &field_elements[1..] { + match parse_cairo_short_string(field_element) { + Ok(value) => { + result.push_str(&value); + } + Err(_) => { + return Err(ParseError::ShortStringError); + } + } + } + Ok(result) + } + } +} + +#[cfg(test)] +mod tests { + use crate::cairo_string_parser::ParseError; + + use super::parse_cairo_long_string; + use starknet::core::types::FieldElement; + + #[test] + fn should_handle_single_field_element() { + let long_string = vec![FieldElement::from_hex_be("0x68").unwrap()]; + let result = parse_cairo_long_string(long_string); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), "h"); + } + + #[test] + fn should_return_error_for_empty_vector() { + let long_string = vec![]; + + let result = parse_cairo_long_string(long_string); + + // First, check that the result is an error. + assert!(result.is_err()); + + // Then, check that it's the specific error you expect. + match result { + Err(ParseError::NoValueFound) => {} // expected error, do nothing + Err(_) => panic!("Unexpected error type returned"), + Ok(_) => panic!("Expected an error but got a success result"), + } + } + #[test] + fn should_parse_field_elements_with_array_length() { + let long_string = vec![ + FieldElement::from_hex_be("0x4").unwrap(), + FieldElement::from_hex_be( + "0x68747470733a2f2f6170692e627269712e636f6e737472756374696f6e", + ) + .unwrap(), + FieldElement::from_hex_be("0x2f76312f7572692f7365742f").unwrap(), + FieldElement::from_hex_be("0x737461726b6e65742d6d61696e6e65742f").unwrap(), + FieldElement::from_hex_be("0x2e6a736f6e").unwrap(), + ]; + + let result = parse_cairo_long_string(long_string); + assert!(result.is_ok()); + + let value = result.unwrap(); + println!("Value: {}", value); + assert!(value == "https://api.briq.construction/v1/uri/set/starknet-mainnet/.json"); + } + + #[test] + fn should_parse_one_field_element_per_character_to_url() { + let long_string = vec![ + FieldElement::from_hex_be("0x2d").unwrap(), + FieldElement::from_hex_be("0x68").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x3a").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x61").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + FieldElement::from_hex_be("0x69").unwrap(), + FieldElement::from_hex_be("0x2e").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x61").unwrap(), + FieldElement::from_hex_be("0x72").unwrap(), + FieldElement::from_hex_be("0x6b").unwrap(), + FieldElement::from_hex_be("0x6e").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x2e").unwrap(), + FieldElement::from_hex_be("0x71").unwrap(), + FieldElement::from_hex_be("0x75").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x71").unwrap(), + FieldElement::from_hex_be("0x75").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x75").unwrap(), + FieldElement::from_hex_be("0x72").unwrap(), + FieldElement::from_hex_be("0x69").unwrap(), + FieldElement::from_hex_be("0x3f").unwrap(), + FieldElement::from_hex_be("0x6c").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x76").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x6c").unwrap(), + FieldElement::from_hex_be("0x3d").unwrap(), + FieldElement::from_hex_be("0x30").unwrap(), + ]; + + let result = parse_cairo_long_string(long_string); + assert!(result.is_ok()); + assert!(result.unwrap() == "https://api.starknet.quest/quests/uri?level=0"); + } + + #[test] + fn should_parse_field_elements_to_url_with_array_length() { + let long_string = vec![ + FieldElement::from_hex_be("0x44").unwrap(), + FieldElement::from_hex_be("0x69").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + FieldElement::from_hex_be("0x66").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x3a").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x62").unwrap(), + FieldElement::from_hex_be("0x61").unwrap(), + FieldElement::from_hex_be("0x66").unwrap(), + FieldElement::from_hex_be("0x79").unwrap(), + FieldElement::from_hex_be("0x62").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x69").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x6f").unwrap(), + FieldElement::from_hex_be("0x63").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x7a").unwrap(), + FieldElement::from_hex_be("0x35").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x67").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + FieldElement::from_hex_be("0x37").unwrap(), + FieldElement::from_hex_be("0x7a").unwrap(), + FieldElement::from_hex_be("0x72").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x37").unwrap(), + FieldElement::from_hex_be("0x66").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x72").unwrap(), + FieldElement::from_hex_be("0x64").unwrap(), + FieldElement::from_hex_be("0x6e").unwrap(), + FieldElement::from_hex_be("0x65").unwrap(), + FieldElement::from_hex_be("0x79").unwrap(), + FieldElement::from_hex_be("0x6a").unwrap(), + FieldElement::from_hex_be("0x34").unwrap(), + FieldElement::from_hex_be("0x6b").unwrap(), + FieldElement::from_hex_be("0x7a").unwrap(), + FieldElement::from_hex_be("0x71").unwrap(), + FieldElement::from_hex_be("0x32").unwrap(), + FieldElement::from_hex_be("0x76").unwrap(), + FieldElement::from_hex_be("0x34").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x35").unwrap(), + FieldElement::from_hex_be("0x67").unwrap(), + FieldElement::from_hex_be("0x33").unwrap(), + FieldElement::from_hex_be("0x61").unwrap(), + FieldElement::from_hex_be("0x73").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x34").unwrap(), + FieldElement::from_hex_be("0x35").unwrap(), + FieldElement::from_hex_be("0x6d").unwrap(), + FieldElement::from_hex_be("0x36").unwrap(), + FieldElement::from_hex_be("0x35").unwrap(), + FieldElement::from_hex_be("0x63").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x37").unwrap(), + FieldElement::from_hex_be("0x72").unwrap(), + FieldElement::from_hex_be("0x67").unwrap(), + FieldElement::from_hex_be("0x78").unwrap(), + FieldElement::from_hex_be("0x75").unwrap(), + FieldElement::from_hex_be("0x2f").unwrap(), + FieldElement::from_hex_be("0x30").unwrap(), + ]; + + let result = parse_cairo_long_string(long_string); + assert!(result.is_ok()); + + let value = result.unwrap(); + assert!(value == "ipfs://bafybeieocsz5txpxgp7zrx7fexrdneyj4kzq2v4x5g3asx45m65cx7rgxu/0"); + } +} diff --git a/crates/ark-metadata/src/file_manager.rs b/crates/ark-metadata/src/file_manager.rs new file mode 100644 index 000000000..a604b9196 --- /dev/null +++ b/crates/ark-metadata/src/file_manager.rs @@ -0,0 +1,155 @@ +/// Provides file management capabilities. +/// +/// This module offers functionality to save files both locally and to AWS S3. +use std::fs::{create_dir_all, File}; +use std::io::prelude::*; +use std::path::Path; + +use anyhow::{Context, Ok, Result}; +use async_trait::async_trait; +use aws_sdk_s3::primitives::ByteStream; +use log::{debug, info}; + +#[cfg(any(test, feature = "mock"))] +use mockall::automock; + +/// Represents information about a file. +/// +/// This struct contains the name, content, and directory path (if any) of a file. +pub struct FileInfo { + pub name: String, + pub content: Vec, + pub dir_path: Option, +} + +/// A trait that defines file management operations. +/// +/// Implementors of this trait provide functionality to save files. +#[cfg_attr(any(test, feature = "mock"), automock)] +#[async_trait] +pub trait FileManager { + /// Save the provided file. + /// + /// Implementors will provide the logic to save `file` and will return a `Result`. + async fn save(&self, file: &FileInfo) -> Result<()>; +} + +/// FileManager implementation that saves files locally. +#[derive(Default)] +pub struct LocalFileManager; + +#[async_trait] +impl FileManager for LocalFileManager { + async fn save(&self, file: &FileInfo) -> Result<()> { + let dir_path = file.dir_path.clone().unwrap_or(String::from("./tmp")); + + // Construct the path + let path = Path::new("images").join(dir_path.as_str()).join(&file.name); + + // Ensure directory exists + create_dir_all(path.parent().unwrap()).context("Failed to create directory")?; + + // Create and write to the file + let mut dest_file = File::create(&path).context("Failed to create file")?; + + dest_file + .write_all(&file.content) + .context("Failed to write to file")?; + + info!("File saved: {}", file.name); + Ok(()) + } +} + +/// FileManager implementation that saves files to AWS S3. +/// +/// This implementation requires a bucket name for storing files in AWS S3. +#[derive(Default)] +pub struct AWSFileManager { + bucket_name: String, +} + +impl AWSFileManager { + /// Create a new AWSFileManager with the specified bucket name. + pub fn new(bucket_name: String) -> Self { + Self { bucket_name } + } +} + +#[async_trait] +impl FileManager for AWSFileManager { + async fn save(&self, file: &FileInfo) -> Result<()> { + debug!("Uploading {} to AWS...", file.name); + + let config = aws_config::load_from_env().await; + let client = aws_sdk_s3::Client::new(&config); + let body = ByteStream::from(file.content.clone()); + + let key = match &file.dir_path { + Some(dir_path) => format!("{}/{}", dir_path, &file.name), + None => file.name.clone(), + }; + + let _ = client + .put_object() + .bucket(&self.bucket_name) + .key(key) + .body(body) + .send() + .await?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[tokio::test] + async fn test_local_file_save() { + // Prepare a dummy file + let file_info = FileInfo { + name: "test_file.txt".to_string(), + content: b"Hello, world!".to_vec(), + dir_path: Some("some_subdir".to_string()), + }; + + // Use the LocalFileManager to save the file + let manager = LocalFileManager; + let result = manager.save(&file_info).await; + assert!(result.is_ok()); + + // Verify that the file has been saved correctly + let content = fs::read("./images/some_subdir/test_file.txt").unwrap(); + assert_eq!(content, b"Hello, world!"); + + // Clean up + fs::remove_file("./images/some_subdir/test_file.txt").unwrap(); + fs::remove_dir("./images/some_subdir").unwrap(); + } + + #[tokio::test] + async fn test_local_file_save_without_subdir() { + // Prepare a dummy file without subdir + let file_info = FileInfo { + name: "test_file.txt".to_string(), + content: b"Hello, world!".to_vec(), + dir_path: None, + }; + + // Use the LocalFileManager to save the file + let manager = LocalFileManager; + let result = manager.save(&file_info).await; + assert!(result.is_ok()); + + // Verify that the file has been saved correctly + let content = fs::read("./images/tmp/test_file.txt").unwrap(); + assert_eq!(content, b"Hello, world!"); + + // Clean up + fs::remove_file("./images/tmp/test_file.txt").unwrap(); + fs::remove_dir("./images/tmp").unwrap(); + } +} diff --git a/crates/ark-metadata/src/get.rs b/crates/ark-metadata/src/get.rs deleted file mode 100644 index b674f4c4c..000000000 --- a/crates/ark-metadata/src/get.rs +++ /dev/null @@ -1,145 +0,0 @@ -use aws_sdk_dynamodb::types::AttributeValue; -use log::{error, info, warn}; -use reqwest::Client as ReqwestClient; -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use std::collections::HashMap; -use std::error::Error; -use std::time::Duration; - -#[derive(Debug, Serialize, Deserialize, Clone)] -struct MetadataAttribute { - trait_type: String, - value: String, - display_type: String, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct NormalizedMetadata { - pub description: String, - pub external_url: String, - pub image: String, - pub name: String, - attributes: Vec, -} - -impl From for HashMap { - fn from(metadata: NormalizedMetadata) -> Self { - let mut attributes: HashMap = HashMap::new(); - - attributes.insert( - "description".to_string(), - AttributeValue::S(metadata.description), - ); - attributes.insert( - "external_url".to_string(), - AttributeValue::S(metadata.external_url), - ); - attributes.insert("image".to_string(), AttributeValue::S(metadata.image)); - attributes.insert("name".to_string(), AttributeValue::S(metadata.name)); - - let attributes_list: Vec = metadata - .attributes - .into_iter() - .map(|attribute| { - let mut attribute_map: HashMap = HashMap::new(); - attribute_map.insert( - "trait_type".to_string(), - AttributeValue::S(attribute.trait_type), - ); - attribute_map.insert("value".to_string(), AttributeValue::S(attribute.value)); - attribute_map.insert( - "display_type".to_string(), - AttributeValue::S(attribute.display_type), - ); - AttributeValue::M(attribute_map) - }) - .collect(); - - attributes.insert("attributes".to_string(), AttributeValue::L(attributes_list)); - - attributes - } -} - -pub async fn get_metadata( - client: &ReqwestClient, - metadata_uri: &str, - initial_metadata_uri: &str, -) -> Result<(Value, NormalizedMetadata), Box> { - info!("Fetching metadata: {}", metadata_uri); - - let response = client - .get(metadata_uri) - .timeout(Duration::from_secs(10)) - .send() - .await; - - match response { - Ok(resp) => { - let raw_metadata: Value = resp.json().await?; - - info!("Metadata: {:?}", raw_metadata); - - let empty_vec = Vec::new(); - - let attributes = raw_metadata - .get("attributes") - .and_then(|attr| attr.as_array()) - .unwrap_or(&empty_vec); - - let normalized_attributes: Vec = attributes - .iter() - .map(|attribute| MetadataAttribute { - trait_type: attribute - .get("trait_type") - .and_then(|trait_type| trait_type.as_str()) - .unwrap_or("") - .to_string(), - value: attribute - .get("value") - .and_then(|value| value.as_str()) - .unwrap_or("") - .to_string(), - display_type: attribute - .get("display_type") - .and_then(|display_type| display_type.as_str()) - .unwrap_or("") - .to_string(), - }) - .collect(); - - let normalized_metadata = NormalizedMetadata { - description: raw_metadata - .get("description") - .and_then(|desc| desc.as_str()) - .unwrap_or("") - .to_string(), - external_url: initial_metadata_uri.to_string(), - image: raw_metadata - .get("image") - .and_then(|img| img.as_str()) - .unwrap_or("") - .to_string(), - name: raw_metadata - .get("name") - .and_then(|name| name.as_str()) - .unwrap_or("") - .to_string(), - attributes: normalized_attributes, - }; - - Ok((raw_metadata, normalized_metadata)) - } - Err(e) => { - // Gérer l'erreur, y compris les timeouts - if e.is_timeout() { - warn!("Metadata request timeout: {:?}", e); - } else { - error!("Metadata request error : {:?}", e); - } - - Err(e.into()) - } - } -} diff --git a/crates/ark-metadata/src/lib.rs b/crates/ark-metadata/src/lib.rs index 125ca70d5..4883c6e90 100644 --- a/crates/ark-metadata/src/lib.rs +++ b/crates/ark-metadata/src/lib.rs @@ -1 +1,6 @@ -pub mod get; +mod cairo_string_parser; +pub mod file_manager; +pub mod metadata_manager; +pub mod storage; +pub mod types; +mod utils; diff --git a/crates/ark-metadata/src/metadata_manager.rs b/crates/ark-metadata/src/metadata_manager.rs new file mode 100644 index 000000000..0c7c478b5 --- /dev/null +++ b/crates/ark-metadata/src/metadata_manager.rs @@ -0,0 +1,423 @@ +use crate::{ + cairo_string_parser::parse_cairo_long_string, + file_manager::{FileInfo, FileManager}, + storage::Storage, + utils::{extract_metadata_from_headers, get_token_metadata}, +}; +use anyhow::{anyhow, Result}; +use ark_starknet::{client::StarknetClient, CairoU256}; +use log::{debug, error}; +use reqwest::Client as ReqwestClient; +use starknet::core::types::{BlockId, BlockTag, FieldElement}; +use starknet::macros::selector; + +/// `MetadataManager` is responsible for managing metadata information related to tokens. +/// It works with the underlying storage and Starknet client to fetch and update token metadata. +pub struct MetadataManager<'a, T: Storage, C: StarknetClient, F: FileManager> { + storage: &'a T, + starknet_client: &'a C, + request_client: ReqwestClient, + file_manager: &'a F, +} + +pub struct MetadataImage { + pub file_type: String, + pub content_length: u64, + pub is_cache_updated: bool, +} + +#[derive(Copy, Clone)] +pub enum CacheOption { + Cache, + NoCache, + Default, +} + +/// Represents possible errors that can arise while working with metadata in the manager. +#[derive(Debug)] +pub enum MetadataError { + DatabaseError, + ParsingError, + RequestTokenUriError, + RequestImageError, + EnvVarMissingError, +} + +impl<'a, T: Storage, C: StarknetClient, F: FileManager> MetadataManager<'a, T, C, F> { + /// Creates a new instance of `MetadataManager` with the given storage, Starknet client, and a new request client. + pub fn new(storage: &'a T, starknet_client: &'a C, file_manager: &'a F) -> Self { + MetadataManager { + storage, + starknet_client, + request_client: ReqwestClient::new(), + file_manager, + } + } + + /// Refreshes the metadata for a specific token within a given collection. + /// + /// This function retrieves the URI for the token, fetches its metadata, and updates the stored + /// metadata in the database. If the metadata includes an image URI, this function also handles + /// the fetching and optional caching of the image. + /// + /// # Parameters + /// - `contract_address`: The address of the contract representing the token collection. + /// - `token_id`: The ID of the token whose metadata needs to be refreshed. + /// - `cache`: Specifies whether the token's image should be cached. + /// + /// # Returns + /// - A `Result` indicating the success or failure of the metadata refresh operation. + pub async fn refresh_token_metadata( + &mut self, + contract_address: FieldElement, + token_id: CairoU256, + cache: CacheOption, + ipfs_gateway_uri: &str, + ) -> Result<(), MetadataError> { + let token_uri = self + .get_token_uri(&token_id, contract_address) + .await + .map_err(|_| MetadataError::ParsingError)?; + + let token_metadata = get_token_metadata(&self.request_client, token_uri.as_str()) + .await + .map_err(|_| MetadataError::RequestTokenUriError)?; + + if token_metadata.image.is_some() { + let ipfs_url = ipfs_gateway_uri.to_string(); + let url = token_metadata + .image + .as_ref() + .map(|s| s.replace("ipfs://", &ipfs_url)) + .unwrap_or_default(); + + let image_name = url.rsplit('/').next().unwrap_or_default(); + let image_ext = image_name.rsplit('.').next().unwrap_or_default(); + + self.fetch_token_image(url.as_str(), image_ext, cache, contract_address, &token_id) + .await + .map_err(|_| MetadataError::RequestImageError)?; + } + + self.storage + .register_token_metadata(&contract_address, token_id, token_metadata) + .map_err(|_e| MetadataError::DatabaseError)?; + + Ok(()) + } + + /// Refreshes the metadata for all tokens in a given collection. + /// + /// This function retrieves a list of token IDs within a collection that + /// lack metadata and then individually refreshes the metadata for each token. + /// + /// # Parameters + /// - `contract_address`: The address of the contract representing the token collection. + /// - `cache`: Specifies whether the token's image should be cached. + /// + /// # Returns + /// - A `Result` indicating the success or failure of the metadata refresh operation. + pub async fn refresh_collection_token_metadata( + &mut self, + contract_address: FieldElement, + cache: CacheOption, + ipfs_gateway_uri: &str, + ) -> Result<(), MetadataError> { + let token_ids = self + .storage + .find_token_ids_without_metadata_in_collection(contract_address) + .map_err(|_| MetadataError::DatabaseError)?; + + for token_id in token_ids { + self.refresh_token_metadata(contract_address, token_id, cache, ipfs_gateway_uri) + .await?; + } + + Ok(()) + } + + /// Fetches the image for a given token and optionally caches it. + /// + /// Depending on the provided `CacheOption`, this function might directly fetch + /// the image's metadata without actually fetching the image, or it might fetch and cache the image. + /// + /// # Parameters + /// - `url`: The URL from which the token image can be fetched. + /// - `file_ext`: The file extension of the token image (e.g., "jpg", "png"). + /// - `cache`: Specifies whether the token's image should be cached. + /// - `contract_address`: The address of the contract representing the token collection. + /// - `token_id`: The ID of the token whose image is to be fetched. + /// + /// # Returns + /// - A `Result` containing `MetadataImage` which provides details about the fetched image, + /// or an error if the image fetch operation fails. + pub async fn fetch_token_image( + &mut self, + url: &str, + file_ext: &str, + cache: CacheOption, + contract_address: FieldElement, + token_id: &CairoU256, + ) -> Result { + match cache { + CacheOption::NoCache => { + let response = self.request_client.head(url).send().await?; + let (content_type, content_length) = + extract_metadata_from_headers(response.headers())?; + + Ok(MetadataImage { + file_type: content_type, + content_length, + is_cache_updated: false, + }) + } + _ => { + debug!("Fetching image... {}", url); + let response = reqwest::get(url).await?; + let headers = response.headers().clone(); + let bytes = response.bytes().await?; + let (content_type, content_length) = extract_metadata_from_headers(&headers)?; + + self.file_manager + .save(&FileInfo { + name: format!("{}.{}", token_id.to_hex(), file_ext), + content: bytes.to_vec(), + dir_path: Some(format!("{:#064x}", contract_address)), + }) + .await?; + + Ok(MetadataImage { + file_type: content_type, + content_length, + is_cache_updated: true, + }) + } + } + } + + /// Retrieves the URI for a token based on its ID and the contract address. + /// The function first checks the `tokenURI` selector and then the `token_uri` selector. + /// If both checks fail, an error is returned indicating the token URI was not found. + async fn get_token_uri( + &mut self, + token_id: &CairoU256, + contract_address: FieldElement, + ) -> Result { + let token_uri_cairo0 = self + .get_contract_property_string( + contract_address, + selector!("tokenURI"), + vec![token_id.low.into(), token_id.high.into()], + BlockId::Tag(BlockTag::Latest), + ) + .await?; + + if self.is_valid_uri(&token_uri_cairo0) { + return Ok(token_uri_cairo0); + } + + let token_uri_cairo1 = self + .get_contract_property_string( + contract_address, + selector!("token_uri"), + vec![token_id.low.into(), token_id.high.into()], + BlockId::Tag(BlockTag::Latest), + ) + .await?; + + if self.is_valid_uri(&token_uri_cairo1) { + Ok(token_uri_cairo1) + } else { + error!("Token URI not found"); + Err(anyhow!("Token URI not found")) + } + } + + /// Checks if the given URI is valid. + /// A URI is considered invalid if it's "undefined" or empty. + fn is_valid_uri(&self, uri: &str) -> bool { + uri != "undefined" && !uri.is_empty() + } + + /// Gets a property string value from a Starknet contract. + /// This function calls the contract and parses the returned value as a string. + async fn get_contract_property_string( + &mut self, + contract_address: FieldElement, + selector: FieldElement, + calldata: Vec, + block: BlockId, + ) -> Result { + let value = self + .starknet_client + .call_contract(contract_address, selector, calldata, block) + .await + .map_err(|_| anyhow!("Error calling contract"))?; + + parse_cairo_long_string(value).map_err(|_| anyhow!("Error parsing string")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use crate::{file_manager::MockFileManager, storage::MockStorage}; + use ark_starknet::client::MockStarknetClient; + use mockall::predicate::*; + use reqwest::header::HeaderMap; + use std::vec; + + #[test] + fn test_extract_metadata_from_headers() { + // Create a mock HeaderMap with some sample headers + let mut headers = HeaderMap::new(); + headers.insert("Content-Type", "image/png".parse().unwrap()); + headers.insert("Content-Length", "12345".parse().unwrap()); + + // Call the function under test + let data = extract_metadata_from_headers(&headers); + + // Assert that the returned values match the headers we set + assert!(data.is_ok()); + + let (content_type, content_length) = data.unwrap(); + + assert_eq!(content_type, "image/png"); + assert_eq!(content_length, 12345u64); + } + + #[tokio::test] + async fn test_get_token_uri() { + // SETUP: Mocking and Initializing + let mut mock_client = MockStarknetClient::default(); + let storage_manager = MockStorage::default(); + let mock_file = MockFileManager::default(); + + mock_client + .expect_call_contract() + .times(1) + .returning(|_, _, _, _| { + Ok(vec![ + FieldElement::from_dec_str("4").unwrap(), + FieldElement::from_hex_be("0x68").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + ]) + }); + + let mut metadata_manager = MetadataManager::new(&storage_manager, &mock_client, &mock_file); + + // EXECUTION: Call the function under test + let result = metadata_manager + .get_token_uri( + &CairoU256 { low: 0, high: 1 }, + FieldElement::from_hex_be( + "0x0727a63f78ee3f1bd18f78009067411ab369c31dece1ae22e16f567906409905", + ) + .unwrap(), + ) + .await; + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_refresh_collection_token_metadata() { + // SETUP: Mocking and Initializing + let mut mock_client = MockStarknetClient::default(); + let mut mock_storage = MockStorage::default(); + let mock_file = MockFileManager::default(); + + let contract_address = FieldElement::ONE; + let ipfs_gateway_uri = "https://ipfs.example.com"; + + // Mocking expected behaviors + mock_storage + .expect_find_token_ids_without_metadata_in_collection() + .times(1) + .with(eq(contract_address)) + .returning(|_| Ok(vec![CairoU256 { low: 1, high: 0 }])); + + mock_client + .expect_call_contract() + .times(1) + .with(always(), always(), always(), always()) + .returning(|_, _, _, _| { + Ok(vec![ + FieldElement::from_dec_str("4").unwrap(), + FieldElement::from_hex_be("0x68").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + ]) + }); + + mock_storage + .expect_register_token_metadata() + .times(1) + .with(always(), always(), always()) + .returning(|_, _, _| Ok(())); + + let mut metadata_manager = MetadataManager::new(&mock_storage, &mock_client, &mock_file); + + // EXECUTION: Call the function under test + let result = metadata_manager + .refresh_collection_token_metadata( + contract_address, + CacheOption::NoCache, + ipfs_gateway_uri, + ) + .await; + + assert!(result.is_ok()); + } + + #[tokio::test] + async fn test_get_contract_property_string() { + // SETUP: Mocking and Initializing + let mut mock_client = MockStarknetClient::default(); + let mock_file = MockFileManager::default(); + + let contract_address = FieldElement::ONE; + let selector_name = selector!("tokenURI"); + + // Configure the mock client to expect a call to 'call_contract' and return a predefined result + mock_client + .expect_call_contract() + .times(1) + .with( + eq(contract_address), + eq(selector_name), + eq(vec![FieldElement::ZERO, FieldElement::ZERO]), + eq(BlockId::Tag(BlockTag::Latest)), + ) + .returning(|_, _, _, _| { + Ok(vec![ + FieldElement::from_dec_str("4").unwrap(), + FieldElement::from_hex_be("0x68").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x74").unwrap(), + FieldElement::from_hex_be("0x70").unwrap(), + ]) + }); + + let storage_manager = MockStorage::default(); + let mut metadata_manager = MetadataManager::new(&storage_manager, &mock_client, &mock_file); + + // EXECUTION: Call the function under test + let result = metadata_manager + .get_contract_property_string( + contract_address, + selector_name, + vec![FieldElement::ZERO, FieldElement::ZERO], + BlockId::Tag(BlockTag::Latest), + ) + .await; + + // ASSERTION: Verify the outcome + let parsed_string = result.expect("Failed to get contract property string"); + assert_eq!(parsed_string, "http"); + } +} diff --git a/crates/ark-metadata/src/storage.rs b/crates/ark-metadata/src/storage.rs new file mode 100644 index 000000000..113e8452e --- /dev/null +++ b/crates/ark-metadata/src/storage.rs @@ -0,0 +1,26 @@ +use crate::types::{StorageError, TokenMetadata}; +use ark_starknet::CairoU256; +#[cfg(any(test, feature = "mock"))] +use mockall::automock; +use starknet::core::types::FieldElement; + +#[cfg_attr(any(test, feature = "mock"), automock)] +pub trait Storage { + fn register_token_metadata( + &self, + contract_address: &FieldElement, + token_id: CairoU256, + token_metadata: TokenMetadata, + ) -> Result<(), StorageError>; + + fn has_token_metadata( + &self, + contract_address: FieldElement, + token_id: CairoU256, + ) -> Result; + + fn find_token_ids_without_metadata_in_collection( + &self, + contract_address: FieldElement, + ) -> Result, StorageError>; +} diff --git a/crates/ark-metadata/src/types.rs b/crates/ark-metadata/src/types.rs new file mode 100644 index 000000000..ea2b78900 --- /dev/null +++ b/crates/ark-metadata/src/types.rs @@ -0,0 +1,56 @@ +use serde_derive::{Deserialize, Serialize}; +use serde_json::Number; + +#[derive(Debug, PartialEq)] +pub enum MetadataType { + Http(String), + Ipfs(String), + OnChain(String), +} + +#[derive(Debug)] +pub enum StorageError { + DatabaseError, + NotFound, + DuplicateToken, + InvalidMintData, +} + +#[derive(Debug, Deserialize, Serialize)] +pub enum DisplayType { + Number, + BoostPercentage, + BoostNumber, + Date, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(untagged)] +pub enum AttributeValue { + String(String), + Number(Number), + Bool(bool), + StringVec(Vec), + NumberVec(Vec), + BoolVec(Vec), +} + +#[derive(Debug, Deserialize, Serialize)] +pub struct Attribute { + pub display_type: Option, + pub trait_type: Option, + pub value: AttributeValue, +} + +#[derive(Debug, Default, Deserialize, Serialize)] +pub struct TokenMetadata { + pub image: Option, + pub image_data: Option, + pub external_url: Option, + pub description: Option, + pub name: Option, + pub attributes: Option>, + pub background_color: Option, + pub animation_url: Option, + pub youtube_url: Option, +} diff --git a/crates/ark-metadata/src/utils.rs b/crates/ark-metadata/src/utils.rs new file mode 100644 index 000000000..f1e08eb99 --- /dev/null +++ b/crates/ark-metadata/src/utils.rs @@ -0,0 +1,172 @@ +use crate::types::{MetadataType, TokenMetadata}; +use anyhow::{anyhow, Result}; +use base64::{engine::general_purpose, Engine as _}; +use log::debug; +use reqwest::header::{HeaderMap, CONTENT_LENGTH, CONTENT_TYPE}; +use reqwest::Client; +use std::{env, time::Duration}; + +pub async fn get_token_metadata(client: &Client, uri: &str) -> Result { + let metadata_type = get_metadata_type(uri); + let metadata = match metadata_type { + MetadataType::Ipfs(uri) => get_ipfs_metadata(&uri, client).await?, + MetadataType::Http(uri) => get_http_metadata(&uri, client).await?, + MetadataType::OnChain(uri) => get_onchain_metadata(&uri)?, + }; + Ok(metadata) +} + +pub fn get_metadata_type(uri: &str) -> MetadataType { + if uri.starts_with("ipfs://") { + MetadataType::Ipfs(uri.to_string()) + } else if uri.starts_with("http://") || uri.starts_with("https://") { + MetadataType::Http(uri.to_string()) + } else { + MetadataType::OnChain(uri.to_string()) + } +} + +async fn get_ipfs_metadata(uri: &str, client: &Client) -> Result { + let mut ipfs_url = env::var("IPFS_GATEWAY_URI").expect("IPFS_GATEWAY_URI must be set"); + let ipfs_hash = uri.trim_start_matches("ipfs://"); + ipfs_url.push_str(ipfs_hash); + let request = client.get(ipfs_url).timeout(Duration::from_secs(3)); + let response = request.send().await?; + let metadata = response.json::().await?; + Ok(metadata) +} + +async fn get_http_metadata(uri: &str, client: &Client) -> Result { + let resp = client.get(uri).send().await?; + let metadata: TokenMetadata = resp.json().await?; + Ok(metadata) +} + +fn get_onchain_metadata(uri: &str) -> Result { + // Try to split from the comma as it is the standard with on chain metadata + let url_encoded = urlencoding::decode(uri).map(|s| String::from(s.as_ref())); + let uri_string = match url_encoded { + Ok(encoded) => encoded, + Err(_) => String::from(uri), + }; + + match uri_string.split_once(',') { + Some(("data:application/json;base64", uri)) => { + // If it is base64 encoded, decode it, parse and return + let decoded = general_purpose::STANDARD.decode(uri)?; + let decoded = std::str::from_utf8(&decoded)?; + let metadata: TokenMetadata = serde_json::from_str(decoded)?; + Ok(metadata) + } + Some(("data:application/json", uri)) => { + // If it is plain json, parse it and return + //println!("Handling {:?}", uri); + let metadata: TokenMetadata = serde_json::from_str(uri)?; + Ok(metadata) + } + _ => match serde_json::from_str(uri) { + // If it is only the URI without the data format information, try to format it + // and if it fails, return empty metadata + Ok(v) => Ok(v), + Err(_) => Ok(TokenMetadata::default()), + }, + } +} + +pub fn extract_metadata_from_headers(headers: &HeaderMap) -> Result<(String, u64)> { + debug!("Extracting metadata from headers..."); + + let content_type = headers + .get(CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .ok_or_else(|| { + debug!("Failed to extract content type."); + anyhow!("Failed to extract content type") + })?; + + let content_length = headers + .get(CONTENT_LENGTH) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| { + debug!("Failed to extract or parse content length."); + anyhow!("Failed to extract or parse content length") + })?; + + debug!( + "Successfully extracted content type: {} and content length: {}", + content_type, content_length + ); + Ok((content_type.to_string(), content_length)) +} + +#[cfg(test)] +mod tests { + use super::*; + use reqwest::header::{HeaderMap, HeaderValue, CONTENT_LENGTH, CONTENT_TYPE}; + + #[tokio::test] + async fn test_determining_metadata_type() { + let metadata_type = + get_metadata_type("ipfs://QmZkPTq6AGnsoCkYiDPCFMaAjHpZAfHipyJeAdwtJh1fP5"); + assert!( + metadata_type + == MetadataType::Ipfs( + "ipfs://QmZkPTq6AGnsoCkYiDPCFMaAjHpZAfHipyJeAdwtJh1fP5".to_string() + ) + ); + + let metadata_type = get_metadata_type("https://everai.xyz/metadata/1"); + assert!(metadata_type == MetadataType::Http("https://everai.xyz/metadata/1".to_string())); + } + + #[test] + fn test_extract_valid_headers() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + headers.insert(CONTENT_LENGTH, HeaderValue::from_static("42")); + + let metadata = extract_metadata_from_headers(&headers).unwrap(); + assert_eq!(metadata, ("text/plain".to_string(), 42)); + } + + #[test] + fn test_extract_missing_content_type() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_LENGTH, HeaderValue::from_static("42")); + + let result = extract_metadata_from_headers(&headers); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Failed to extract content type" + ); + } + + #[test] + fn test_extract_missing_content_length() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + + let result = extract_metadata_from_headers(&headers); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Failed to extract or parse content length" + ); + } + + #[test] + fn test_extract_invalid_content_length_format() { + let mut headers = HeaderMap::new(); + headers.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain")); + headers.insert(CONTENT_LENGTH, HeaderValue::from_static("invalid")); + + let result = extract_metadata_from_headers(&headers); + assert!(result.is_err()); + assert_eq!( + result.unwrap_err().to_string(), + "Failed to extract or parse content length" + ); + } +} diff --git a/crates/ark-starknet/Cargo.toml b/crates/ark-starknet/Cargo.toml index 914a0cb07..00b9dfc3c 100644 --- a/crates/ark-starknet/Cargo.toml +++ b/crates/ark-starknet/Cargo.toml @@ -7,6 +7,10 @@ edition = "2021" anyhow.workspace = true async-trait.workspace = true starknet.workspace = true - url = "2.3.1" regex = "1.9.1" +mockall = "0.11.2" +num-bigint = { version = "0.4.3", default-features = false } + +[features] +mock = [] \ No newline at end of file diff --git a/crates/ark-starknet/src/client/http.rs b/crates/ark-starknet/src/client/http.rs index 6edad5b14..7dbe2088d 100644 --- a/crates/ark-starknet/src/client/http.rs +++ b/crates/ark-starknet/src/client/http.rs @@ -2,6 +2,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; + use regex::Regex; use starknet::{ core::{types::FieldElement, types::*}, @@ -26,7 +27,7 @@ impl StarknetClient for StarknetClientHttp { let rpc_url = Url::parse(rpc_url)?; let provider = AnyProvider::JsonRpcHttp(JsonRpcClient::new(HttpTransport::new(rpc_url))); - Ok(StarknetClientHttp { provider }) + Ok(Self { provider }) } /// diff --git a/crates/ark-starknet/src/client/mod.rs b/crates/ark-starknet/src/client/mod.rs index 762faae99..5beb8fd1d 100644 --- a/crates/ark-starknet/src/client/mod.rs +++ b/crates/ark-starknet/src/client/mod.rs @@ -3,12 +3,15 @@ pub use http::StarknetClientHttp; use anyhow::Result; use async_trait::async_trait; +#[cfg(any(test, feature = "mock"))] +use mockall::automock; use starknet::core::{types::FieldElement, types::*}; use std::collections::HashMap; use std::marker::Sized; /// Starknet client interface with required methods /// for arkproject capabilities only. +#[cfg_attr(any(test, feature = "mock"), automock)] #[async_trait] pub trait StarknetClient { /// diff --git a/crates/ark-starknet/src/lib.rs b/crates/ark-starknet/src/lib.rs index b9babe5bc..43e186e1c 100644 --- a/crates/ark-starknet/src/lib.rs +++ b/crates/ark-starknet/src/lib.rs @@ -1 +1,38 @@ pub mod client; + +use num_bigint::BigUint; + +#[derive(Debug)] +pub struct CairoU256 { + pub low: u128, + pub high: u128, +} + +impl CairoU256 { + pub fn to_biguint(&self) -> BigUint { + let low_bytes = self.low.to_be_bytes(); + let high_bytes = self.high.to_be_bytes(); + + let mut bytes: Vec = Vec::new(); + bytes.extend(high_bytes); + bytes.extend(low_bytes); + + BigUint::from_bytes_be(&bytes[..]) + } + + pub fn to_hex(&self) -> String { + let token_id_big_uint = self.to_biguint(); + format!("{:#064x}", token_id_big_uint) + } + + pub fn to_decimal(&self, padded: bool) -> String { + let token_id_big_uint = self.to_biguint(); + let token_id_str: String = token_id_big_uint.to_str_radix(10); + + if padded { + format!("{:0>width$}", token_id_str, width = 78) + } else { + token_id_str + } + } +} diff --git a/crates/ark-storage/Cargo.toml b/crates/ark-storage/Cargo.toml index 810e5d6c3..7b7930c06 100644 --- a/crates/ark-storage/Cargo.toml +++ b/crates/ark-storage/Cargo.toml @@ -7,4 +7,9 @@ edition = "2021" starknet = "0.5.0" num-bigint = { version = "0.4.3", default-features = false } serde = { version = "1.0.130", features = ["derive"] } -log = "0.4.14" \ No newline at end of file +log = "0.4.14" +serde_json = "1.0" +mockall = "0.11.2" + +[features] +mock = [] \ No newline at end of file diff --git a/crates/ark-storage/src/storage_manager.rs b/crates/ark-storage/src/storage_manager.rs index 29e9c4f9b..0fb2ae899 100644 --- a/crates/ark-storage/src/storage_manager.rs +++ b/crates/ark-storage/src/storage_manager.rs @@ -3,8 +3,11 @@ use crate::types::{ TokenFromEvent, }; use log; +#[cfg(any(test, feature = "mock"))] +use mockall::automock; use starknet::core::types::FieldElement; +#[cfg_attr(any(test, feature = "mock"), automock)] pub trait StorageManager { fn register_mint(&self, token: &TokenFromEvent) -> Result<(), StorageError>; diff --git a/crates/ark-storage/src/types.rs b/crates/ark-storage/src/types.rs index 44467ca65..ac2210cd0 100644 --- a/crates/ark-storage/src/types.rs +++ b/crates/ark-storage/src/types.rs @@ -13,7 +13,8 @@ pub enum StorageError { } impl fmt::Display for StorageError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Note the lifetime parameter <'_> match self { StorageError::DatabaseError => write!(f, "Database error occurred"), StorageError::NotFound => write!(f, "Item not found in storage"), diff --git a/scripts/clippy.sh b/scripts/clippy.sh old mode 100755 new mode 100644 diff --git a/src/lib.rs b/src/lib.rs index cdd051c45..48ab7b7f8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,3 +11,7 @@ pub mod nft_indexer { pub mod nft_storage { pub use ark_storage::*; } + +pub mod nft_metadata { + pub use ark_metadata::*; +}