From 89670a2ed99e5472597865c9eded8a5f6cc99c52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9mi=20Roycourt?= Date: Fri, 15 Sep 2023 13:22:42 +0200 Subject: [PATCH] feat(ark-metadata): use storage --- Cargo.lock | 1 + crates/ark-metadata/src/lib.rs | 2 +- crates/ark-metadata/src/metadata/mod.rs | 26 --- .../src/metadata/normalization.rs | 8 +- crates/ark-metadata/src/metadata_manager.rs | 150 +++++++++++------- crates/ark-storage/Cargo.toml | 3 +- crates/ark-storage/src/storage_manager.rs | 18 ++- crates/ark-storage/src/types.rs | 30 ++++ 8 files changed, 150 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 40c1d48e1..6de32c13c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,6 +189,7 @@ dependencies = [ "log", "num-bigint", "serde", + "serde_json", "starknet", ] diff --git a/crates/ark-metadata/src/lib.rs b/crates/ark-metadata/src/lib.rs index 741afb6ea..6bf5d26ff 100644 --- a/crates/ark-metadata/src/lib.rs +++ b/crates/ark-metadata/src/lib.rs @@ -1,3 +1,3 @@ -mod metadata; +pub mod metadata; mod cairo_string_parser; pub mod metadata_manager; \ No newline at end of file diff --git a/crates/ark-metadata/src/metadata/mod.rs b/crates/ark-metadata/src/metadata/mod.rs index dc4ad342c..2f516bc8a 100644 --- a/crates/ark-metadata/src/metadata/mod.rs +++ b/crates/ark-metadata/src/metadata/mod.rs @@ -1,27 +1 @@ pub mod normalization; - -use serde_derive::{Serialize, Deserialize}; -use serde_json::Value; - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] -pub enum MetadataAttributeValue { - Str(String), - Int(i64), - Float(f64), - Value(Value), -} - -#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] -pub struct MetadataAttribute { - pub trait_type: String, - pub value: MetadataAttributeValue, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct NormalizedMetadata { - pub description: String, - pub external_url: String, - pub image: String, - pub name: String, - pub attributes: Vec, -} \ No newline at end of file diff --git a/crates/ark-metadata/src/metadata/normalization.rs b/crates/ark-metadata/src/metadata/normalization.rs index ea47171b5..47a67f58b 100644 --- a/crates/ark-metadata/src/metadata/normalization.rs +++ b/crates/ark-metadata/src/metadata/normalization.rs @@ -1,9 +1,8 @@ use anyhow::{anyhow, Ok, Result}; +use ark_storage::types::{NormalizedMetadata, MetadataAttribute, MetadataAttributeValue}; use log::warn; use serde_json::Value; -use super::{NormalizedMetadata, MetadataAttribute, MetadataAttributeValue}; - // fn normalize_metadata_attributes_with_eip721_standard( // metadata_uri: String, // raw_metadata: &Value, @@ -154,9 +153,10 @@ pub fn normalize_metadata( #[cfg(test)] mod tests { - use crate::metadata::{normalization::{ + use crate::metadata::normalization::{ normalize_metadata, normalize_metadata_attributes_with_opensea_standard, - }, MetadataAttributeValue}; + }; + use ark_storage::types::MetadataAttributeValue; use serde_json::json; #[test] diff --git a/crates/ark-metadata/src/metadata_manager.rs b/crates/ark-metadata/src/metadata_manager.rs index a82d4bf50..e34e7be27 100644 --- a/crates/ark-metadata/src/metadata_manager.rs +++ b/crates/ark-metadata/src/metadata_manager.rs @@ -1,12 +1,11 @@ - use crate::cairo_string_parser::parse_cairo_long_string; -use crate::metadata::NormalizedMetadata; use crate::metadata::normalization::normalize_metadata; use anyhow::{anyhow, Result}; use ark_starknet::client::StarknetClient; use ark_storage::storage_manager::StorageManager; -use log::{info, debug}; +use ark_storage::types::{NormalizedMetadata, TokenId, TokenMetadata}; +use log::{debug, error, info}; use reqwest::Client as ReqwestClient; use serde_json::Value; use starknet::core::types::{BlockId, BlockTag, FieldElement}; @@ -19,6 +18,13 @@ pub struct MetadataManager<'a, T: StorageManager, C: StarknetClient> { request_client: ReqwestClient, } +#[derive(Debug)] +pub enum MetadataError { + DatabaseError, + ParsingError, + RequestError, +} + impl<'a, T: StorageManager, C: StarknetClient> MetadataManager<'a, T, C> { pub fn new(storage: &'a T, starknet_client: &'a C) -> Self { MetadataManager { @@ -28,30 +34,56 @@ impl<'a, T: StorageManager, C: StarknetClient> MetadataManager<'a, T, C> { } } + /// Refreshes the metadata for a given token. + /// + /// - `contract_address`: The address of the contract. + /// - `token_id_low`: The low end of the token ID range. + /// - `token_id_high`: The high end of the token ID range. + /// - `block_id`: The ID of the block. + /// + /// Returns an `Err` variant of `MetadataError` if there's a problem in parsing the token URI, fetching metadata, or database interaction. pub async fn refresh_metadata_for_token( &mut self, contract_address: FieldElement, token_id_low: FieldElement, token_id_high: FieldElement, - block_id: BlockId, - ) -> Result<()> { + ) -> Result<(), MetadataError> { let token_uri = self .get_token_uri(token_id_low, token_id_high, contract_address) - .await?; + .await + .map_err(|_| MetadataError::ParsingError)?; - // TODO: check if token_uri is already in db + let has_token_metadata = self + .storage + .has_token_metadata( + contract_address, + TokenId { + low: token_id_low, + high: token_id_high, + }, + ) + .map_err(|_| MetadataError::DatabaseError)?; - // TODO: save token uri in db + if has_token_metadata { + return Ok(()); + } - let (raw_metadata, normalized_metadata) = - self.fetch_metadata(&token_uri, &token_uri).await?; + let (raw_metadata, normalized_metadata) = self + .fetch_metadata(&token_uri, &token_uri) + .await + .map_err(|_| MetadataError::RequestError)?; - // TODO: save metadata in db + self.storage + .register_token_metadata(TokenMetadata { + normalized_metadata, + raw_metadata, + }) + .map_err(|_e| MetadataError::DatabaseError)?; Ok(()) } - pub async fn refresh_metadata_for_token_collection()-> Result<()> { + pub async fn refresh_metadata_for_token_collection() -> Result<()> { Ok(()) } @@ -61,41 +93,53 @@ impl<'a, T: StorageManager, C: StarknetClient> MetadataManager<'a, T, C> { token_id_high: FieldElement, contract_address: FieldElement, ) -> Result { - debug!("get_token_id: [{:?}, {:?}]", token_id_low, token_id_high); - match self - .get_contract_property_string( - contract_address, + let token_uri_cairo0 = self + .fetch_token_uri( selector!("tokenURI"), - vec![token_id_low.clone(), token_id_high.clone()], - BlockId::Tag(BlockTag::Latest), + token_id_low.clone(), + token_id_high.clone(), + contract_address.clone(), ) - .await - { - Ok(token_uri_cairo0) => { - if token_uri_cairo0 != "undefined" && !token_uri_cairo0.is_empty() { - return Err(anyhow!("Token URI not found")); - } - - match self - .get_contract_property_string( - contract_address, - selector!("token_uri"), - vec![token_id_low.clone(), token_id_high.clone()], - BlockId::Tag(BlockTag::Latest), - ) - .await - { - Ok(token_uri_cairo1) => { - if token_uri_cairo1 != "undefined" && !token_uri_cairo1.is_empty() { - return Ok(token_uri_cairo1); - } - return Err(anyhow!("Token URI not found")); - } - Err(_) => Err(anyhow!("Token URI not found")), - } - } - Err(_) => Err(anyhow!("Token URI not found")), + .await?; + + if self.is_valid_uri(&token_uri_cairo0) { + return Err(anyhow!("Token URI not found")); } + + let token_uri_cairo1 = self + .fetch_token_uri( + selector!("token_uri"), + token_id_low, + token_id_high, + contract_address, + ) + .await?; + + if self.is_valid_uri(&token_uri_cairo1) { + return Ok(token_uri_cairo1); + } else { + return Err(anyhow!("Token URI not found")); + } + } + + async fn fetch_token_uri( + &mut self, + selector: FieldElement, + token_id_low: FieldElement, + token_id_high: FieldElement, + contract_address: FieldElement, + ) -> Result { + self.get_contract_property_string( + contract_address, + selector, + vec![token_id_low, token_id_high], + BlockId::Tag(BlockTag::Latest), + ) + .await + } + + fn is_valid_uri(&self, uri: &String) -> bool { + uri != "undefined" && !uri.is_empty() } async fn fetch_metadata( @@ -110,17 +154,14 @@ impl<'a, T: StorageManager, C: StarknetClient> MetadataManager<'a, T, C> { .get(metadata_uri) .timeout(Duration::from_secs(3)) .send() - .await; + .await?; - match response { - Ok(resp) => match resp.json::().await { - Ok(raw_metadata) => { - let normalized_metadata = - normalize_metadata(initial_metadata_uri.to_string(), raw_metadata.clone())?; - Ok((raw_metadata, normalized_metadata)) - } - Err(e) => Err(e.into()), - }, + match response.json::().await { + Ok(raw_metadata) => { + let normalized_metadata = + normalize_metadata(initial_metadata_uri.to_string(), raw_metadata.clone())?; + Ok((raw_metadata, normalized_metadata)) + } Err(e) => Err(e.into()), } } @@ -182,7 +223,6 @@ mod tests { // FieldElement::from_hex_be("0x0727a63f78ee3f1bd18f78009067411ab369c31dece1ae22e16f567906409905").unwrap()) // .await; - // assert!(result.is_ok()); // } diff --git a/crates/ark-storage/Cargo.toml b/crates/ark-storage/Cargo.toml index 810e5d6c3..1af4431f1 100644 --- a/crates/ark-storage/Cargo.toml +++ b/crates/ark-storage/Cargo.toml @@ -7,4 +7,5 @@ edition = "2021" starknet = "0.5.0" num-bigint = { version = "0.4.3", default-features = false } serde = { version = "1.0.130", features = ["derive"] } -log = "0.4.14" \ No newline at end of file +log = "0.4.14" +serde_json = "1.0" \ No newline at end of file diff --git a/crates/ark-storage/src/storage_manager.rs b/crates/ark-storage/src/storage_manager.rs index 29e9c4f9b..44119b88a 100644 --- a/crates/ark-storage/src/storage_manager.rs +++ b/crates/ark-storage/src/storage_manager.rs @@ -1,6 +1,6 @@ use crate::types::{ BlockIndexing, BlockIndexingStatus, BlockInfo, ContractInfo, StorageError, TokenEvent, - TokenFromEvent, + TokenFromEvent, TokenMetadata, TokenId, }; use log; use starknet::core::types::FieldElement; @@ -30,6 +30,10 @@ pub trait StorageManager { fn set_block_info(&self, block_number: u64, info: BlockInfo) -> Result<(), StorageError>; fn set_indexer_progress(&self, progress: BlockIndexing) -> Result<(), StorageError>; + + fn register_token_metadata(&self, token_metadata: TokenMetadata) -> Result<(), StorageError>; + + fn has_token_metadata(&self, contract_address: FieldElement, token_id: TokenId) -> Result; } pub struct DefaultStorage; @@ -126,4 +130,16 @@ impl StorageManager for DefaultStorage { // Err(StorageError::DatabaseError) Ok(()) } + + fn register_token_metadata(&self, token_metadata: TokenMetadata) -> Result<(), StorageError> { + log::debug!("Registering token metadata"); + // TODO: In future, handle and return potential errors + // Err(StorageError::DatabaseError) + Ok(()) + } + + fn has_token_metadata(&self, contract_address: FieldElement, token_id: TokenId) -> Result { + log::debug!("Checking if token metadata exists"); + Ok(false) + } } diff --git a/crates/ark-storage/src/types.rs b/crates/ark-storage/src/types.rs index 5c97bc1a8..af8882860 100644 --- a/crates/ark-storage/src/types.rs +++ b/crates/ark-storage/src/types.rs @@ -1,6 +1,7 @@ use crate::utils::format_token_id; use num_bigint::BigUint; use serde::{Deserialize, Serialize}; +use serde_json::Value; use starknet::core::types::FieldElement; #[derive(Debug)] @@ -161,6 +162,35 @@ pub struct BlockInfo { pub status: BlockIndexingStatus, } +#[derive(Debug)] +pub struct TokenMetadata { + pub raw_metadata: Value, + pub normalized_metadata: NormalizedMetadata, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct NormalizedMetadata { + pub description: String, + pub external_url: String, + pub image: String, + pub name: String, + pub attributes: Vec, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub struct MetadataAttribute { + pub trait_type: String, + pub value: MetadataAttributeValue, +} + +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] +pub enum MetadataAttributeValue { + Str(String), + Int(i64), + Float(f64), + Value(Value), +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(rename_all = "snake_case")] pub enum ContractType {