Store jsonb extra_metadata correctly

gallery-so · Jul 1, 2024 · 37297fb · 37297fb
1 parent 53bd520
commit 37297fb
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 30 deletions.
diff --git a/db/gen/mirrordb/batch.go b/db/gen/mirrordb/batch.go
diff --git a/db/queries/mirror/query.sql b/db/queries/mirror/query.sql
@@ -318,7 +318,7 @@ token_insert as (
             @last_sale,
             @first_created,
             @rarity,
-            @extra_metadata::jsonb,
+            @extra_metadata,
             @image_properties,
             @video_properties,
             @audio_properties,
@@ -328,7 +328,7 @@ token_insert as (
             @kafka_offset,
             @kafka_partition,
             @kafka_timestamp,
-            @extra_metadata,
+            @extra_metadata_jsonb::jsonb,
             now()
         where @should_upsert::bool
         on conflict (simplehash_kafka_key) do update
@@ -364,7 +364,7 @@ token_insert as (
                 kafka_offset = excluded.kafka_offset,
                 kafka_partition = excluded.kafka_partition,
                 kafka_timestamp = excluded.kafka_timestamp,
-                extra_metadata_jsonb = excluded.extra_metadata::jsonb,
+                extra_metadata_jsonb = excluded.extra_metadata_jsonb,
                 last_metadata_conversion = now()
 )
 select @simplehash_nft_id::text
@@ -454,7 +454,7 @@ token_insert as (
             @last_sale,
             @first_created,
             @rarity,
-            @extra_metadata::jsonb,
+            @extra_metadata,
             @image_properties,
             @video_properties,
             @audio_properties,
@@ -464,7 +464,7 @@ token_insert as (
             @kafka_offset,
             @kafka_partition,
             @kafka_timestamp,
-            @extra_metadata,
+            @extra_metadata_jsonb::jsonb,
             now()
         where @should_upsert::bool
         on conflict (simplehash_kafka_key) do update
@@ -500,7 +500,7 @@ token_insert as (
                 kafka_offset = excluded.kafka_offset,
                 kafka_partition = excluded.kafka_partition,
                 kafka_timestamp = excluded.kafka_timestamp,
-                extra_metadata_jsonb = excluded.extra_metadata::jsonb,
+                extra_metadata_jsonb = excluded.extra_metadata_jsonb,
                 last_metadata_conversion = now()
 )
 select @simplehash_nft_id::text
@@ -590,7 +590,7 @@ token_insert as (
             @last_sale,
             @first_created,
             @rarity,
-            @extra_metadata::jsonb,
+            @extra_metadata,
             @image_properties,
             @video_properties,
             @audio_properties,
@@ -600,7 +600,7 @@ token_insert as (
             @kafka_offset,
             @kafka_partition,
             @kafka_timestamp,
-            @extra_metadata,
+            @extra_metadata_jsonb::jsonb,
             now()
         where @should_upsert::bool
         on conflict (simplehash_kafka_key) do update
@@ -636,7 +636,7 @@ token_insert as (
                 kafka_offset = excluded.kafka_offset,
                 kafka_partition = excluded.kafka_partition,
                 kafka_timestamp = excluded.kafka_timestamp,
-                extra_metadata_jsonb = excluded.extra_metadata::jsonb,
+                extra_metadata_jsonb = excluded.extra_metadata_jsonb,
                 last_metadata_conversion = now()
 )
 select @simplehash_nft_id::text
@@ -726,7 +726,7 @@ token_insert as (
             @last_sale,
             @first_created,
             @rarity,
-            @extra_metadata::jsonb,
+            @extra_metadata,
             @image_properties,
             @video_properties,
             @audio_properties,
@@ -736,7 +736,7 @@ token_insert as (
             @kafka_offset,
             @kafka_partition,
             @kafka_timestamp,
-            @extra_metadata,
+            @extra_metadata_jsonb::jsonb,
             now()
         where @should_upsert::bool
         on conflict (simplehash_kafka_key) do update
@@ -772,7 +772,7 @@ token_insert as (
                 kafka_offset = excluded.kafka_offset,
                 kafka_partition = excluded.kafka_partition,
                 kafka_timestamp = excluded.kafka_timestamp,
-                extra_metadata_jsonb = excluded.extra_metadata::jsonb,
+                extra_metadata_jsonb = excluded.extra_metadata_jsonb,
                 last_metadata_conversion = now()
 )
 select @simplehash_nft_id::text

diff --git a/kafka-streamer/main.go b/kafka-streamer/main.go
@@ -665,7 +665,7 @@ func parseTokenMessage(ctx context.Context, deserializer *avro.GenericDeserializ
 		return mirrordb.ProcessEthereumTokenEntryParams{}, err
 	}
 
-	extraMetadata, err := toJSONB(nft.Extra_metadata)
+	extraMetadataJsonb, err := toJSONB(nft.Extra_metadata)
 	if err != nil {
 		err = fmt.Errorf("failed to convert Extra_metadata to JSONB: %w", err)
 		return mirrordb.ProcessEthereumTokenEntryParams{}, err
@@ -774,7 +774,8 @@ func parseTokenMessage(ctx context.Context, deserializer *avro.GenericDeserializ
 				LastSale:           lastSale,
 				FirstCreated:       firstCreated,
 				Rarity:             rarity,
-				ExtraMetadata:      extraMetadata,
+				ExtraMetadata:      cleanString(nft.Extra_metadata),
+				ExtraMetadataJsonb: extraMetadataJsonb,
 				ImageProperties:    imageProperties,
 				VideoProperties:    videoProperties,
 				AudioProperties:    audioProperties,