Merge remote-tracking branch 'upstream/master' into qn_mappings_review

distributor-node/CHANGELOG.md

@@ -7,6 +7,7 @@
 - Include response headers in `http` logs
 - Disable open-api express response validation if NODE_ENV is set to 'production' or 'prod'. This should improve response times when serving assets.
 - Include `nodeEnv` in `/api/v1/status` response, to help detect mis-configured nodes.
+- **FIX** Axios Error Logging: Logging the error, when asset download from storage-node time outs, has been fixed to include the _only_ error message, response, status code and bunch of other fields. Previously, logging error object (which includes axios client instance), failed with `Converting circular structure to JSON` error and causing the distributor-node to crash.
 
 ### 1.3.1
 

distributor-node/src/services/networking/NetworkingService.ts

@@ -332,7 +332,9 @@ export class NetworkingService {
       })
 
       objectDownloadQueue.on('error', (err) => {
-        this.logger.error('Download attempt from storage node failed after availability was confirmed:', { err })
+        this.logger.error('Download attempt from storage node failed after availability was confirmed:', {
+          err: axios.isAxiosError(err) ? parseAxiosError(err) : err,
+        })
       })
 
       objectDownloadQueue.on('end', () => {

distributor-node/src/services/networking/query-node/schema.graphql

@@ -3196,7 +3196,10 @@ type Membership implements BaseGraphQLObject {
   deletedById: ID
   version: Int!
 
-  """The unique handle chosen by member"""
+  """The unique handle chosen by member as raw hex representation"""
+  handleRaw: String!
+
+  """The unique handle chosen by member as utf-8"""
   handle: String!
   metadata: MemberMetadata!
   metadataId: String!
@@ -17569,6 +17572,11 @@ input MemberProfileUpdatedEventWhereInput {
   newHandle_startsWith: String
   newHandle_endsWith: String
   newHandle_in: [String!]
+  newHandleRaw_eq: String
+  newHandleRaw_contains: String
+  newHandleRaw_startsWith: String
+  newHandleRaw_endsWith: String
+  newHandleRaw_in: [String!]
   member: MembershipWhereInput
   newMetadata: MemberMetadataWhereInput
   AND: [MemberProfileUpdatedEventWhereInput!]
@@ -17587,6 +17595,7 @@ input MemberProfileUpdatedEventCreateInput {
   indexInBlock: Float!
   member: ID!
   newHandle: String
+  newHandleRaw: String
   newMetadata: ID!
 }
 
@@ -17597,6 +17606,7 @@ input MemberProfileUpdatedEventUpdateInput {
   indexInBlock: Float
   member: ID
   newHandle: String
+  newHandleRaw: String
   newMetadata: ID
 }
 
@@ -20798,6 +20808,11 @@ input MembershipWhereInput {
   deletedAt_gte: DateTime
   deletedById_eq: ID
   deletedById_in: [ID!]
+  handleRaw_eq: String
+  handleRaw_contains: String
+  handleRaw_startsWith: String
+  handleRaw_endsWith: String
+  handleRaw_in: [String!]
   handle_eq: String
   handle_contains: String
   handle_startsWith: String
@@ -21094,10 +21109,11 @@ input MembershipWhereInput {
 
 input MembershipWhereUniqueInput {
   id: ID
-  handle: String
+  handleRaw: String
 }
 
 input MembershipCreateInput {
+  handleRaw: String!
   handle: String!
   metadata: ID!
   controllerAccount: String!
@@ -21114,6 +21130,7 @@ input MembershipCreateInput {
 }
 
 input MembershipUpdateInput {
+  handleRaw: String
   handle: String
   metadata: ID
   controllerAccount: String
@@ -26271,8 +26288,15 @@ type MemberProfileUpdatedEvent implements Event & BaseGraphQLObject {
   member: Membership!
   memberId: String!
 
-  """New member handle. Null means no new value was provided."""
+  """
+  New member handle (utf-8 string). Null means no new value was provided.
+  """
   newHandle: String
+
+  """
+  New member handle (raw hex string). Null means no new value was provided.
+  """
+  newHandleRaw: String
   newMetadata: MemberMetadata!
   newMetadataId: String!
 }
@@ -30509,6 +30533,8 @@ enum MemberProfileUpdatedEventOrderByInput {
   member_DESC
   newHandle_ASC
   newHandle_DESC
+  newHandleRaw_ASC
+  newHandleRaw_DESC
   newMetadata_ASC
   newMetadata_DESC
 }
@@ -30635,6 +30661,8 @@ enum MembershipOrderByInput {
   updatedAt_DESC
   deletedAt_ASC
   deletedAt_DESC
+  handleRaw_ASC
+  handleRaw_DESC
   handle_ASC
   handle_DESC
   metadata_ASC

entrypoints/graphql-server.sh

@@ -4,5 +4,5 @@
 if [[ "$TELEMETRY_ENABLED" = "yes" ]]; then
     yarn workspace query-node-root query-node:start:prod:with-instrumentation $*
 else
-    yarn workspace query-node-root query-node:start:prod $*
+    yarn workspace query-node-root query-node:start:prod:pm2 $*
 fi

joystreamjs/CHANGELOG.md

@@ -1,3 +1,7 @@
+### 1.5.0
+
+- Update dependency on metadata protobuf to v2.10.0
+
 ### 1.4.0
 
 - Add `generateAppActionCommitment` to utils

joystreamjs/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joystream/js",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "license": "GPL-3.0-only",
   "description": "Joystream JS package provides  utilities required to work with Joystream network.",
   "main": "lib/index.js",
@@ -41,7 +41,7 @@
     "generate:all": "yarn generate:schema-typings"
   },
   "dependencies": {
-    "@joystream/metadata-protobuf": "^2.8.1",
+    "@joystream/metadata-protobuf": "^2.10.0",
     "@joystream/types": "^2.0.0",
     "@polkadot/util-crypto": "9.5.1",
     "axios": "^1.2.1",

metadata-protobuf/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@joystream/metadata-protobuf",
-  "version": "2.8.1",
+  "version": "2.10.0",
   "description": "Joystream Metadata Protobuf Library",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",

metadata-protobuf/proto/Token.proto

@@ -0,0 +1,35 @@
+syntax = "proto2";
+message CreatorTokenIssuerRemarked {
+    oneof creator_token_issuer_remarked {
+        UpdateTokenMetadata update_token_metadata = 1;
+    }
+}
+
+message UpdateTokenMetadata {
+    optional TokenMetadata new_metadata = 1;
+}
+
+message TokenMetadata {
+  optional string name = 1; // Title
+  optional string symbol = 2; // Symbol
+  optional string description = 3; // token description
+  oneof avatar {
+    uint32 avatar_object = 4; // avatar for token - index into external [assets array](#.Assets)
+    string avatar_uri = 5; // Url to member's avatar
+  }
+  repeated Benefit benefits = 6; // benefits for tokne
+  optional string whitelist_application_note = 7; // note for applicant
+  optional string whitelist_application_apply_link = 8; // link to application process
+  optional uint64 trailer_video_id = 9; // runtime id for video trailer
+}
+
+message Benefit {
+  optional string title = 1;
+  optional string description = 2;
+  optional string emoji = 3;
+  optional uint32 display_order = 4;
+}
+
+message SaleMetadata {
+  optional string terms_and_conditions = 1;
+}

query-node/CHANGELOG.md

@@ -1,3 +1,12 @@
+### 1.6.0
+
+- Store membership handles both as utf-8 string and raw bytes - [#4950](https://github.com/Joystream/joystream/pull/4950)
+
+### 1.5.0
+
+- Add add linked-in to membership external resource [#4927](https://github.com/Joystream/joystream/pull/4927)
+- Schema updated, processor needs to be re-synced.
+
 ### 1.4.0
 
 - Adds `StorageBag.objectsSize` field to `StorageBag` entity schema. This enables to query the total size of all objects in a storage bag. [#4818](https://github.com/Joystream/joystream/pull/4818)
@@ -6,7 +15,7 @@
 
 ### 1.3.0
 
-- Fix external resources mapping of membership metadata to ignore unrecognized type.
+- Fix external resources mapping of membership metadata to ignore unrecognized type. [#4838](https://github.com/Joystream/joystream/pull/4838)
 
 ### 1.2.2
 

query-node/build.sh

@@ -40,7 +40,8 @@ cp ./generated/graphql-server/generated/schema.graphql ../distributor-node/src/s
 
 # Add command to run Query Node's Graphql server with Opentelemetry instrumentation
 sed -i -e '/"start:prod": "WARTHOG_ENV=production yarn dotenv:generate && node dist\/src\/index.js"/a \
-   "start:prod:with-instrumentation": "export OTEL_APPLICATION=query-node; WARTHOG_ENV=production yarn dotenv:generate && node --require @joystream/opentelemetry dist/src/index.js",' ./generated/graphql-server/package.json
+   "start:prod:with-instrumentation": "export OTEL_APPLICATION=query-node; WARTHOG_ENV=production yarn dotenv:generate && node --require @joystream/opentelemetry dist/src/index.js", \
+   "start:prod:pm2": "WARTHOG_ENV=production yarn dotenv:generate && yarn pm2-runtime start dist/src/index.js -i 1 --name graphql-server",' ./generated/graphql-server/package.json
 
 # Add @joystream/opentelemetry dependency symlink, as it is not specified in generated/graphql-server/package.json dependencies
 mkdir -p ./generated/graphql-server/node_modules/@joystream

query-node/mappings/package.json

@@ -1,6 +1,6 @@
 {
   "name": "query-node-mappings",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "Mappings for hydra-processor",
   "main": "lib/src/index.js",
   "license": "MIT",
@@ -19,7 +19,7 @@
     "@apollo/client": "^3.2.5",
     "@joystream/hydra-common": "5.0.0-alpha.4",
     "@joystream/hydra-db-utils": "5.0.0-alpha.4",
-    "@joystream/js": "^1.4.0",
+    "@joystream/js": "^1.5.0",
     "@joystream/warthog": "^2.41.9",
     "@polkadot/types": "8.9.1"
   },

query-node/mappings/src/membership.ts

@@ -7,6 +7,7 @@ import { DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
 import { isSet } from '@joystream/metadata-protobuf/utils'
 import { MemberId } from '@joystream/types/primitives'
 import { AccountId32, Balance } from '@polkadot/types/interfaces'
+import { Bytes } from '@polkadot/types'
 import {
   PalletMembershipBuyMembershipParameters as BuyMembershipParameters,
   PalletMembershipCreateMemberParameters as CreateMemberParameters,
@@ -179,6 +180,12 @@ async function saveMembershipMetadata(
   return metadataEntity
 }
 
+function setMemberHandle(member: Membership, handle: Bytes): Membership {
+  member.handle = bytesToString(handle)
+  member.handleRaw = handle.toHex()
+  return member
+}
+
 async function createNewMemberFromParams(
   store: DatabaseManager,
   memberId: MemberId,
@@ -190,11 +197,11 @@ async function createNewMemberFromParams(
   const { rootAccount, controllerAccount, handle, metadata: metadataBytes } = params
   const metadata = deserializeMetadata(MembershipMetadata, metadataBytes)
 
+  const memberHandle = 'unwrap' in handle ? handle.unwrap() : handle
   const member = new Membership({
     id: memberId.toString(),
     rootAccount: rootAccount.toString(),
     controllerAccount: controllerAccount.toString(),
-    handle: bytesToString('unwrap' in handle ? handle.unwrap() : handle),
     metadata: await saveMembershipMetadata(store, undefined, metadata),
     entry: entryMethod,
     referredBy:
@@ -222,6 +229,8 @@ async function createNewMemberFromParams(
     councilMembers: [],
   })
 
+  setMemberHandle(member, memberHandle)
+
   await store.save<Membership>(member)
 
   member.metadata.externalResources = await saveMembershipExternalResources(
@@ -347,7 +356,7 @@ export async function members_MemberProfileUpdated({ store, event }: EventContex
   }
 
   if (newHandle.isSome) {
-    member.handle = bytesToString(newHandle.unwrap())
+    setMemberHandle(member, newHandle.unwrap())
   }
 
   await store.save<MemberMetadata>(member.metadata)
@@ -372,6 +381,7 @@ export async function members_MemberProfileUpdated({ store, event }: EventContex
     ...genericEventFields(event),
     member: member,
     newHandle: member.handle,
+    newHandleRaw: member.handleRaw,
     newMetadata: await saveMembershipMetadata(store, member),
   })
 

query-node/package.json

@@ -1,6 +1,6 @@
 {
   "name": "query-node-root",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "GraphQL server and mappings. Generated with ♥ by Hydra-CLI",
   "scripts": {
     "build": "./build.sh",
@@ -15,6 +15,7 @@
     "query-node:build": "yarn workspace query-node build",
     "query-node:start:dev": "yarn workspace query-node start:dev",
     "query-node:start:prod": "yarn workspace query-node start:prod",
+    "query-node:start:prod:pm2": "yarn workspace query-node start:prod:pm2",
     "query-node:start:prod:with-instrumentation": "yarn workspace query-node start:prod:with-instrumentation",
     "query-node:configure": "yarn workspace query-node config:dev",
     "db:up": "yarn docker:db:up",
@@ -41,6 +42,7 @@
     "@joystream/hydra-processor": "5.0.0-alpha.4",
     "@types/bn.js": "^5.1.0",
     "bn.js": "^5.2.1",
+    "pm2": "^5.3.0",
     "tslib": "^2.0.0"
   },
   "volta": {

query-node/schemas/membership.graphql

@@ -93,8 +93,11 @@ type Membership @entity {
   "MemberId: runtime identifier for a user"
   id: ID!
 
-  "The unique handle chosen by member"
-  handle: String! @unique @fulltext(query: "membersByHandle")
+  "The unique handle chosen by member as raw hex representation"
+  handle_raw: String! @unique
+
+  "The unique handle chosen by member as utf-8"
+  handle: String! @fulltext(query: "membersByHandle")
 
   "Member's metadata"
   metadata: MemberMetadata!

query-node/schemas/membershipEvents.graphql

@@ -177,9 +177,12 @@ type MemberProfileUpdatedEvent implements Event @entity {
   "Membership being updated."
   member: Membership!
 
-  "New member handle. Null means no new value was provided."
+  "New member handle (utf-8 string). Null means no new value was provided."
   newHandle: String
 
+  "New member handle (raw hex string). Null means no new value was provided."
+  newHandleRaw: String
+
   "New member metadata. (empty values inside metadata mean no new value was provided)"
   newMetadata: MemberMetadata!
 }