Merge pull request #4856 from zeeshanakram3/qn_mappings_review

QN mappings review & fixes

distributor-node/src/services/networking/query-node/schema.graphql

@@ -2108,7 +2108,7 @@ type Proposal implements BaseGraphQLObject {
   proposalexecutedeventproposal: [ProposalExecutedEvent!]
 }
 
-union ProposalDetails = SignalProposalDetails | RuntimeUpgradeProposalDetails | FundingRequestProposalDetails | SetMaxValidatorCountProposalDetails | CreateWorkingGroupLeadOpeningProposalDetails | FillWorkingGroupLeadOpeningProposalDetails | UpdateWorkingGroupBudgetProposalDetails | DecreaseWorkingGroupLeadStakeProposalDetails | SlashWorkingGroupLeadProposalDetails | SetWorkingGroupLeadRewardProposalDetails | TerminateWorkingGroupLeadProposalDetails | AmendConstitutionProposalDetails | CancelWorkingGroupLeadOpeningProposalDetails | SetMembershipPriceProposalDetails | SetCouncilBudgetIncrementProposalDetails | SetCouncilorRewardProposalDetails | SetInitialInvitationBalanceProposalDetails | SetInitialInvitationCountProposalDetails | SetMembershipLeadInvitationQuotaProposalDetails | SetReferralCutProposalDetails | VetoProposalDetails | UpdateChannelPayoutsProposalDetails
+union ProposalDetails = SignalProposalDetails | RuntimeUpgradeProposalDetails | FundingRequestProposalDetails | SetMaxValidatorCountProposalDetails | CreateWorkingGroupLeadOpeningProposalDetails | FillWorkingGroupLeadOpeningProposalDetails | UpdateWorkingGroupBudgetProposalDetails | DecreaseWorkingGroupLeadStakeProposalDetails | SlashWorkingGroupLeadProposalDetails | SetWorkingGroupLeadRewardProposalDetails | TerminateWorkingGroupLeadProposalDetails | AmendConstitutionProposalDetails | CancelWorkingGroupLeadOpeningProposalDetails | SetMembershipPriceProposalDetails | SetCouncilBudgetIncrementProposalDetails | SetCouncilorRewardProposalDetails | SetInitialInvitationBalanceProposalDetails | SetInitialInvitationCountProposalDetails | SetMembershipLeadInvitationQuotaProposalDetails | SetReferralCutProposalDetails | VetoProposalDetails | UpdateChannelPayoutsProposalDetails | UpdateGlobalNftLimitProposalDetails
 
 union ProposalStatus = ProposalStatusDeciding | ProposalStatusGracing | ProposalStatusDormant | ProposalStatusVetoed | ProposalStatusExecuted | ProposalStatusExecutionFailed | ProposalStatusSlashed | ProposalStatusRejected | ProposalStatusExpired | ProposalStatusCancelled | ProposalStatusCanceledByRuntime
 
@@ -2826,6 +2826,14 @@ type UpdateChannelPayoutsProposalDetails {
   payloadHash: String
 }
 
+type UpdateGlobalNftLimitProposalDetails {
+  """New daily NFT limit set in the proposal (if any)"""
+  newDailyNftLimit: Int
+
+  """New weekly NFT limit set in the proposal (if any)"""
+  newWeeklyNftLimit: Int
+}
+
 type UpdateWorkingGroupBudgetProposalDetails {
   """
   Amount to increase / decrease the working group budget by (will be decudted from / appended to council budget accordingly)

eslint-local-rules.js

@@ -0,0 +1,24 @@
+module.exports = {
+  'no-throw': {
+    meta: {
+      type: 'problem',
+      docs: {
+        description: 'disallow the use of throw keyword',
+        category: 'Possible Errors',
+        recommended: true,
+      },
+      schema: [],
+    },
+
+    create: function (context) {
+      return {
+        ThrowStatement(node) {
+          context.report({
+            node: node,
+            message: "The use of 'throw' keyword is not allowed.",
+          })
+        },
+      }
+    },
+  },
+}

query-node/.prettierignore

@@ -0,0 +1 @@
+chain-metadata/

query-node/CHANGELOG.md

@@ -1,3 +1,9 @@
+### 1.7.0
+
+- Refactor of mappings for more better handling of error cases. [#4856](https://github.com/Joystream/joystream/pull/4856)
+- Bug fix [#4855](https://github.com/Joystream/joystream/issues/4855)
+- Add support for UpdateGlobalNftLimit proposal. 
+
 ### 1.6.0
 
 - Store membership handles both as utf-8 string and raw bytes - [#4950](https://github.com/Joystream/joystream/pull/4950)

query-node/mappings/.eslintrc.js

@@ -3,7 +3,9 @@ module.exports = {
   env: {
     node: true,
   },
+  plugins: ['eslint-plugin-local-rules'],
   rules: {
+    'local-rules/no-throw': 'error',
     '@typescript-eslint/naming-convention': 'off',
     // TODO: Remove all the rules below, they seem quite useful
     '@typescript-eslint/no-explicit-any': 'off',

query-node/mappings/package.json

@@ -1,6 +1,6 @@
 {
   "name": "query-node-mappings",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "description": "Mappings for hydra-processor",
   "main": "lib/src/index.js",
   "license": "MIT",
@@ -16,14 +16,15 @@
     "bootstrap-data:fetch": "yarn bootstrap-data:fetch:members && yarn bootstrap-data:fetch:workingGroups && yarn bootstrap-data:fetch:categories"
   },
   "dependencies": {
-    "@polkadot/types": "8.9.1",
+    "@apollo/client": "^3.2.5",
     "@joystream/hydra-common": "5.0.0-alpha.4",
     "@joystream/hydra-db-utils": "5.0.0-alpha.4",
-    "@joystream/warthog": "^2.41.9",
     "@joystream/js": "^1.5.0",
-    "@apollo/client": "^3.2.5"
+    "@joystream/warthog": "^2.41.9",
+    "@polkadot/types": "8.9.1"
   },
   "devDependencies": {
+    "eslint-plugin-local-rules": "2.0.0",
     "prettier": "^2.2.1",
     "ts-node": "^10.2.1",
     "typescript": "^4.4.3"

query-node/mappings/src/common.ts

@@ -1,22 +1,29 @@
-import { DatabaseManager, SubstrateEvent, FindOneOptions } from '@joystream/hydra-common'
-import { Bytes, Option } from '@polkadot/types'
-import { Codec } from '@polkadot/types/types'
+import {
+  DatabaseManager,
+  FindOneOptions,
+  FindOptionsOrderValue,
+  FindOptionsWhere,
+  SubstrateEvent,
+} from '@joystream/hydra-common'
+import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
+import { metaToObject } from '@joystream/metadata-protobuf/utils'
 import { MemberId, WorkerId } from '@joystream/types/primitives'
+import { BaseModel } from '@joystream/warthog'
+import { Bytes, Option } from '@polkadot/types'
 import { PalletCommonWorkingGroupIterableEnumsWorkingGroup as WGType } from '@polkadot/types/lookup'
+import { Codec } from '@polkadot/types/types'
+import BN from 'bn.js'
 import {
-  Worker,
   Event,
-  Network,
-  WorkingGroup as WGEntity,
-  MetaprotocolTransactionStatusEvent,
+  Membership,
   MetaprotocolTransactionErrored,
+  MetaprotocolTransactionStatusEvent,
   MetaprotocolTransactionSuccessful,
-  Membership,
+  Network,
+  WorkingGroup as WGEntity,
+  Worker,
 } from 'query-node/dist/model'
-import { BaseModel } from '@joystream/warthog'
-import { metaToObject } from '@joystream/metadata-protobuf/utils'
-import { AnyMetadataClass, DecodedMetadataObject } from '@joystream/metadata-protobuf/types'
-import BN from 'bn.js'
+import { In } from 'typeorm'
 
 export const CURRENT_NETWORK = Network.OLYMPIA
 
@@ -88,19 +95,20 @@ export function inconsistentState(extraInfo: string, data?: unknown): never {
   // log error
   logger.error(errorMessage, data)
 
+  // eslint-disable-next-line local-rules/no-throw
   throw errorMessage
 }
 
-/*
-  Reports that insurmountable unexpected data has been encountered and throws an exception.
-*/
-export function unexpectedData(extraInfo: string, data?: unknown): never {
-  const errorMessage = 'Unexpected data: ' + extraInfo
+/**
+ * Reports an unimplemented mapping/variant error for which a runtime implementation logic exists and is not filtered.
+ */
+export function unimplementedError(extraInfo: string, data?: unknown): never {
+  const errorMessage = 'unimplemented error: ' + extraInfo
 
   // log error
   logger.error(errorMessage, data)
 
-  throw errorMessage
+  process.exit(1)
 }
 
 /*
@@ -124,7 +132,7 @@ export function deserializeMetadata<T>(
     const message = metadataType.decode(metadataBytes.toU8a(true))
     Object.keys(message).forEach((key) => {
       if (key in message && typeof message[key] === 'string') {
-        message[key] = perpareString(message[key])
+        message[key] = prepareString(message[key])
       }
     })
     return metaToObject(metadataType, message)
@@ -145,7 +153,7 @@ export function bytesToString(b: Bytes): string {
   )
 }
 
-export function perpareString(s: string): string {
+export function prepareString(s: string): string {
   // eslint-disable-next-line no-control-regex
   return s.replace(/\u0000/g, '')
 }
@@ -211,55 +219,33 @@ export function getWorkingGroupModuleName(group: WGType): WorkingGroupModuleName
     return 'operationsWorkingGroupGamma'
   }
 
-  unexpectedData('Unsupported working group encountered:', group.type)
+  unimplementedError('Unsupported working group encountered:', group.type)
 }
 
-export async function getWorkingGroupByName(
+export async function getWorkingGroupByNameOrFail(
   store: DatabaseManager,
   name: WorkingGroupModuleName,
-  relations: string[] = []
+  relations: RelationsArr<WGEntity> = []
 ): Promise<WGEntity> {
-  const group = await store.get(WGEntity, { where: { name }, relations })
-  if (!group) {
-    return inconsistentState(`Working group ${name} not found!`)
-  }
-  return group
+  return getOneByOrFail(store, WGEntity, { name }, relations)
 }
 
-export async function getMemberById(
+export async function getMembershipById(
   store: DatabaseManager,
   id: MemberId,
-  relations: string[] = []
+  relations: RelationsArr<Membership> = []
 ): Promise<Membership> {
-  const member = await store.get(Membership, { where: { id: id.toString() }, relations })
-  if (!member) {
-    throw new Error(`Member(${id}) not found`)
-  }
-  return member
-}
-
-export async function getWorkingGroupLead(store: DatabaseManager, groupName: WorkingGroupModuleName) {
-  const lead = await store.get(Worker, { where: { groupId: groupName, isLead: true, isActive: true } })
-  if (!lead) {
-    return inconsistentState(`Couldn't find an active lead for ${groupName}`)
-  }
-
-  return lead
+  return getByIdOrFail(store, Membership, id.toString(), relations)
 }
 
-export async function getWorker(
+export async function getWorkerOrFail(
   store: DatabaseManager,
   groupName: WorkingGroupModuleName,
   runtimeId: WorkerId | number,
-  relations: string[] = []
+  relations: RelationsArr<Worker> = []
 ): Promise<Worker> {
   const workerDbId = `${groupName}-${runtimeId}`
-  const worker = await store.get(Worker, { where: { id: workerDbId }, relations })
-  if (!worker) {
-    return inconsistentState(`Expected worker not found by id ${workerDbId}`)
-  }
-
-  return worker
+  return getByIdOrFail(store, Worker, workerDbId, relations)
 }
 
 type EntityClass<T extends BaseModel> = {
@@ -277,15 +263,93 @@ export async function getById<T extends BaseModel>(
   entityClass: EntityClass<T>,
   id: string,
   relations?: RelationsArr<T>
+): Promise<T | undefined> {
+  return store.get(entityClass, { where: { id }, relations } as FindOneOptions<T>)
+}
+
+export async function getByIdOrFail<T extends BaseModel>(
+  store: DatabaseManager,
+  entityClass: EntityClass<T>,
+  id: string,
+  relations?: RelationsArr<T>,
+  errMessage?: string
+): Promise<T> {
+  const result = await getById<T>(store, entityClass, id, relations)
+  if (!result) {
+    // eslint-disable-next-line local-rules/no-throw
+    throw new Error(`Expected "${entityClass.name}" not found by ID: ${id} ${errMessage ? `- ${errMessage}` : ''}`)
+  }
+
+  return result
+}
+
+export async function getOneBy<T extends BaseModel>(
+  store: DatabaseManager,
+  entityClass: EntityClass<T>,
+  where?: FindOptionsWhere<T>,
+  relations?: RelationsArr<T>,
+  order?: Partial<{ [K in keyof T]: FindOptionsOrderValue }>
+): Promise<T | undefined> {
+  return store.get(entityClass, { where, relations, order } as FindOneOptions<T>)
+}
+
+/**
+ * Retrieves a entity by any field(s) or throws an error if not found
+ */
+export async function getOneByOrFail<T extends BaseModel>(
+  store: DatabaseManager,
+  entityClass: EntityClass<T>,
+  where?: FindOptionsWhere<T>,
+  relations?: RelationsArr<T>,
+  order?: Partial<{ [K in keyof T]: FindOptionsOrderValue }>,
+  errMessage?: string
 ): Promise<T> {
-  const result = await store.get(entityClass, { where: { id }, relations } as FindOneOptions<T>)
+  const result = await getOneBy(store, entityClass, where, relations, order)
   if (!result) {
-    throw new Error(`Expected ${entityClass.name} not found by ID: ${id}`)
+    // eslint-disable-next-line local-rules/no-throw
+    throw new Error(
+      `Expected "${entityClass.name}" not found by filter: ${JSON.stringify({
+        where,
+        relations,
+        order,
+      })} ${errMessage ? `- ${errMessage}` : ''}`
+    )
   }
 
   return result
 }
 
+export async function getManyBy<T extends BaseModel>(
+  store: DatabaseManager,
+  entityClass: EntityClass<T>,
+  entityIds: string[],
+  where?: FindOptionsWhere<T>,
+  relations?: RelationsArr<T>
+): Promise<T[]> {
+  return store.getMany(entityClass, { where: { id: In(entityIds), ...where }, relations } as FindOneOptions<T>)
+}
+
+export async function getManyByOrFail<T extends BaseModel>(
+  store: DatabaseManager,
+  entityClass: EntityClass<T>,
+  entityIds: string[],
+  where?: FindOptionsWhere<T>,
+  relations?: RelationsArr<T>,
+  errMessage?: string
+): Promise<T[]> {
+  const entities = await getManyBy(store, entityClass, entityIds, where, relations)
+  const loadedEntityIds = entities.map((item) => item.id)
+  if (loadedEntityIds.length !== entityIds.length) {
+    const missingIds = entityIds.filter((item) => !loadedEntityIds.includes(item))
+
+    // eslint-disable-next-line local-rules/no-throw
+    throw new Error(
+      `"${entityClass.name}" missing records for following IDs: ${missingIds} ${errMessage ? `- ${errMessage}` : ''}`
+    )
+  }
+  return entities
+}
+
 export function deterministicEntityId(createdInEvent: SubstrateEvent, additionalIdentifier?: string | number): string {
   return (
     `${createdInEvent.blockNumber}-${createdInEvent.indexInBlock}` +
@@ -332,10 +396,10 @@ export async function saveMetaprotocolTransactionSuccessful(
 export async function saveMetaprotocolTransactionErrored(
   store: DatabaseManager,
   event: SubstrateEvent,
-  message: string
+  error: MetaprotocolTxError
 ): Promise<void> {
   const status = new MetaprotocolTransactionErrored()
-  status.message = message
+  status.message = error
 
   const metaprotocolTransaction = new MetaprotocolTransactionStatusEvent({
     ...genericEventFields(event),
@@ -344,3 +408,30 @@ export async function saveMetaprotocolTransactionErrored(
 
   await store.save(metaprotocolTransaction)
 }
+
+export enum MetaprotocolTxError {
+  InvalidMetadata = 'InvalidMetadata',
+
+  // App errors
+  AppAlreadyExists = 'AppAlreadyExists',
+  AppNotFound = 'AppNotFound',
+  InvalidAppOwnerMember = 'InvalidAppOwnerMember',
+
+  // Video errors
+  VideoNotFound = 'VideoNotFound',
+  VideoNotFoundInChannel = 'VideoNotFoundInChannel',
+  VideoReactionsDisabled = 'VideoReactionsDisabled',
+
+  // Comment errors
+  CommentSectionDisabled = 'CommentSectionDisabled',
+  CommentNotFound = 'CommentNotFound',
+  ParentCommentNotFound = 'ParentCommentNotFound',
+  InvalidCommentAuthor = 'InvalidCommentAuthor',
+
+  // Membership error
+  MemberNotFound = 'MemberNotFound',
+  MemberBannedFromChannel = 'MemberBannedFromChannel',
+
+  // Channel errors
+  InvalidChannelRewardAccount = 'InvalidChannelRewardAccount',
+}