Skip to content

Commit

Permalink
Add support for migrating subgraph graphql schemas
Browse files Browse the repository at this point in the history
  • Loading branch information
stwiname committed Jul 24, 2024
1 parent fe7e496 commit 550d3d1
Show file tree
Hide file tree
Showing 4 changed files with 386 additions and 5 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`Subgraph Graphql Schema migration correctly migrates a schema 1`] = `
""""The Example entity has an example of various fields"""
type ExampleEntity @entity @fullText(fields: ["optionalString", "requiredString", "optionalStringList"], language: "english") {
"""The id is required and a unique identifier of the entity"""
id: ID!
optionalBoolean: Boolean
requiredBoolean: Boolean!
optionalBooleanList: [Boolean!]
requiredBooleanList: [Boolean!]!
optionalString: String
requiredString: String!
optionalStringList: [String!]
requiredStringList: [String!]!
optionalBytes: Bytes
requiredBytes: Bytes!
optionalBytesList: [Bytes!]
requiredBytesList: [Bytes!]!
optionalInt: Int
requiredInt: Int!
optionalIntList: [Int!]
requiredIntList: [Int!]!
optionalInt8: Int
requiredInt8: Int!
optionalInt8List: [Int!]
requiredInt8List: [Int!]!
optionalBigInt: BigInt
requiredBigInt: BigInt!
optionalBigIntList: [BigInt!]
requiredBigIntList: [BigInt!]!
optionalBigDecimal: BigDecimal
requiredBigDecimal: BigDecimal!
optionalBigDecimalList: [BigDecimal!]
requiredBigDecimalList: [BigDecimal!]!
optionalTimestamp: Date
requiredTimestamp: Date!
optionalTimestampList: [Date!]
requiredTimestampList: [Date!]!
optionalReference: OtherEntity
requiredReference: OtherEntity!
optionalReferenceList: [OtherEntity!]
requiredReferenceList: [OtherEntity!]!
derivedEntity: [FooEntity!]! @derivedFrom(field: "example")
}
type OtherEntity @entity {
id: ID!
}
type FooEntity @entity {
id: ID!
example: ExampleEntity!
}
"
`;
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors
// SPDX-License-Identifier: GPL-3.0

import {migrateSchemaFromString} from './migrate-schema.controller';

describe('Subgraph Graphql Schema migration', () => {
it('correctly migrates a schema', () => {
// This schema is a mix of schemas from https://github.com/graphprotocol/graph-tooling/tree/main/examples as well as the docs
const schema = `
type _Schema_
@fulltext(
name: "exampleSearch"
language: en
algorithm: rank
include: [{ entity: "ExampleEntity", fields: [{ name: "optionalString" }, { name: "requiredString" }, { name: "optionalStringList" }] }]
)
"""
The Example entity has an example of various fields
"""
type ExampleEntity @entity {
"""
The id is required and a unique identifier of the entity
"""
id: ID!
optionalBoolean: Boolean
requiredBoolean: Boolean!
optionalBooleanList: [Boolean!]
requiredBooleanList: [Boolean!]!
optionalString: String
requiredString: String!
optionalStringList: [String!]
requiredStringList: [String!]!
optionalBytes: Bytes
requiredBytes: Bytes!
optionalBytesList: [Bytes!]
requiredBytesList: [Bytes!]!
optionalInt: Int
requiredInt: Int!
optionalIntList: [Int!]
requiredIntList: [Int!]!
optionalInt8: Int8
requiredInt8: Int8!
optionalInt8List: [Int8!]
requiredInt8List: [Int8!]!
optionalBigInt: BigInt
requiredBigInt: BigInt!
optionalBigIntList: [BigInt!]
requiredBigIntList: [BigInt!]!
optionalBigDecimal: BigDecimal
requiredBigDecimal: BigDecimal!
optionalBigDecimalList: [BigDecimal!]
requiredBigDecimalList: [BigDecimal!]!
optionalTimestamp: Timestamp
requiredTimestamp: Timestamp!
optionalTimestampList: [Timestamp!]
requiredTimestampList: [Timestamp!]!
optionalReference: OtherEntity
requiredReference: OtherEntity!
optionalReferenceList: [OtherEntity!]
requiredReferenceList: [OtherEntity!]!
derivedEntity: [FooEntity!]! @derivedFrom(field: "example")
}
type OtherEntity @entity(immutable: true, timeseries: true) {
id: ID!
}
type FooEntity @entity {
id: Bytes!
example: ExampleEntity!
}
type Stats @aggregation(intervals: ["hour", "day"], source: "Block") {
# The id; it is the id of one of the data points that were aggregated into
# this bucket, but which one is undefined and should not be relied on
id: Int8!
# The timestamp of the bucket is always the timestamp of the beginning of
# the interval
timestamp: Timestamp!
# The aggregates
# A count of the number of data points that went into this bucket
count: Int! @aggregate(fn: "count")
# The max(number) of the data points for this bucket
max: BigDecimal! @aggregate(fn: "max", arg: "number")
min: BigDecimal! @aggregate(fn: "min", arg: "number")
# sum_{i=n}^m i = (m - n + 1) * (n + m) / 2
sum: BigInt! @aggregate(fn: "sum", arg: "number")
first: Int! @aggregate(fn: "first", arg: "number")
last: Int! @aggregate(fn: "last", arg: "number")
maxGas: BigInt! @aggregate(fn: "max", arg: "gasUsed")
maxDifficulty: BigInt! @aggregate(fn: "max", arg: "difficulty")
}
`;

const migratedSchema = migrateSchemaFromString(schema);

console.log('Migrated schema', migratedSchema);

expect(migratedSchema).toMatchSnapshot();
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,222 @@
// SPDX-License-Identifier: GPL-3.0

import fs from 'fs';
import {
Source,
parse,
print,
TypeNode,
NamedTypeNode,
Kind,
StringValueNode,
ValueNode,
ObjectValueNode,
ObjectTypeDefinitionNode,
DefinitionNode,
DirectiveNode,
ListValueNode,
} from 'graphql';

export async function migrateSchema(subgraphSchemaPath: string, subqlSchemaPath: string) {
// Extra schema definitions used by the graph
// const subgraphScalars = gql`
// scalar Int8
// scalar Timestamp
// `;

// const subgraphDirectives = gql`
// type FulltextInclude {
// name: String!
// }

// directive @fulltext(name: String!, language: String!, algorithm: String!, include: [FulltextInclude!]!) on OBJECT
// `;

export async function migrateSchema(subgraphSchemaPath: string, subqlSchemaPath: string): Promise<void> {
await fs.promises.rm(subqlSchemaPath, {force: true});
// copy over schema
fs.copyFileSync(subgraphSchemaPath, subqlSchemaPath);

const file = await fs.promises.readFile(subgraphSchemaPath);
const output = migrateSchemaFromString(file.toString('utf8'));
await fs.promises.writeFile(subqlSchemaPath, output);
console.log(
`* schema.graphql have been copied over, they will need to be updated to work with SubQuery. See our documentation for more details https://academy.subquery.network/build/graphql.html`
`* schema.graphql has been migrated. If there are any issues see our documentation for more details https://academy.subquery.network/build/graphql.html`
);
}

// TODO work around readonly props
export function migrateSchemaFromString(input: string): string {
const src = new Source(input);
const doc = parse(src);

const updated = doc.definitions
.filter((definition) => {
if (isObjectType(definition)) {
const aggregationDirective = definition.directives?.find((d) => d.name.value === 'aggregation');
if (aggregationDirective) {
console.warn(`The Aggregation directive is not supported. type="${definition.name.value}" has been removed`);
return false;
}
}

return true;
})
.map((definition) => {
if (isObjectType(definition)) {
// Convert fulltext search directives
if (definition.name.value === '_Schema_') {
definition.directives?.forEach((directive) => {
convertFulltextDirective(directive, doc.definitions);
});
// No mutations to the global schema
return definition;
}

// Map field types to known types
(definition as any).fields = definition.fields?.map((field) => {
modifyTypeNode(field.type, (type) => {
// SubQuery only supports ID type for id
if (field.name.value === 'id') {
(type.name as any).value = 'ID';
}
if (type.name.value === 'Int8') {
(type.name as any).value = 'Int';
}
if (type.name.value === 'Timestamp') {
(type.name as any).value = 'Date';
}
return type;
});
return field;
});

// Remove unsupported arguments from entity directive
const entityDirective = definition.directives?.find((d) => d.name.value === 'entity');
if (!entityDirective) throw new Error('Object is missing entity directive');

(entityDirective.arguments as any) = entityDirective.arguments?.filter((arg) => {
if (arg.name.value === 'immutable') {
console.warn(`Immutable option is not supported. Removing from entity="${definition.name.value}"`);
return false;
}
if (arg.name.value === 'timeseries') {
console.warn(`Timeseries option is not supported. Removing from entity="${definition.name.value}"`);
return false;
}
return true;
});
}

return definition;
})
// Remove the _Schema_ type
.filter((def) => !(isObjectType(def) && def.name.value === '_Schema_'));

return print({...doc, definitions: updated});
}

function convertFulltextDirective(directive: DirectiveNode, definitions: readonly DefinitionNode[]) {
if (directive.name.value !== 'fulltext') return;
// TODO should add runtime check for StringValueNode
const name = (directive.arguments?.find((arg) => arg.name.value === 'name')?.value as StringValueNode)?.value;

const includeOpt = directive.arguments?.find((arg) => arg.name.value === 'include');
if (!includeOpt) throw new Error("Expected fulltext directive to have an 'include' argument");
if (includeOpt.value.kind !== Kind.LIST) throw new Error('Expected include argument to be a list');
if (includeOpt.value.values.length !== 1) {
throw new Error(`SubQuery only supports fulltext search on a single entity. name=${name}`);
}

const includeParams = includeOpt.value.values[0];

// Get the entity name
if (!isObjectValueNode(includeParams)) throw new Error(`Expected object value, received ${includeParams.kind}`);
const entityName = includeParams.fields.find((f) => f.name.value === 'entity')?.value;
if (!entityName || !isStringValueNode(entityName)) throw new Error('Entity name is invalid');

// Get the entity fields
const fields = includeParams.fields.find((f) => f.name.value === 'fields')?.value;
if (!fields) throw new Error('Unable to find fields for fulltext search');
if (!isListValueNode(fields)) throw new Error('Expected fields to be a list');
const fieldNames = fields.values.map((field) => {
if (!isObjectValueNode(field)) throw new Error('Field is invalid');

const nameField = field.fields.find((f) => f.name.value === 'name');
if (!nameField) throw new Error('Fields field is missing name');
return nameField.value;
});
if (!fieldNames.length) throw new Error('Fulltext search requires at least one field');

// Find the entity to add the directive
const entity = findEntity(definitions, entityName.value);
if (!entity) throw new Error(`Unable to find entity ${entityName.value} for fulltext search`);

// Add the fulltext directive to the entity
(entity.directives as any).push(makeFulltextDirective(fieldNames.map((f) => (f as any).value)));
}

// Drills down to the inner type and runs the modFn on it, this runs in place
function modifyTypeNode(type: TypeNode, modFn: (innerType: NamedTypeNode) => NamedTypeNode): TypeNode {
if (type.kind === Kind.NON_NULL_TYPE || type.kind === Kind.LIST_TYPE) {
return modifyTypeNode(type.type, modFn);
}
return modFn(type);
}

function findEntity(definitions: readonly DefinitionNode[], name: string): ObjectTypeDefinitionNode | undefined {
// Cast can be removed with newver version of typescript
return definitions.find((def) => isObjectType(def) && def.name.value === name) as
| ObjectTypeDefinitionNode
| undefined;
}

function makeFulltextDirective(fields: string[], language = 'english'): DirectiveNode {
return {
kind: Kind.DIRECTIVE,
name: {
kind: Kind.NAME,
value: 'fullText',
},
arguments: [
{
kind: Kind.ARGUMENT,
name: {
kind: Kind.NAME,
value: 'fields',
},
value: {
kind: Kind.LIST,
values: fields.map((field) => ({
kind: Kind.STRING,
value: field,
})),
},
},
{
kind: Kind.ARGUMENT,
name: {
kind: Kind.NAME,
value: 'language',
},
value: {
kind: Kind.STRING,
value: language,
},
},
],
} satisfies DirectiveNode;
}

function isObjectType(node: DefinitionNode): node is ObjectTypeDefinitionNode {
return node.kind === Kind.OBJECT_TYPE_DEFINITION;
}

function isObjectValueNode(node: ValueNode): node is ObjectValueNode {
return node.kind === Kind.OBJECT;
}

function isListValueNode(node: ValueNode): node is ListValueNode {
return node.kind === Kind.LIST;
}

function isStringValueNode(node: ValueNode): node is StringValueNode {
return node.kind === Kind.STRING;
}
Loading

0 comments on commit 550d3d1

Please sign in to comment.