From bc316a782c439ab2f2af6723ae1f0a6dd56abe8b Mon Sep 17 00:00:00 2001 From: Daniel Chambers Date: Mon, 19 Feb 2024 15:11:43 +1100 Subject: [PATCH] Support for the Connector Deployment Spec (#18) --- changelog.md | 17 ++++ package-lock.json | 20 ++--- package.json | 10 +-- src/configuration-server.ts | 169 ------------------------------------ src/connector.ts | 38 ++------ src/index.ts | 60 ++++--------- src/server.ts | 34 +------- 7 files changed, 56 insertions(+), 292 deletions(-) delete mode 100644 src/configuration-server.ts diff --git a/changelog.md b/changelog.md index fa53aa3..96e00b8 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,22 @@ # Changelog +## 4.0.0 +Breaking change: support for the [Connector Deployment Spec](https://github.com/hasura/ndc-hub/blob/main/rfcs/0000-deployment.md). + +- The connector configuration server has been removed +- The way configuration is handled on `Connector` interface has changed + - `getRawConfigurationSchema`, `makeEmptyConfiguration`, `updateConfiguration` have been removed. + - `parseConfiguation` replaces `validateRawConfiguration`, and is given the directory path in which the connector's configuration files can be found + - The `RawConfiguration` type parameter has been removed +- The default port has changed from 8100 to 8080 +- The command line arguments passed to the `serve` command have changed: + - The `--configuration` argument now takes the connector's configuration directory. Its associated environment variable is now `HASURA_CONFIGURATION_DIRECTORY` + - The `--otlp_endpoint` argument has been renamed to `--otlp-endpoint` and its environment variable is now `OTEL_EXPORTER_OTLP_ENDPOINT` + - The `PORT` environment variable has changed to `HASURA_CONNECTOR_PORT` + - The `SERVICE_TOKEN_SECRET` environment variable has changed to `HASURA_SERVICE_TOKEN_SECRET` + - The `LOG_LEVEL` environment variable has changed to `HASURA_LOG_LEVEL` + - The `PRETTY_PRINT_LOGS` environment variable has changed to `HASURA_PRETTY_PRINT_LOGS` + ## 3.0.0 Breaking change: support for the [v0.1.0-rc.15 of NDC Spec](https://github.com/hasura/ndc-spec/compare/v0.1.0-rc.14...v0.1.0-rc.15). diff --git a/package-lock.json b/package-lock.json index a3d29be..b5923da 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,13 @@ { "name": "@hasura/ndc-sdk-typescript", - "version": "3.0.0", + "version": "4.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@hasura/ndc-sdk-typescript", - "version": "3.0.0", - "license": "ISC", + "version": "4.0.0", + "license": "Apache-2.0", "dependencies": { "@json-schema-tools/meta-schema": "^1.7.0", "commander": "^11.0.0", @@ -17,7 +17,7 @@ "devDependencies": { "@types/node": "^20.6.0", "json-schema-to-typescript": "^13.1.1", - "typescript": "^5.2.2" + "typescript": "^5.3.3" } }, "node_modules/@bcherny/json-schema-ref-parser": { @@ -1242,9 +1242,9 @@ "dev": true }, "node_modules/typescript": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", - "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -2246,9 +2246,9 @@ "dev": true }, "typescript": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.2.2.tgz", - "integrity": "sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", "dev": true }, "uri-js": { diff --git a/package.json b/package.json index 519f8e8..4590122 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@hasura/ndc-sdk-typescript", - "version": "3.0.0", + "version": "4.0.0", "description": "", "main": "./dist/index.js", "types": "./dist/index.d.ts", @@ -9,9 +9,9 @@ "regenerate-schema": "./typegen/regenerate-schema.sh", "build": "tsc" }, - "keywords": [], - "author": "", - "license": "ISC", + "keywords": ["hasura", "ndc"], + "author": "Hasura", + "license": "Apache-2.0", "dependencies": { "@json-schema-tools/meta-schema": "^1.7.0", "commander": "^11.0.0", @@ -21,6 +21,6 @@ "devDependencies": { "@types/node": "^20.6.0", "json-schema-to-typescript": "^13.1.1", - "typescript": "^5.2.2" + "typescript": "^5.3.3" } } diff --git a/src/configuration-server.ts b/src/configuration-server.ts deleted file mode 100644 index 7bbb225..0000000 --- a/src/configuration-server.ts +++ /dev/null @@ -1,169 +0,0 @@ -import Fastify, { FastifyRequest } from "fastify"; -import JSONSchema, { JSONSchemaObject } from "@json-schema-tools/meta-schema"; - -import { Connector } from "./connector"; -import { ConnectorError } from "./error"; - -import { ErrorResponseSchema, ValidateResponse, ValidateResponseSchema } from "./schema"; -import { configureFastifyLogging } from "./logging"; - -export interface ConfigurationServerOptions { - port: number; - logLevel: string; - prettyPrintLogs: string; -} - -const errorResponses = { - 400: ErrorResponseSchema, - 403: ErrorResponseSchema, - 409: ErrorResponseSchema, - 422: ErrorResponseSchema, - 500: ErrorResponseSchema, - 501: ErrorResponseSchema, - 502: ErrorResponseSchema, -}; - -export async function startConfigurationServer< - RawConfiguration, - Configuration, - State ->( - connector: Connector, - options: ConfigurationServerOptions -) { - const server = Fastify({ - logger: configureFastifyLogging(options), - }); - - // temporary: use JSON.stringify instead of https://github.com/fastify/fast-json-stringify - // todo: remove this once issue is addressed https://github.com/fastify/fastify/issues/5073 - server.setSerializerCompiler( - ({ schema, method, url, httpStatus, contentType }) => { - return (data) => JSON.stringify(data); - } - ); - - const rawConfigurationSchema = connector.getRawConfigurationSchema(); - - server.get( - "/", - { - schema: { - response: { - 200: rawConfigurationSchema, - ...errorResponses, - }, - }, - }, - async function get_schema( - _request: FastifyRequest - ): Promise { - return connector.makeEmptyConfiguration(); - } - ); - - server.post( - "/", - { - schema: { - body: rawConfigurationSchema, - response: { - 200: rawConfigurationSchema, - ...errorResponses, - }, - }, - }, - async ( - request: FastifyRequest<{ - Body: RawConfiguration; - }> - ): Promise => { - return connector.updateConfiguration( - // type assertion required because Configuration is a generic parameter - request.body as RawConfiguration - ); - } - ); - - server.get( - "/schema", - { - schema: { - response: { - 200: JSONSchema, - ...errorResponses, - }, - }, - }, - async (): Promise => rawConfigurationSchema - ); - - server.post( - "/validate", - { - schema: { - body: rawConfigurationSchema, - response: { - 200: ValidateResponseSchema, - ...errorResponses, - }, - }, - }, - async ( - request: FastifyRequest<{ Body: RawConfiguration }> - ): Promise => { - const resolvedConfiguration = await connector.validateRawConfiguration( - // type assertion required because Configuration is a generic parameter - request.body as RawConfiguration - ); - const schema = await connector.getSchema(resolvedConfiguration); - const capabilities = connector.getCapabilities(resolvedConfiguration); - - return { - schema, - capabilities, - resolved_configuration: encodeJSON(resolvedConfiguration), - }; - } - ); - - server.get("/health", async () => {}); - - server.setErrorHandler(function (error, _request, reply) { - if (error.validation) { - reply.status(400).send({ - message: - "Validation Error - https://fastify.dev/docs/latest/Reference/Validation-and-Serialization#error-handling", - details: error.validation, - }); - } else if (error instanceof ConnectorError) { - // Log error - this.log.error(error); - // Send error response - reply.status(error.statusCode).send({ - message: error.message, - details: error.details ?? {}, - }); - } else { - reply.status(500).send({ - message: error.message, - details: {}, - }); - } - }); - - try { - await server.listen({ port: options.port, host: "0.0.0.0" }); - } catch (error) { - server.log.error(error); - process.exit(1); - } -} - -function encodeJSON(payload: unknown): string { - return Buffer.from(JSON.stringify(payload), "utf8").toString("base64"); -} -// unused for now, but keeping as reference in case it is needed later. -function decodeJSON(payload: string): T { - return JSON.parse(Buffer.from(payload, "base64").toString("utf8")); -} diff --git a/src/connector.ts b/src/connector.ts index bb2764c..1ce29b8 100644 --- a/src/connector.ts +++ b/src/connector.ts @@ -8,43 +8,15 @@ import { MutationResponse, } from "./schema"; -import { JSONSchemaObject } from "@json-schema-tools/meta-schema"; +export interface Connector { -export interface Connector { /** - * Return jsonschema for the raw configuration for this connector - */ - getRawConfigurationSchema(): JSONSchemaObject; - - /** - * Return an empty raw configuration, to be manually filled in by the user to allow connection to the data source. - * - * The exact shape depends on your connector's configuration. Example: - * - * ```json - * { - * "connection_string": "", - * "tables": [] - * } - * ``` - */ - makeEmptyConfiguration(): RawConfiguration; - /** - * Take a raw configuration, update it where appropriate by connecting to the underlying data source, and otherwise return it as-is - * For example, if our configuration includes a list of tables, we may want to fetch an updated list from the data source. - * This is also used to "hidrate" an "empty" configuration where a user has provided connection details and little else. - * @param rawConfiguration a base raw configuration - */ - updateConfiguration( - rawConfiguration: RawConfiguration - ): Promise; - /** - * Validate the raw configuration provided by the user, - * returning a configuration error or a validated [`Connector::Configuration`]. + * Validate the configuration files provided by the user, returning a validated 'Configuration', + * or throwing an 'Error'. Throwing an error prevents Connector startup. * @param configuration */ - validateRawConfiguration( - rawConfiguration: RawConfiguration + parseConfiguration( + configurationDir: string ): Promise; /** diff --git a/src/index.ts b/src/index.ts index 3f306b8..5e9baeb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,55 +1,49 @@ import { Connector } from "./connector"; import { Command, Option, InvalidOptionArgumentError } from "commander"; import { ServerOptions, startServer } from "./server"; -import { - ConfigurationServerOptions, - startConfigurationServer, -} from "./configuration-server"; - export * from "./error"; export * from "./schema"; -export { Connector, ServerOptions, ConfigurationServerOptions, startConfigurationServer, startServer }; +export { Connector, ServerOptions, startServer }; /** * Starts the connector. - * Will read runtimeflags or environment variables to determine startup mode. + * Will read command line arguments or environment variables to determine runtime configuration. * - * This shoudl be the entrypoint of your connector + * This should be the entrypoint of your connector * @param connector An object that implements the Connector interface */ -export function start( - connector: Connector +export function start( + connector: Connector ) { const program = new Command(); program.addCommand(getServeCommand(connector)); - program.addCommand(getServeConfigurationCommand(connector)); program.parseAsync(process.argv).catch(console.error); } -export function getServeCommand( - connector?: Connector +export function getServeCommand( + connector?: Connector ) { const command = new Command("serve") .addOption( - new Option("--configuration ") - .env("CONFIGURATION_FILE") + new Option("--configuration ") + .env("HASURA_CONFIGURATION_DIRECTORY") .makeOptionMandatory(true) ) .addOption( new Option("--port ") - .env("PORT") - .default(8100) + .env("HASURA_CONNECTOR_PORT") + .default(8080) .argParser(parseIntOption) ) .addOption( - new Option("--service-token-secret ").env("SERVICE_TOKEN_SECRET") + new Option("--service-token-secret ").env("HASURA_SERVICE_TOKEN_SECRET") ) - .addOption(new Option("--otlp_endpoint ").env("OTLP_ENDPOINT")) + .addOption(new Option("--otlp-endpoint ").env("OTEL_EXPORTER_OTLP_ENDPOINT")) .addOption(new Option("--service-name ").env("OTEL_SERVICE_NAME")) - .addOption(new Option("--log-level ").env("LOG_LEVEL").default("info")) - .addOption(new Option("--pretty-print-logs").env("PRETTY_PRINT_LOGS").default(false)); + .addOption(new Option("--log-level ").env("HASURA_LOG_LEVEL").default("info")) + .addOption(new Option("--pretty-print-logs").env("HASURA_PRETTY_PRINT_LOGS").default(false)); if (connector) { command.action(async (options: ServerOptions) => { @@ -59,30 +53,6 @@ export function getServeCommand( return command; } -export function getServeConfigurationCommand< - RawConfiguration, - Configuration, - State ->(connector?: Connector) { - const serveCommand = new Command("serve") - .addOption( - new Option("--port ") - .env("PORT") - .default(9100) - .argParser(parseIntOption) - ) - .addOption(new Option("--log-level ").env("LOG_LEVEL").default("info")) - .addOption(new Option("--pretty-print-logs").env("PRETTY_PRINT_LOGS").default(false)); - - if (connector) { - serveCommand.action(async (options: ConfigurationServerOptions) => { - await startConfigurationServer(connector, options); - }); - } - - return new Command("configuration").addCommand(serveCommand); -} - function parseIntOption(value: string, _previous: number): number { // parseInt takes a string and a radix const parsedValue = parseInt(value, 10); diff --git a/src/server.ts b/src/server.ts index 58e369a..a79c58f 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1,4 +1,3 @@ -import fs from "fs"; import Fastify, { FastifyRequest } from "fastify"; import { Connector } from "./connector"; @@ -21,7 +20,7 @@ import { QueryRequest, } from "./schema"; -import Ajv, { Options as AjvOptions, ErrorObject as AjvErrorObject } from "ajv"; +import { Options as AjvOptions } from "ajv"; // Create custom Ajv options to handle Rust's uint32 which is a format used in the JSON schemas, so this converts that to a number const customAjvOptions: AjvOptions = { @@ -62,36 +61,11 @@ export interface ServerOptions { prettyPrintLogs: string; } -class ConfigurationError extends Error { - validationErrors: AjvErrorObject[]; - - constructor(message: string, errors: AjvErrorObject[]) { - super(message); - this.validationErrors = errors; - } -} - -export async function startServer( - connector: Connector, +export async function startServer( + connector: Connector, options: ServerOptions ) { - const ajv = new Ajv(customAjvOptions); - const validateRawConfigurationAgainstSchema = ajv.compile( - connector.getRawConfigurationSchema() - ); - - const data = fs.readFileSync(options.configuration); - const rawConfiguration: unknown = JSON.parse(data.toString("utf8")); - if (!validateRawConfigurationAgainstSchema(rawConfiguration)) { - throw new ConfigurationError( - "Invalid configuration provided", - validateRawConfigurationAgainstSchema.errors ?? [] - ); - } - - const configuration = await connector.validateRawConfiguration( - rawConfiguration - ); + const configuration = await connector.parseConfiguration(options.configuration); const metrics = {}; // todo