From 1ba800798216058453b0e7368d3b846137392fe9 Mon Sep 17 00:00:00 2001 From: Aitor Algorta Date: Mon, 12 Feb 2024 15:15:43 +0100 Subject: [PATCH] general improvements --- .../schema-registry-manager/Dockerfile | 12 +- .../schema-registry-manager/src/app.js | 237 ++++++++++++++++++ .../schema-registry-manager/src/package.json | 9 +- .../src/providers/karapace.js | 182 ++++++++++++++ .../schema-registry-manager/src/types.js | 8 + 5 files changed, 439 insertions(+), 9 deletions(-) create mode 100644 backend/components/schema-registry-manager/src/app.js create mode 100644 backend/components/schema-registry-manager/src/providers/karapace.js create mode 100644 backend/components/schema-registry-manager/src/types.js diff --git a/backend/components/schema-registry-manager/Dockerfile b/backend/components/schema-registry-manager/Dockerfile index 49fdcebab..e06995486 100644 --- a/backend/components/schema-registry-manager/Dockerfile +++ b/backend/components/schema-registry-manager/Dockerfile @@ -3,17 +3,17 @@ FROM node:18 WORKDIR /app COPY ./src/package*.json ./ - COPY ./src/tsconfig*.json ./ RUN npm install - -RUN npm install -g typescript - -RUN npm install -g ts-node +RUN npm install typescript -g COPY ./src/app.ts ./ +COPY ./src/types.ts ./ +COPY ./src/providers/karapace.ts ./providers/ + +RUN tsc EXPOSE 3000 -CMD [ "ts-node", "app.ts" ] +CMD [ "node", "app.js" ] diff --git a/backend/components/schema-registry-manager/src/app.js b/backend/components/schema-registry-manager/src/app.js new file mode 100644 index 000000000..4102e6203 --- /dev/null +++ b/backend/components/schema-registry-manager/src/app.js @@ -0,0 +1,237 @@ +'use strict'; +var __awaiter = + (this && this.__awaiter) || + function (thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P(function (resolve) { + resolve(value); + }); + } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator['throw'](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); + } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + }; +var __importDefault = + (this && this.__importDefault) || + function (mod) { + return mod && mod.__esModule ? mod : {default: mod}; + }; +Object.defineProperty(exports, '__esModule', {value: true}); +const dotenv_1 = __importDefault(require('dotenv')); +const express_1 = __importDefault(require('express')); +const types_1 = require('./types'); +const karapace_1 = require('./providers/karapace'); +dotenv_1.default.config(); +const app = (0, express_1.default)(); +const port = process.env.PORT || 3000; +const bodyParser = require('body-parser'); +const currentProvider = types_1.SchemaProvider.karapace; +// Middleware +app.use(bodyParser.json()); +app.get('/schemas.provider', (req, res) => { + res.status(200).send(currentProvider); +}); +app.get('/schemas.list', (req, res) => { + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getSchemas)() + .then(response => { + res.send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.get('/schemas.versions', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getSchemaVersions)(req.query.topicName) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.get('/schemas.info', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + let version = 'latest'; + if (req.query.version) { + version = req.query.version; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getSchemaInfo)(req.query.topicName, version) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.update', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.updateSchema)(req.query.topicName, req.body.schema) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.create', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.createSchema)(req.query.topicName, req.body.schema) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.compatibility', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + if (!req.query.version) { + res.status(400).send('Missing version'); + return; + } + if (!req.body.schema) { + res.status(400).send('Missing schema'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.checkCompatibilityOfNewSchema)(req.query.topicName, req.body.schema, req.query.version) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.post('/schemas.delete', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.deleteSchema)(req.query.topicName) + .then(response => { + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +app.get('/schemas.lastMessage', (req, res) => { + if (!req.query.topicName) { + res.status(400).send('Missing topicName'); + return; + } + switch (currentProvider) { + case types_1.SchemaProvider.karapace: + (0, karapace_1.getLastMessage)(req.query.topicName) + .then(response => { + console.log(response); + res.status(200).send(response); + }) + .catch(e => { + res.status(500).send(e); + }); + break; + default: + res.status(404).send('Provider Not Found'); + break; + } +}); +function main() { + return __awaiter(this, void 0, void 0, function* () { + app.listen(port, () => { + console.log(`Server is running on http://localhost:${port}`); + }); + }); +} +main().catch(console.error); diff --git a/backend/components/schema-registry-manager/src/package.json b/backend/components/schema-registry-manager/src/package.json index 00f591b08..1cdd291b3 100644 --- a/backend/components/schema-registry-manager/src/package.json +++ b/backend/components/schema-registry-manager/src/package.json @@ -1,12 +1,15 @@ { - "dependencies": { + "dependencies": { "@types/express": "^4.17.21", "@types/node": "^20.10.3", - "node-fetch": "^2.6.1" + "dotenv": "^16.4.2", + "express": "^4.18.2", + "node-fetch": "^2.6.1" }, "devDependencies": { "@types/express": "^4.17.21", "@types/node": "^20.10.3", - "typescript": "^5.3.2" + "ts-node": "^10.9.2", + "typescript": "^5.3.3" } } diff --git a/backend/components/schema-registry-manager/src/providers/karapace.js b/backend/components/schema-registry-manager/src/providers/karapace.js new file mode 100644 index 000000000..32c74cddf --- /dev/null +++ b/backend/components/schema-registry-manager/src/providers/karapace.js @@ -0,0 +1,182 @@ +'use strict'; +var __awaiter = + (this && this.__awaiter) || + function (thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P(function (resolve) { + resolve(value); + }); + } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator['throw'](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); + } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + }; +Object.defineProperty(exports, '__esModule', {value: true}); +exports.getLastMessage = + exports.deleteSchema = + exports.checkCompatibilityOfNewSchema = + exports.createSchema = + exports.updateSchema = + exports.getSchemaInfo = + exports.getSchemaVersions = + exports.getSchemas = + void 0; +function getSchemas() { + return __awaiter(this, void 0, void 0, function* () { + return getData('subjects').then(response => { + return response; + }); + }); +} +exports.getSchemas = getSchemas; +function getSchemaVersions(topicName) { + return __awaiter(this, void 0, void 0, function* () { + return getData(`subjects/${topicName}/versions`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); + }); +} +exports.getSchemaVersions = getSchemaVersions; +function getSchemaInfo(topicName, version) { + return __awaiter(this, void 0, void 0, function* () { + return getData(`subjects/${topicName}/versions/${version}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); + }); +} +exports.getSchemaInfo = getSchemaInfo; +function updateSchema(topicName, schema) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), + }; + return postData(`subjects/${topicName}/versions`, body).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }); + }); +} +exports.updateSchema = updateSchema; +function createSchema(topicName, schema) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), + }; + return postData(`subjects/${topicName}/versions`, body) + .then(response => { + if (response.id) return response; + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); + }); +} +exports.createSchema = createSchema; +function checkCompatibilityOfNewSchema(topicName, schema, version) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + schema: JSON.stringify(Object.assign({}, JSON.parse(schema))), + }; + return postData(`compatibility/subjects/${topicName}/versions/${version}`, body) + .then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + if (response.is_compatible !== undefined) { + if (response.is_compatible === true) { + return response; + } + return Promise.reject('Schema Not Compatible'); + } + if (response.message) return Promise.reject(response.message); + return Promise.reject('Unknown Error'); + }) + .catch(e => { + return Promise.reject(e); + }); + }); +} +exports.checkCompatibilityOfNewSchema = checkCompatibilityOfNewSchema; +function deleteSchema(topicName) { + return __awaiter(this, void 0, void 0, function* () { + return deleteData(`subjects/${topicName}`).then(response => { + if (response.error_code && response.error_code.toString().includes('404') && !topicName.includes('-value')) { + return Promise.reject('404 Not Found'); + } + return response; + }); + }); +} +exports.deleteSchema = deleteSchema; +function getLastMessage(topicName) { + return __awaiter(this, void 0, void 0, function* () { + const body = { + ksql: `PRINT '${topicName}' FROM BEGINNING LIMIT 1;`, + streamsProperties: {}, + }; + return postData('query', body).then(response => { + console.log(response); + return response; + }); + }); +} +exports.getLastMessage = getLastMessage; +function getData(url) { + return __awaiter(this, void 0, void 0, function* () { + const response = yield fetch(process.env.URL + '/' + url, { + method: 'GET', + }); + return response.json(); + }); +} +function deleteData(url) { + return __awaiter(this, void 0, void 0, function* () { + const response = yield fetch(process.env.URL + '/' + url, { + method: 'DELETE', + }); + return response.json(); + }); +} +function postData(url, body) { + return __awaiter(this, void 0, void 0, function* () { + const response = yield fetch(process.env.URL + '/' + url, { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.schemaregistry.v1+json', + }, + body: JSON.stringify(body), + }); + return response.json(); + }); +} diff --git a/backend/components/schema-registry-manager/src/types.js b/backend/components/schema-registry-manager/src/types.js new file mode 100644 index 000000000..3571994e8 --- /dev/null +++ b/backend/components/schema-registry-manager/src/types.js @@ -0,0 +1,8 @@ +'use strict'; +Object.defineProperty(exports, '__esModule', {value: true}); +exports.SchemaProvider = void 0; +var SchemaProvider; +(function (SchemaProvider) { + SchemaProvider['karapace'] = 'karapace'; + SchemaProvider['confluentCloud'] = 'confluent-cloud'; +})(SchemaProvider || (exports.SchemaProvider = SchemaProvider = {}));