diff --git a/package.json b/package.json index a9a2bdc..403a746 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "dependencies": { "@azure/storage-blob": "^12.13.0", "@nestjs/axios": "^1.0.0", + "@nestjs/cache-manager": "^1.0.0", "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.0.0", @@ -34,6 +35,8 @@ "@types/objects-to-csv": "^1.3.1", "@types/pg": "^8.6.5", "aws-sdk": "^2.1336.0", + "cache-manager": "^5.2.1", + "cache-manager-redis-store": "^3.0.1", "csvtojson": "^2.0.10", "express": "^4.18.2", "fs": "^0.0.1-security", diff --git a/src/app.controller.ts b/src/app.controller.ts index cd1c485..9fdaa20 100644 --- a/src/app.controller.ts +++ b/src/app.controller.ts @@ -1,4 +1,4 @@ -import { Body, Controller, Get, Post, Query, Res, UseGuards } from '@nestjs/common'; +import { Body, Controller, Get, Inject, Post, Query, Res, UseGuards, UseInterceptors } from '@nestjs/common'; import { AppService } from './app.service'; import { DatabaseService } from './database/database.service'; import { Response } from 'express'; @@ -8,6 +8,8 @@ import { UpdatedDateService } from './services/updated-date/updated-date.service import { ConfigService } from '@nestjs/config'; import { HttpService } from '@nestjs/axios'; import { Public } from 'nest-keycloak-connect'; +import { CACHE_MANAGER, CacheInterceptor, CacheKey } from '@nestjs/cache-manager'; +import { Cache } from 'cache-manager'; @Controller() diff --git a/src/app.module.ts b/src/app.module.ts index 9327233..29b97c7 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -11,15 +11,12 @@ import { join } from 'path'; import { UpdatedDateService } from './services/updated-date/updated-date.service'; import { KeycloakConnectModule, - ResourceGuard, - RoleGuard, AuthGuard, } from 'nest-keycloak-connect'; import { APP_GUARD } from '@nestjs/core'; import { HttpModule, HttpService } from '@nestjs/axios'; @Module({ - controllers: [AppController], providers: [AppService, { @@ -28,7 +25,7 @@ import { HttpModule, HttpService } from '@nestjs/axios'; }, MetricCsvService, UpdatedDateService], imports: [ - DatabaseModule,HttpModule, + DatabaseModule, HttpModule, ConfigModule.forRoot({ isGlobal: true }), TypeOrmModule.forRootAsync({ imports: [ConfigModule], @@ -49,15 +46,13 @@ import { HttpModule, HttpService } from '@nestjs/axios'; renderPath: new RegExp('^/assets') }), KeycloakConnectModule.register({ - authServerUrl: process.env.KEY_CLOCK_URL, + authServerUrl: process.env.KEY_CLOCK_URL, realm: process.env.REALM, clientId: process.env.KEY_CLOAK_CLIENT_ID, secret: process.env.KEY_CLOAK_SECRET, }), - + ], }) -export class AppModule { - -} \ No newline at end of file +export class AppModule {} \ No newline at end of file diff --git a/src/database/database.module.ts b/src/database/database.module.ts index 1344913..2098d88 100644 --- a/src/database/database.module.ts +++ b/src/database/database.module.ts @@ -3,6 +3,9 @@ import { Logger, Module, OnApplicationShutdown } from '@nestjs/common'; import { ConfigModule, ConfigService } from '@nestjs/config'; import { ModuleRef } from '@nestjs/core'; import { DatabaseService } from './database.service'; +import { CacheModule, CacheStore } from '@nestjs/cache-manager'; +import { redisStore } from 'cache-manager-redis-store'; + const databasePoolFactory = async (configService: ConfigService) => { return new Pool({ user: configService.get('DB_USERNAME'), @@ -13,6 +16,26 @@ const databasePoolFactory = async (configService: ConfigService) => { }); }; @Module({ + imports: [ + CacheModule.registerAsync({ + imports: [ConfigModule], + useFactory: async (config: ConfigService) => { + const store = await redisStore({ + socket: { + host: config.get('REDIS_HOST'), + port: +config.get('REDIS_PORT'), + }, + }); + + return { + store: store as unknown as CacheStore, + ttl: 1000 * 60 * 60 * 24, // 1 day in ms + max: 100 * 1000 * 1000, // 100 MB + }; + }, + inject: [ConfigService], + }), + ], providers: [ DatabaseService, { @@ -25,7 +48,7 @@ const databasePoolFactory = async (configService: ConfigService) => { }) export class DatabaseModule implements OnApplicationShutdown { private readonly logger = new Logger(DatabaseModule.name); - constructor(private readonly moduleRef: ModuleRef) { } + constructor(private readonly moduleRef: ModuleRef) {} onApplicationShutdown(signal?: string): any { this.logger.log(`Shutting down on signal ${signal}`); const pool = this.moduleRef.get('DATABASE_POOL') as Pool; diff --git a/src/database/database.service.ts b/src/database/database.service.ts index 816d275..10904ea 100644 --- a/src/database/database.service.ts +++ b/src/database/database.service.ts @@ -2,11 +2,13 @@ import { Inject, Injectable, Logger, NotFoundException } from '@nestjs/common'; import { Pool, QueryResult } from 'pg'; import * as mappings from '../maps/table_names.json'; import * as whitelist from '../maps/whitelist.json'; +import { CACHE_MANAGER } from '@nestjs/cache-manager'; +import { Cache } from 'cache-manager'; @Injectable() export class DatabaseService { private readonly logger = new Logger(DatabaseService.name); - constructor(@Inject('DATABASE_POOL') private pool: Pool) { + constructor(@Inject('DATABASE_POOL') private pool: Pool, @Inject(CACHE_MANAGER) private cacheManager: Cache) { } preprocessQuery(queryText: string): string { @@ -19,11 +21,17 @@ export class DatabaseService { return result; } - executeQuery(queryText: string, values: any[] = []): Promise { + async executeQuery(queryText: string, values: any[] = []): Promise { // pre processing query here const preprocessedQuery = this.preprocessQuery(queryText); + const cachedResponse = await this.cacheManager.get(preprocessedQuery); + if (cachedResponse) { + this.logger.debug(`Cache hit for query: ${preprocessedQuery} (${values})`); + return cachedResponse as Promise; + } this.logger.debug(`Executing query: ${preprocessedQuery} (${values})`); return this.pool.query(preprocessedQuery, values).then((result: QueryResult) => { + this.cacheManager.set(preprocessedQuery, result.rows, 1000 * 60 * 60 * 24); return result.rows; }); } diff --git a/src/main.ts b/src/main.ts index 2064b61..6158e22 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,7 +15,7 @@ async function bootstrap() { .setTitle('CQUBE') .setDescription('[ Base URL: /v0 ]') .setVersion('1.0.0') - .build(); + .build(); const document = SwaggerModule.createDocument(app, config); SwaggerModule.setup('swagger', app, document); app.enableCors(); diff --git a/yarn.lock b/yarn.lock index e6f5fec..2ffb89f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -779,6 +779,11 @@ dependencies: axios "1.2.1" +"@nestjs/cache-manager@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@nestjs/cache-manager/-/cache-manager-1.0.0.tgz#092b96fb6b083bdfc08cdf92230c514e8f6631d6" + integrity sha512-XMNdgsj3H+Ng/SYwFl13vRGNFA3e5Obk8LNwIuHLVSocnK2exReAWtscxEjQhoBc4FW4jAYOgU/U+mt18Q9T0g== + "@nestjs/cli@^9.0.0": version "9.4.2" resolved "https://registry.yarnpkg.com/@nestjs/cli/-/cli-9.4.2.tgz#d74668fde59129a9992b70ea72f9f66845efe11d" @@ -931,6 +936,40 @@ resolved "https://registry.yarnpkg.com/@opentelemetry/api/-/api-1.4.1.tgz#ff22eb2e5d476fbc2450a196e40dd243cc20c28f" integrity sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA== +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.5.7": + version "1.5.7" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.7.tgz#92cc5c98c76f189e37d24f0e1e17e104c6af17d4" + integrity sha512-gaOBOuJPjK5fGtxSseaKgSvjiZXQCdLlGg9WYQst+/GRUjmXaiB5kVkeQMRtPc7Q2t93XZcJfBMSwzs/XS9UZw== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" + integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== + +"@redis/json@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.4.tgz#f372b5f93324e6ffb7f16aadcbcb4e5c3d39bda1" + integrity sha512-LUZE2Gdrhg0Rx7AN+cZkb1e6HjoSKaeeW8rYnt89Tly13GBI5eP4CwDVr+MY8BAYfCg4/N15OUrtLoona9uSgw== + +"@redis/search@1.1.2": + version "1.1.2" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.2.tgz#6a8f66ba90812d39c2457420f859ce8fbd8f3838" + integrity sha512-/cMfstG/fOh/SsE+4/BQGeuH/JJloeWuH+qJzM8dbxuWvdWibWAOAHHCZTMPhV3xIlH4/cUEIA8OV5QnYpaVoA== + +"@redis/time-series@1.0.4": + version "1.0.4" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.4.tgz#af85eb080f6934580e4d3b58046026b6c2b18717" + integrity sha512-ThUIgo2U/g7cCuZavucQTQzA9g9JbDDY2f64u3AbAoz/8vE2lt2U37LamDUVChhaDA3IRT9R6VvJwqnUfTJzng== + "@sinclair/typebox@^0.24.1": version "0.24.51" resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.24.51.tgz#645f33fe4e02defe26f2f5c0410e1c094eac7f5f" @@ -2062,6 +2101,21 @@ bytes@3.1.2: resolved "https://registry.yarnpkg.com/bytes/-/bytes-3.1.2.tgz#8b0beeb98605adf1b128fa4386403c009e0221a5" integrity sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg== +cache-manager-redis-store@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/cache-manager-redis-store/-/cache-manager-redis-store-3.0.1.tgz#8eeb211212763d04cef4058666182d624f714299" + integrity sha512-o560kw+dFqusC9lQJhcm6L2F2fMKobJ5af+FoR2PdnMVdpQ3f3Bz6qzvObTGyvoazQJxjQNWgMQeChP4vRTuXQ== + dependencies: + redis "^4.3.1" + +cache-manager@^5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/cache-manager/-/cache-manager-5.2.1.tgz#c1fbc1b56da2f364c4f8eb8385bf94a19a00befd" + integrity sha512-qYHx0DlM0mepUqXkpDg83K1dYEXOinq9+sYdHxs1c5LQjR1MPgm34im+JVtsy9+uoeE2T1JLzJSAB+nV4IH5dQ== + dependencies: + lodash.clonedeep "^4.5.0" + lru-cache "~9.1.1" + call-bind@^1.0.0, call-bind@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/call-bind/-/call-bind-1.0.2.tgz#b1d4e89e688119c3c9a903ad30abb2f6a919be3c" @@ -2229,6 +2283,11 @@ clone@^1.0.2: resolved "https://registry.yarnpkg.com/clone/-/clone-1.0.4.tgz#da309cc263df15994c688ca902179ca3c7cd7c7e" integrity sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg== +cluster-key-slot@1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + co@^4.6.0: version "4.6.0" resolved "https://registry.yarnpkg.com/co/-/co-4.6.0.tgz#6ea6bdf3d853ae54ccb8e47bfa0bf3f9031fb184" @@ -3183,6 +3242,11 @@ function-bind@^1.1.1: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -4284,6 +4348,11 @@ locate-path@^6.0.0: dependencies: p-locate "^5.0.0" +lodash.clonedeep@^4.5.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/lodash.clonedeep/-/lodash.clonedeep-4.5.0.tgz#e23f3f9c4f8fbdde872529c1071857a086e5ccef" + integrity sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ== + lodash.memoize@4.x: version "4.1.2" resolved "https://registry.yarnpkg.com/lodash.memoize/-/lodash.memoize-4.1.2.tgz#bcc6c49a42a2840ed997f323eada5ecd182e0bfe" @@ -4321,7 +4390,7 @@ lru-cache@^6.0.0: dependencies: yallist "^4.0.0" -lru-cache@^9.0.0: +lru-cache@^9.0.0, lru-cache@~9.1.1: version "9.1.1" resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-9.1.1.tgz#c58a93de58630b688de39ad04ef02ef26f1902f1" integrity sha512-65/Jky17UwSb0BuB9V+MyDpsOtXKmYwzhyl+cOa9XUiI4uV2Ouy/2voFP3+al0BjZbJgMBD8FojMpAf+Z+qn4A== @@ -6304,6 +6373,18 @@ rechoir@^0.6.2: dependencies: resolve "^1.1.6" +redis@^4.3.1: + version "4.6.6" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.6.tgz#46d4f2d149d1634d6ef53db5747412a0ef7974ec" + integrity sha512-aLs2fuBFV/VJ28oLBqYykfnhGGkFxvx0HdCEBYdJ99FFbSEMZ7c1nVKwR6ZRv+7bb7JnC0mmCzaqu8frgOYhpA== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.5.7" + "@redis/graph" "1.1.0" + "@redis/json" "1.0.4" + "@redis/search" "1.1.2" + "@redis/time-series" "1.0.4" + reflect-metadata@^0.1.13: version "0.1.13" resolved "https://registry.yarnpkg.com/reflect-metadata/-/reflect-metadata-0.1.13.tgz#67ae3ca57c972a2aa1642b10fe363fe32d49dc08" @@ -7348,16 +7429,16 @@ y18n@^5.0.5: resolved "https://registry.yarnpkg.com/y18n/-/y18n-5.0.8.tgz#7f4934d0f7ca8c56f95314939ddcd2dd91ce1d55" integrity sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA== +yallist@4.0.0, yallist@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" + integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== + yallist@^3.0.2: version "3.1.1" resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd" integrity sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g== -yallist@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" - integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== - yaml@^1.10.0: version "1.10.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"