diff --git a/src/app.ts b/src/app.ts index 82d7112a..5a3d202c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -52,8 +52,9 @@ const build = (opts: buildOpts = {}): FastifyInstance => { app.addSchema(schemas.errorSchema) app.register(plugins.tenantId) - app.register(plugins.metrics({ enabledEndpoint: !isMultitenant })) app.register(plugins.logTenantId) + app.register(plugins.metrics({ enabledEndpoint: !isMultitenant })) + app.register(plugins.tracing) app.register(plugins.logRequest({ excludeUrls: ['/status', '/metrics', '/health'] })) app.register(routes.tus, { prefix: 'upload/resumable' }) app.register(routes.bucket, { prefix: 'bucket' }) diff --git a/src/config.ts b/src/config.ts index 8b0f6deb..bcc53680 100644 --- a/src/config.ts +++ b/src/config.ts @@ -112,7 +112,10 @@ type StorageConfigType = { s3ProtocolAccessKeyId?: string s3ProtocolAccessKeySecret?: string s3ProtocolNonCanonicalHostHeader?: string + tracingEnabled?: boolean tracingMode?: string + tracingTimeMinDuration: number + tracingReturnServerTimings: boolean } function getOptionalConfigFromEnv(key: string, fallback?: string): string | undefined { @@ -160,16 +163,19 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { } envPaths.map((envPath) => dotenv.config({ path: envPath, override: false })) + const isMultitenant = getOptionalConfigFromEnv('MULTI_TENANT', 'IS_MULTITENANT') === 'true' config = { isProduction: process.env.NODE_ENV === 'production', exposeDocs: getOptionalConfigFromEnv('EXPOSE_DOCS') !== 'false', // Tenant tenantId: - getOptionalConfigFromEnv('PROJECT_REF') || - getOptionalConfigFromEnv('TENANT_ID') || - 'storage-single-tenant', - isMultitenant: getOptionalConfigFromEnv('MULTI_TENANT', 'IS_MULTITENANT') === 'true', + getOptionalConfigFromEnv('PROJECT_REF') ?? + getOptionalConfigFromEnv('TENANT_ID') ?? + isMultitenant + ? '' + : 'storage-single-tenant', + isMultitenant: isMultitenant, // Server region: getOptionalConfigFromEnv('SERVER_REGION', 'REGION') || 'not-specified', @@ -312,7 +318,13 @@ export function getConfig(options?: { reload?: boolean }): StorageConfigType { defaultMetricsEnabled: !( getOptionalConfigFromEnv('DEFAULT_METRICS_ENABLED', 'ENABLE_DEFAULT_METRICS') === 'false' ), + tracingEnabled: getOptionalConfigFromEnv('TRACING_ENABLED') === 'true', tracingMode: getOptionalConfigFromEnv('TRACING_MODE') ?? 'basic', + tracingTimeMinDuration: parseFloat( + getOptionalConfigFromEnv('TRACING_SERVER_TIME_MIN_DURATION') ?? '100.0' + ), + tracingReturnServerTimings: + getOptionalConfigFromEnv('TRACING_RETURN_SERVER_TIMINGS') === 'true', // Queue pgQueueEnable: getOptionalConfigFromEnv('PG_QUEUE_ENABLE', 'ENABLE_QUEUE_EVENTS') === 'true', diff --git a/src/http/plugins/index.ts b/src/http/plugins/index.ts index 64698631..228d222e 100644 --- a/src/http/plugins/index.ts +++ b/src/http/plugins/index.ts @@ -9,4 +9,4 @@ export * from './tenant-feature' export * from './metrics' export * from './xml' export * from './signature-v4' -export * from './tracing-mode' +export * from './tracing' diff --git a/src/http/plugins/log-request.ts b/src/http/plugins/log-request.ts index 26bd191e..50b44073 100644 --- a/src/http/plugins/log-request.ts +++ b/src/http/plugins/log-request.ts @@ -79,6 +79,7 @@ export const logRequest = (options: RequestLoggerOptions) => owner: req.owner, operation: req.operation?.type ?? req.routeConfig.operation?.type, resources: req.resources, + serverTimes: req.serverTimings, }) }) @@ -112,6 +113,7 @@ export const logRequest = (options: RequestLoggerOptions) => owner: req.owner, resources: req.resources, operation: req.operation?.type ?? req.routeConfig.operation?.type, + serverTimes: req.serverTimings, }) }) }) diff --git a/src/http/plugins/tracing-mode.ts b/src/http/plugins/tracing-mode.ts deleted file mode 100644 index 431fd501..00000000 --- a/src/http/plugins/tracing-mode.ts +++ /dev/null @@ -1,23 +0,0 @@ -import fastifyPlugin from 'fastify-plugin' -import { getTenantConfig } from '@internal/database' - -import { getConfig } from '../../config' - -declare module 'fastify' { - interface FastifyRequest { - tracingMode?: string - } -} - -const { isMultitenant, tracingMode: defaultTracingMode } = getConfig() - -export const tracingMode = fastifyPlugin(async function tracingMode(fastify) { - fastify.addHook('onRequest', async (request, reply) => { - if (isMultitenant) { - const tenantConfig = await getTenantConfig(request.tenantId) - request.tracingMode = tenantConfig.tracingMode - } else { - request.tracingMode = defaultTracingMode - } - }) -}) diff --git a/src/http/plugins/tracing.ts b/src/http/plugins/tracing.ts new file mode 100644 index 00000000..e3c26beb --- /dev/null +++ b/src/http/plugins/tracing.ts @@ -0,0 +1,141 @@ +import fastifyPlugin from 'fastify-plugin' +import { isIP } from 'net' +import { getTenantConfig } from '@internal/database' + +import { getConfig } from '../../config' +import { context, trace } from '@opentelemetry/api' +import { traceCollector } from '@internal/monitoring/otel-processor' +import { ReadableSpan } from '@opentelemetry/sdk-trace-base' +import { logger, logSchema } from '@internal/monitoring' + +declare module 'fastify' { + interface FastifyRequest { + tracingMode?: string + serverTimings?: { spanName: string; duration: number }[] + } +} + +const { + isMultitenant, + tracingEnabled, + tracingMode: defaultTracingMode, + tracingReturnServerTimings, +} = getConfig() + +export const tracing = fastifyPlugin(async function tracingMode(fastify) { + if (!tracingEnabled) { + return + } + fastify.register(traceServerTime) + + fastify.addHook('onRequest', async (request) => { + if (isMultitenant && request.tenantId) { + const tenantConfig = await getTenantConfig(request.tenantId) + request.tracingMode = tenantConfig.tracingMode + } else { + request.tracingMode = defaultTracingMode + } + }) +}) + +export const traceServerTime = fastifyPlugin(async function traceServerTime(fastify) { + if (!tracingEnabled) { + return + } + fastify.addHook('onResponse', async (request, reply, payload) => { + const traceId = trace.getSpan(context.active())?.spanContext().traceId + + if (traceId) { + const spans = traceCollector.getSpansForTrace(traceId) + if (spans) { + try { + const serverTimingHeaders = spansToServerTimings(spans) + + request.serverTimings = serverTimingHeaders + + // Return Server-Timing if enabled + if (tracingReturnServerTimings) { + const httpServerTimes = serverTimingHeaders + .map(({ spanName, duration }) => { + return `${spanName};dur=${duration.toFixed(3)}` // Convert to milliseconds + }) + .join(',') + reply.header('Server-Timing', httpServerTimes) + } + } catch (e) { + logSchema.error(logger, 'failed parsing server times', { error: e, type: 'otel' }) + } + + traceCollector.clearTrace(traceId) + + return payload + } + } + }) + + fastify.addHook('onRequestAbort', async (req) => { + const traceId = trace.getSpan(context.active())?.spanContext().traceId + + if (traceId) { + const spans = traceCollector.getSpansForTrace(traceId) + if (spans) { + req.serverTimings = spansToServerTimings(spans) + } + traceCollector.clearTrace(traceId) + } + }) +}) + +function enrichSpanName(spanName: string, span: ReadableSpan) { + if (span.attributes['knex.version']) { + const queryOperation = (span.attributes['db.operation'] as string)?.split(' ').shift() + return ( + `pg_query_` + + queryOperation?.toUpperCase() + + (span.attributes['db.sql.table'] ? '_' + span.attributes['db.sql.table'] : '_postgres') + ) + } + + if (['GET', 'PUT', 'HEAD', 'DELETE', 'POST'].includes(spanName)) { + return `HTTP_${spanName}` + } + + return spanName +} + +function spansToServerTimings(spans: ReadableSpan[]) { + return spans + .sort((a, b) => { + return a.startTime[1] - b.startTime[1] + }) + .map((span) => { + const duration = span.duration[1] // Duration in nanoseconds + + let spanName = + span.name + .split('->') + .pop() + ?.trimStart() + .replaceAll('\n', '') + .replaceAll('.', '_') + .replaceAll(' ', '_') + .replaceAll('-', '_') + .replaceAll('___', '_') + .replaceAll(':', '_') + .replaceAll('_undefined', '') || 'UNKNOWN' + + spanName = enrichSpanName(spanName, span) + const hostName = span.attributes['net.peer.name'] as string | undefined + + return { + spanName, + duration: duration / 1e6, + action: span.attributes['db.statement'], + host: hostName + ? isIP(hostName) + ? hostName + : hostName?.split('.').slice(-3).join('.') + : undefined, + } + }) +} diff --git a/src/http/routes/bucket/getAllBuckets.ts b/src/http/routes/bucket/getAllBuckets.ts index 778e0446..ca79c2c6 100644 --- a/src/http/routes/bucket/getAllBuckets.ts +++ b/src/http/routes/bucket/getAllBuckets.ts @@ -43,7 +43,7 @@ export default async function routes(fastify: FastifyInstance) { 'id, name, public, owner, created_at, updated_at, file_size_limit, allowed_mime_types' ) - response.send(results) + return response.send(results) } ) } diff --git a/src/http/routes/bucket/getBucket.ts b/src/http/routes/bucket/getBucket.ts index dbd5c8fb..94ba37e4 100644 --- a/src/http/routes/bucket/getBucket.ts +++ b/src/http/routes/bucket/getBucket.ts @@ -41,7 +41,7 @@ export default async function routes(fastify: FastifyInstance) { 'id, name, owner, public, created_at, updated_at, file_size_limit, allowed_mime_types' ) - response.send(results) + return response.send(results) } ) } diff --git a/src/http/routes/bucket/index.ts b/src/http/routes/bucket/index.ts index 2640b7e1..22546655 100644 --- a/src/http/routes/bucket/index.ts +++ b/src/http/routes/bucket/index.ts @@ -5,13 +5,12 @@ import emptyBucket from './emptyBucket' import getAllBuckets from './getAllBuckets' import getBucket from './getBucket' import updateBucket from './updateBucket' -import { storage, jwt, db, tracingMode } from '../../plugins' +import { storage, jwt, db } from '../../plugins' export default async function routes(fastify: FastifyInstance) { fastify.register(jwt) fastify.register(db) fastify.register(storage) - fastify.register(tracingMode) fastify.register(createBucket) fastify.register(emptyBucket) diff --git a/src/http/routes/object/index.ts b/src/http/routes/object/index.ts index 6bfe3926..10bd2003 100644 --- a/src/http/routes/object/index.ts +++ b/src/http/routes/object/index.ts @@ -1,5 +1,5 @@ import { FastifyInstance } from 'fastify' -import { jwt, storage, dbSuperUser, db, tracingMode } from '../../plugins' +import { jwt, storage, dbSuperUser, db } from '../../plugins' import copyObject from './copyObject' import createObject from './createObject' import deleteObject from './deleteObject' @@ -24,7 +24,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(jwt) fastify.register(db) fastify.register(storage) - fastify.register(tracingMode) fastify.register(deleteObject) fastify.register(deleteObjects) @@ -43,7 +42,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(async (fastify) => { fastify.register(dbSuperUser) fastify.register(storage) - fastify.register(tracingMode) fastify.register(getPublicObject) fastify.register(getSignedObject) diff --git a/src/http/routes/object/listObjects.ts b/src/http/routes/object/listObjects.ts index e4d85b2f..e10f3b40 100644 --- a/src/http/routes/object/listObjects.ts +++ b/src/http/routes/object/listObjects.ts @@ -73,7 +73,7 @@ export default async function routes(fastify: FastifyInstance) { }, }) - response.status(200).send(results) + return response.status(200).send(results) } ) } diff --git a/src/http/routes/render/index.ts b/src/http/routes/render/index.ts index bda6c045..78cc00d4 100644 --- a/src/http/routes/render/index.ts +++ b/src/http/routes/render/index.ts @@ -2,7 +2,7 @@ import { FastifyInstance } from 'fastify' import renderPublicImage from './renderPublicImage' import renderAuthenticatedImage from './renderAuthenticatedImage' import renderSignedImage from './renderSignedImage' -import { jwt, storage, requireTenantFeature, db, dbSuperUser, tracingMode } from '../../plugins' +import { jwt, storage, requireTenantFeature, db, dbSuperUser } from '../../plugins' import { getConfig } from '../../../config' import { rateLimiter } from './rate-limiter' @@ -23,7 +23,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(jwt) fastify.register(db) fastify.register(storage) - fastify.register(tracingMode) fastify.register(renderAuthenticatedImage) }) @@ -37,7 +36,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(dbSuperUser) fastify.register(storage) - fastify.register(tracingMode) fastify.register(renderSignedImage) fastify.register(renderPublicImage) diff --git a/src/http/routes/s3/index.ts b/src/http/routes/s3/index.ts index 41e4a37a..086fb31b 100644 --- a/src/http/routes/s3/index.ts +++ b/src/http/routes/s3/index.ts @@ -1,7 +1,7 @@ import { FastifyInstance, RouteHandlerMethod } from 'fastify' import { JSONSchema } from 'json-schema-to-ts' import { trace } from '@opentelemetry/api' -import { db, jsonToXml, signatureV4, storage, tracingMode } from '../../plugins' +import { db, jsonToXml, signatureV4, storage } from '../../plugins' import { findArrayPathsInSchemas, getRouter, RequestInput } from './router' import { s3ErrorHandler } from './error-handler' @@ -110,7 +110,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(signatureV4) fastify.register(db) fastify.register(storage) - fastify.register(tracingMode) localFastify[method]( routePath, diff --git a/src/http/routes/tus/index.ts b/src/http/routes/tus/index.ts index 988f724b..871058f6 100644 --- a/src/http/routes/tus/index.ts +++ b/src/http/routes/tus/index.ts @@ -4,7 +4,7 @@ import * as http from 'http' import { ServerOptions, DataStore } from '@tus/server' import { getFileSizeLimit } from '@storage/limits' import { Storage } from '@storage/storage' -import { jwt, storage, db, dbSuperUser, tracingMode } from '../../plugins' +import { jwt, storage, db, dbSuperUser } from '../../plugins' import { getConfig } from '../../../config' import { TusServer, @@ -38,6 +38,7 @@ const { tusUrlExpiryMs, tusPath, tusPartSize, + tusMaxConcurrentUploads, storageBackendType, storageFilePath, } = getConfig() @@ -61,7 +62,7 @@ function createTusStore() { partSize: tusPartSize * 1024 * 1024, // Each uploaded part will have ${tusPartSize}MB, expirationPeriodInMilliseconds: tusUrlExpiryMs, cache: new AlsMemoryKV(), - maxConcurrentPartUploads: 500, + maxConcurrentPartUploads: tusMaxConcurrentUploads, s3ClientConfig: { requestHandler: new NodeHttpHandler({ ...agent, @@ -137,7 +138,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(jwt) fastify.register(db) fastify.register(storage) - fastify.register(tracingMode) fastify.register(authenticatedRoutes, { tusServer, @@ -149,7 +149,6 @@ export default async function routes(fastify: FastifyInstance) { async (fastify) => { fastify.register(dbSuperUser) fastify.register(storage) - fastify.register(tracingMode) fastify.register(authenticatedRoutes, { tusServer, @@ -163,7 +162,6 @@ export default async function routes(fastify: FastifyInstance) { fastify.register(async (fastify) => { fastify.register(dbSuperUser) fastify.register(storage) - fastify.register(tracingMode) fastify.register(publicRoutes, { tusServer, @@ -175,7 +173,6 @@ export default async function routes(fastify: FastifyInstance) { async (fastify) => { fastify.register(dbSuperUser) fastify.register(storage) - fastify.register(tracingMode) fastify.register(publicRoutes, { tusServer, diff --git a/src/http/routes/tus/lifecycle.ts b/src/http/routes/tus/lifecycle.ts index afad3657..7317f23f 100644 --- a/src/http/routes/tus/lifecycle.ts +++ b/src/http/routes/tus/lifecycle.ts @@ -87,11 +87,21 @@ export function generateUrl( req: http.IncomingMessage, { proto, host, path, id }: { proto: string; host: string; path: string; id: string } ) { + if (!req.url) { + throw ERRORS.InvalidParameter('url') + } proto = process.env.NODE_ENV === 'production' ? 'https' : proto + const url = new URL( + `${proto}://${(req.headers.x_forwarded_host as string) || (req.headers.host as string) || ''}` + ) const isSigned = req.url?.endsWith(SIGNED_URL_SUFFIX) const fullPath = isSigned ? `${path}${SIGNED_URL_SUFFIX}` : path + if (url.port && req.headers['x-forwarded-port']) { + host += `:${req.headers['x-forwarded-port']}` + } + // remove the tenant-id from the url, since we'll be using the tenant-id from the request id = id.split('/').slice(1).join('/') id = Buffer.from(id, 'utf-8').toString('base64url') diff --git a/src/internal/monitoring/logger.ts b/src/internal/monitoring/logger.ts index 78a4058a..668531a3 100644 --- a/src/internal/monitoring/logger.ts +++ b/src/internal/monitoring/logger.ts @@ -51,6 +51,7 @@ export interface RequestLog { owner?: string operation?: string resources?: string[] + serverTimes?: { spanName: string; duration: number }[] } export interface EventLog { diff --git a/src/internal/monitoring/otel-processor.ts b/src/internal/monitoring/otel-processor.ts new file mode 100644 index 00000000..09a9a56c --- /dev/null +++ b/src/internal/monitoring/otel-processor.ts @@ -0,0 +1,91 @@ +import { SpanProcessor, ReadableSpan } from '@opentelemetry/sdk-trace-base' +import TTLCache from '@isaacs/ttlcache' +import { getConfig } from '../../config' + +const { isProduction, tracingTimeMinDuration } = getConfig() + +interface Trace { + id: string + rootSpanId: string + spans: ReadableSpan[] +} + +export class TraceCollectorSpanProcessor implements SpanProcessor { + private traces: TTLCache + + constructor() { + this.traces = new TTLCache({ + ttl: 120 * 1000, // 120 seconds TTL + noUpdateTTL: true, + noDisposeOnSet: true, + }) + } + + export() { + // no-op + } + + onStart(span: ReadableSpan): void { + // No action needed on start + if (!span.parentSpanId) { + const traceId = span.spanContext().traceId + const spanId = span.spanContext().spanId + + const hasTrace = this.traces.has(traceId) + + if (!hasTrace) { + this.traces.set(traceId, { + id: traceId, + spans: [], + rootSpanId: spanId, + }) + } + } + } + + onEnd(span: ReadableSpan): void { + const minLatency = isProduction ? tracingTimeMinDuration : 0.01 + + // only add span if higher than min latency (no need to waste memory) + if (span.duration[1] / 1e6 > minLatency) { + const traceId = span.spanContext().traceId + const trace = this.traces.get(traceId) + + if (!trace) { + return + } + + const cachedSpans = trace?.spans || [] + const whiteList = ['jwt', 'pg', 'raw', 's3', 'first', 'insert', 'select', 'delete'] + + // only push top level spans + if (whiteList.some((item) => span.name.toLowerCase().includes(item))) { + cachedSpans.push(span) + this.traces.set(traceId, { + ...trace, + spans: cachedSpans, + }) + } + } + } + + shutdown(): Promise { + this.traces.clear() + return Promise.resolve() + } + + forceFlush(): Promise { + this.traces.clear() + return Promise.resolve() + } + + getSpansForTrace(traceId: string): ReadableSpan[] { + return this.traces.get(traceId)?.spans || [] + } + + clearTrace(traceId: string): void { + this.traces.delete(traceId) + } +} + +export const traceCollector = new TraceCollectorSpanProcessor() diff --git a/src/internal/monitoring/otel.ts b/src/internal/monitoring/otel.ts index 1c53ae5f..2bc3fdaa 100644 --- a/src/internal/monitoring/otel.ts +++ b/src/internal/monitoring/otel.ts @@ -18,10 +18,12 @@ import { import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-grpc' import { getNodeAutoInstrumentations } from '@opentelemetry/auto-instrumentations-node' import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base' +import { SpanExporter, BatchSpanProcessor } from '@opentelemetry/sdk-trace-base' import * as grpc from '@grpc/grpc-js' import { HttpInstrumentation } from '@opentelemetry/instrumentation-http' import { IncomingMessage } from 'http' import { logger, logSchema } from '@internal/monitoring/logger' +import { traceCollector } from '@internal/monitoring/otel-processor' const headersEnv = process.env.OTEL_EXPORTER_OTLP_TRACES_HEADERS || '' @@ -40,14 +42,20 @@ Object.keys(exporterHeaders).forEach((key) => { }) const endpoint = process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT +let traceExporter: SpanExporter | undefined = undefined -// Create an OTLP trace exporter -const traceExporter = new OTLPTraceExporter({ - url: endpoint, - compression: process.env.OTEL_EXPORTER_OTLP_COMPRESSION as CompressionAlgorithm, - headers: exporterHeaders, - metadata: grpcMetadata, -}) +if (endpoint) { + // Create an OTLP trace exporter + traceExporter = new OTLPTraceExporter({ + url: endpoint, + compression: process.env.OTEL_EXPORTER_OTLP_COMPRESSION as CompressionAlgorithm, + headers: exporterHeaders, + metadata: grpcMetadata, + }) +} + +// Create a BatchSpanProcessor using the trace exporter +const batchProcessor = traceExporter ? new BatchSpanProcessor(traceExporter) : undefined // Configure the OpenTelemetry Node SDK const sdk = new NodeSDK({ @@ -55,6 +63,7 @@ const sdk = new NodeSDK({ [SEMRESATTRS_SERVICE_NAME]: 'storage', [SEMRESATTRS_SERVICE_VERSION]: version, }), + spanProcessors: batchProcessor ? [batchProcessor, traceCollector] : [traceCollector], traceExporter, instrumentations: [ new HttpInstrumentation({ @@ -120,12 +129,12 @@ const sdk = new NodeSDK({ ], }) -if (process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) { +if (process.env.TRACING_ENABLED === 'true') { // Initialize the OpenTelemetry Node SDK sdk.start() // Gracefully shutdown the SDK on process exit - process.on('SIGTERM', () => { + process.once('SIGTERM', () => { logSchema.info(logger, '[Otel] Stopping', { type: 'otel', }) diff --git a/src/storage/protocols/s3/signature-v4.ts b/src/storage/protocols/s3/signature-v4.ts index 71fec144..e453a2e8 100644 --- a/src/storage/protocols/s3/signature-v4.ts +++ b/src/storage/protocols/s3/signature-v4.ts @@ -325,7 +325,14 @@ export class SignatureV4 { const xForwardedHost = this.getHeader(request, 'x-forwarded-host') if (xForwardedHost) { - return `host:${xForwardedHost.toLowerCase()}` + const url = new URL(request.url) + const port = this.getHeader(request, 'x-forwarded-port') + const host = `host:${xForwardedHost.toLowerCase()}` + + if (port && url.port) { + return host + ':' + port + } + return host } return `host:${this.getHeader(request, 'host')}` diff --git a/tsconfig.json b/tsconfig.json index 78433b52..cfb98e02 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -13,7 +13,7 @@ "rootDir": "src", "moduleResolution": "node", "module": "commonjs", - "target": "ES2020", + "target": "ES2021", "outDir": "dist", "sourceMap": true, "strict": true,