diff --git a/backend/cache/dist/utils/validation.js b/backend/cache/dist/common/blob.js similarity index 100% rename from backend/cache/dist/utils/validation.js rename to backend/cache/dist/common/blob.js diff --git a/backend/cache/dist/common/cdn-metrics.js b/backend/cache/dist/common/cdn-metrics.js new file mode 100644 index 0000000..9e92276 --- /dev/null +++ b/backend/cache/dist/common/cdn-metrics.js @@ -0,0 +1,107 @@ +import dotenv from 'dotenv'; +import { promises as fs } from 'fs'; +import path from 'path'; +const HOME_DIR = process.env.HOME || process.env.USERPROFILE || ''; +dotenv.config({ path: path.join(HOME_DIR, 'suiftly-ops', 'dotenv', '.env.metrics') }); +// Type guard on string to supported Route conversion. +export const ROUTES = ['blob', 'icon48x48', 'icon96x96', 'icon256x256', 'meta', 'metrics', 'view']; +export const isValidRoute = (route) => { + return ROUTES.includes(route); +}; +export async function loadMetricsFile(metricsPath, options = {}) { + const { verbose = false } = options; + try { + const metricsData = await fs.readFile(metricsPath, 'utf-8'); + const metrics = JSON.parse(metricsData); + if (verbose) { + console.log('Metrics loaded:', metrics); + } + return metrics; + } + catch (err) { + if (verbose) { + console.error('Error reading metrics file:', err); + } + return undefined; + } +} +export function isValidMetricsType(metrics, options = {}) { + const { verbose = false } = options; + if (verbose) { + console.log('Validating metrics:', metrics); + } + if (!metrics || typeof metrics !== 'object') { + if (verbose) { + console.error('Metrics is not an object'); + } + return false; + } + // Cast to Metrics (this does not validate anything). + // Must validate the object properties one by one. + const m = metrics; + // Check top properties + if (typeof m.blobId !== 'string') { + if (verbose) { + console.error('blobId is not a string'); + } + return false; + } + if (typeof m.daily !== 'object' || m.daily === null) { + if (verbose) { + console.error('daily is not an object'); + } + return false; + } + if (typeof m.sizes !== 'object' || m.sizes === null) { + if (verbose) { + console.error('sizes is not an object'); + } + return false; + } + // Check each date in daily + for (const date in m.daily) { + if (typeof date !== 'string') { + if (verbose) { + console.error('date is not a string'); + } + return false; + } + const routes = m.daily[date]; + if (typeof routes !== 'object' || routes === null) { + if (verbose) { + console.error('routes is not an object'); + } + return false; + } + // Check each route in daily[date] + for (const route in routes) { + const routeDay = routes[route]; + if (typeof routeDay !== 'object' || routeDay === null) { + if (verbose) { + console.error('routeDay is not an object'); + } + return false; + } + if (typeof routeDay.hits !== 'number' || + typeof routeDay.hitsEdge !== 'number' || + typeof routeDay.visitors !== 'number') { + if (verbose) { + console.error(`routeDay properties are not numbers for route [${route}]`); + } + return false; + } + } + } + // Check each route in sizes + for (const route in m.sizes) { + const size = m.sizes[route]; + if (size !== undefined && typeof size !== 'number') { + if (verbose) { + console.error(`optional size property is not a number for route [${route}]`); + } + return false; + } + } + return true; +} +export const CDN_METRICS_DIR = process.env.CDN_METRICS_DIR || `${HOME_DIR}/cdn-metrics`; diff --git a/backend/cache/dist/utils/strings.js b/backend/cache/dist/common/strings.js similarity index 100% rename from backend/cache/dist/utils/strings.js rename to backend/cache/dist/common/strings.js diff --git a/backend/cache/dist/common/utils.js b/backend/cache/dist/common/utils.js new file mode 100644 index 0000000..482e0b6 --- /dev/null +++ b/backend/cache/dist/common/utils.js @@ -0,0 +1,36 @@ +// Util function to sort object keys (in-place). +// Handles nested objects and arrays recursively. +// Useful to sort prior to JSON.stringify +export function sortObjectKeys(obj) { + if (typeof obj !== 'object' || obj === null) { + return; + } + if (Array.isArray(obj)) { + obj.forEach(sortObjectKeys); + return; + } + const keys = Object.keys(obj); + const sortedKeys = [...keys].sort(); + // Check if the keys are already sorted + let isSorted = true; + for (let i = 0; i < keys.length; i++) { + if (keys[i] !== sortedKeys[i]) { + isSorted = false; + break; + } + } + if (!isSorted) { + sortedKeys.forEach(key => { + const value = obj[key]; + delete obj[key]; + obj[key] = value; + sortObjectKeys(value); + }); + } + else { + // If already sorted, still need to sort nested objects + keys.forEach(key => { + sortObjectKeys(obj[key]); + }); + } +} diff --git a/backend/cache/dist/controllers/blob.js b/backend/cache/dist/controllers/blob.js index db638ab..21fbb10 100644 --- a/backend/cache/dist/controllers/blob.js +++ b/backend/cache/dist/controllers/blob.js @@ -1,8 +1,8 @@ import { spawn } from 'child_process'; import * as fs from 'fs'; import path from 'path'; -import { getIdPrefixes } from '../utils/strings.js'; -import { isValidBlobId } from '../utils/validation.js'; +import { isValidBlobId } from '../common/blob.js'; +import { getIdPrefixes } from '../common/strings.js'; // Test Blob Info // // MIME type: image/png diff --git a/backend/cache/dist/controllers/metrics.js b/backend/cache/dist/controllers/metrics.js index 012b07a..4319b3a 100644 --- a/backend/cache/dist/controllers/metrics.js +++ b/backend/cache/dist/controllers/metrics.js @@ -1,7 +1,7 @@ import * as fs from 'fs'; import path from 'path'; -import { getIdPrefixes } from '../utils/strings.js'; -import { isValidBlobId } from '../utils/validation.js'; +import { isValidBlobId } from '../common/blob.js'; +import { getIdPrefixes } from '../common/strings.js'; // Test Blob Info // // MIME type: image/png @@ -63,8 +63,8 @@ export const getMetrics = async (req, res) => { // File does not exist, return "no_metrics" JSON response res.json({ blobId: id, - status: 'no_metrics', message: 'No metrics accumulated yet for this blob. Metrics updated every ~24 hours.', + status: 'no_metrics', }); } }; diff --git a/backend/cache/dist/controllers/view.js b/backend/cache/dist/controllers/view.js new file mode 100644 index 0000000..e5f7d30 --- /dev/null +++ b/backend/cache/dist/controllers/view.js @@ -0,0 +1,243 @@ +import { format, subDays } from 'date-fns'; +import path from 'path'; +import { isValidBlobId } from '../common/blob.js'; +import { isValidMetricsType, loadMetricsFile } from '../common/cdn-metrics.js'; +import { getIdPrefixes } from '../common/strings.js'; +// Test Blob Info +// +// MIME type: image/png +// +// Blob ID: 0nzvRVLeF0I5kbWO3s_VDa-ixYZ_nhkp4J2EubJUtjo +// Unencoded size: 165 KiB +// Encoded size (including metadata): 62.0 MiB +// Sui object ID: 0x0ebad3b13ee9bc64f8d6370e71d3408a43febaa017a309d2367117afe144ae8c +// Cache-Control value initialized once +//const viewCacheControl = process.env.VIEW_CACHE_CONTROL || 'public, max-age=10'; +export const getView = async (req, res) => { + const { id = '' } = req.params; + // Request validation + try { + await isValidBlobId(id); + } + catch (error) { + if (error instanceof Error) { + res.status(400).send(error.message); + } + else { + res.status(400).send('Unknown error'); + } + } + const { prefix_1, prefix_2 } = getIdPrefixes(id); + if (!prefix_1 || !prefix_2) { + // Should never happen because id was validated. + return res.status(500).send('Internal Server Error (prefix)'); + } + // Generate a self-sufficient single-HTML page that display + // a thumbnail 96x96 of the blob ~/cache/prefix1/prefix2/.blob + // and metrics from ~/cache/prefix1/prefix2/.metrics + // + // If the .blob does not exists, show an empty placeholder for now. + // + // If the .metrics do not exists, just display "No Metrics available yet". + // + // To know if the .blob is an image, parse the MIME type from the .json. + // (See blob.ts for the encoding). + // + // The page will have the Blob ID at the top (as the title). + // Below the title, the thumbnail of the blob. + // + // Metrics will have 3 charts: + // (1) Daily hits count (last 30 days) + // (2) Daily unique visitors count (last 30 days) + // (3) Daily CDN Hits Precentage (last 30 days) + // + // All charts are date(daily) x-axis + // - Always display 30 days. + // - For missing days in .metrics, use a placeholder value + // (0 for counts, 100% for percentage) + // - Each chart should have a title. + // + // The time the page was generated should be shown at the bottom. + // (use UTC time format). + // + // The format of .metrics is JSON. Example: + // { + // "blobId": "qjwu732...", + // "sizes": { + // "blob": 123456, + // "meta": 123456, + // }, + // "daily": { + // "2024-02-03": { + // "blob": { + // "hits": 123, + // "visitors": 2, + // "hitsEdge": 0, + // }, + // "meta": { + // "hits": 123, + // "visitors": 2, + // "hitsEdge": 0, + // }, + // ... + // } + // } + // } + // + // All data is coming from the daily.{data}.blob object. + // + // The CDN Hits Percentage is calculated as: (hitsEdge / hits * 100) and is 0% when hits is 0 + // when stats are not present for a given day. + // + const homePath = process.env.HOME || ''; + const metricsPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.metrics`); + //const jsonPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.json`); + //const blobPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.blob`); + try { + // Read the metrics file. Can fail but won't throw an error. + const metrics = await loadMetricsFile(metricsPath, { verbose: false }); + const isValidMetrics = isValidMetricsType(metrics, { verbose: false }); + if (!isValidMetrics) { + const html = ` + + + + ${id} + + +

No Metrics data available yet for blob [ ${id} ].

+ May take up to 24 hours for next update. + + + `; + // Send the HTML response + res.send(html); + return; + } + const today = new Date(); + const last30Days = Array.from({ length: 30 }, (_, i) => format(subDays(today, i + 1), 'yyyy-MM-dd')).reverse(); + // Calculate CDN Hits Percentage + const blobHits = []; + const blobVisitors = []; + const blobCdnHitsPercentage = []; + last30Days.forEach(date => { + // If the date is not in the metrics, use a placeholder + const dayMetrics = (isValidMetrics && metrics.daily[date]?.blob) || { hits: 0, hitsEdge: 0, visitors: 0 }; + blobHits.push(dayMetrics.hits); + blobVisitors.push(dayMetrics.visitors); + const cdnHitsPercentage = dayMetrics.hits === 0 ? 0 : (dayMetrics.hitsEdge / dayMetrics.hits) * 100; + blobCdnHitsPercentage.push(cdnHitsPercentage); + }); + // Remove the leading yyyy- in each element of last30Days + last30Days.forEach((date, i) => { + last30Days[i] = date.slice(5); + }); + // Generate HTML with Charts + const html = ` + + + + ${id} + + + + +

Dashboard for ${id}

+
+ +
+
+ +
+
+ +
+ + + + `; + // Send the HTML response + res.send(html); + } + catch (error) { + console.error('Error reading metrics:', error); + res.status(500).send('Internal Server Error'); + } +}; diff --git a/backend/cache/dist/routes/viewRoutes.js b/backend/cache/dist/routes/viewRoutes.js new file mode 100644 index 0000000..17977d1 --- /dev/null +++ b/backend/cache/dist/routes/viewRoutes.js @@ -0,0 +1,8 @@ +import express from 'express'; +import { getView } from '../controllers/view.js'; +export const viewRoutes = express.Router(); +viewRoutes.get('/', (req, res) => { + res.set('Cache-Control', 'public, max-age=86400'); + res.send('Blob ID missing.

Useage: https://cdn.suiftly.io/view/<your_blob_id>'); +}); +viewRoutes.get('/:id', getView); diff --git a/backend/cache/dist/server.js b/backend/cache/dist/server.js index fae575e..e4027c9 100644 --- a/backend/cache/dist/server.js +++ b/backend/cache/dist/server.js @@ -3,6 +3,7 @@ import cors from 'cors'; import express from 'express'; import { blobRoutes } from './routes/blobRoutes.js'; import { metricsRoutes } from './routes/metricsRoutes.js'; +import { viewRoutes } from './routes/viewRoutes.js'; const port = process.env.PORT || 3000; const app = express(); app.use(cors()); @@ -19,6 +20,7 @@ app.get('/', (req, res) => { }); app.use('/blob', blobRoutes); app.use('/metrics', metricsRoutes); +app.use('/view', viewRoutes); app.listen(port, () => { console.log(`App listening on port: ${port}`); }); diff --git a/backend/cache/package.json b/backend/cache/package.json index 4b55d91..360f6e6 100644 --- a/backend/cache/package.json +++ b/backend/cache/package.json @@ -18,7 +18,8 @@ "cache": "link:", "cors": "^2.8.5", "dotenv": "^16.4.5", - "express": "^4.19.2" + "express": "^4.19.2", + "date-fns": "^3.0.0" }, "devDependencies": { "@types/cors": "^2.8.17", @@ -29,4 +30,4 @@ "tsx": "^4.17.0", "typescript": "^5.5.4" } -} \ No newline at end of file +} diff --git a/backend/cache/pnpm-lock.yaml b/backend/cache/pnpm-lock.yaml index baeb81a..e84585a 100644 --- a/backend/cache/pnpm-lock.yaml +++ b/backend/cache/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: cors: specifier: ^2.8.5 version: 2.8.5 + date-fns: + specifier: ^3.0.0 + version: 3.6.0 dotenv: specifier: ^16.4.5 version: 16.4.5 @@ -343,6 +346,9 @@ packages: resolution: {integrity: sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==} engines: {node: '>= 8'} + date-fns@3.6.0: + resolution: {integrity: sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww==} + debug@2.6.9: resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} peerDependencies: @@ -1043,6 +1049,8 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + date-fns@3.6.0: {} + debug@2.6.9: dependencies: ms: 2.0.0 diff --git a/backend/cache/src/common b/backend/cache/src/common new file mode 120000 index 0000000..c2ea83d --- /dev/null +++ b/backend/cache/src/common @@ -0,0 +1 @@ +../../common/src \ No newline at end of file diff --git a/backend/cache/src/controllers/blob.ts b/backend/cache/src/controllers/blob.ts index b935ba9..cfcd91e 100644 --- a/backend/cache/src/controllers/blob.ts +++ b/backend/cache/src/controllers/blob.ts @@ -2,8 +2,9 @@ import { spawn } from 'child_process'; import { Request, Response } from 'express'; import * as fs from 'fs'; import path from 'path'; -import { getIdPrefixes } from '@/utils/strings'; -import { isValidBlobId } from '@/utils/validation'; + +import { isValidBlobId } from '../common/blob'; +import { getIdPrefixes } from '../common/strings'; // Test Blob Info // diff --git a/backend/cache/src/controllers/metrics.ts b/backend/cache/src/controllers/metrics.ts index a16a4f5..196aee6 100644 --- a/backend/cache/src/controllers/metrics.ts +++ b/backend/cache/src/controllers/metrics.ts @@ -1,9 +1,9 @@ -import { spawn } from 'child_process'; import { Request, Response } from 'express'; import * as fs from 'fs'; import path from 'path'; -import { getIdPrefixes } from '@/utils/strings'; -import { isValidBlobId } from '@/utils/validation'; + +import { isValidBlobId } from '@/common/blob'; +import { getIdPrefixes } from '@/common/strings'; // Test Blob Info // @@ -70,8 +70,8 @@ export const getMetrics = async (req: Request, res: Response) => { // File does not exist, return "no_metrics" JSON response res.json({ blobId: id, - status: 'no_metrics', message: 'No metrics accumulated yet for this blob. Metrics updated every ~24 hours.', + status: 'no_metrics', }); } }; diff --git a/backend/cache/src/controllers/view.ts b/backend/cache/src/controllers/view.ts new file mode 100644 index 0000000..d0545fc --- /dev/null +++ b/backend/cache/src/controllers/view.ts @@ -0,0 +1,255 @@ +import { format, subDays } from 'date-fns'; +import { Request, Response } from 'express'; +import path from 'path'; + +import { isValidBlobId } from '@/common/blob'; +import { isValidMetricsType, loadMetricsFile } from '@/common/cdn-metrics'; +import { getIdPrefixes } from '@/common/strings'; + +// Test Blob Info +// +// MIME type: image/png +// +// Blob ID: 0nzvRVLeF0I5kbWO3s_VDa-ixYZ_nhkp4J2EubJUtjo +// Unencoded size: 165 KiB +// Encoded size (including metadata): 62.0 MiB +// Sui object ID: 0x0ebad3b13ee9bc64f8d6370e71d3408a43febaa017a309d2367117afe144ae8c + +// Cache-Control value initialized once +//const viewCacheControl = process.env.VIEW_CACHE_CONTROL || 'public, max-age=10'; + +export const getView = async (req: Request, res: Response) => { + const { id = '' } = req.params; + + // Request validation + try { + await isValidBlobId(id); + } catch (error) { + if (error instanceof Error) { + res.status(400).send(error.message); + } else { + res.status(400).send('Unknown error'); + } + } + + const { prefix_1, prefix_2 } = getIdPrefixes(id); + if (!prefix_1 || !prefix_2) { + // Should never happen because id was validated. + return res.status(500).send('Internal Server Error (prefix)'); + } + + // Generate a self-sufficient single-HTML page that display + // a thumbnail 96x96 of the blob ~/cache/prefix1/prefix2/.blob + // and metrics from ~/cache/prefix1/prefix2/.metrics + // + // If the .blob does not exists, show an empty placeholder for now. + // + // If the .metrics do not exists, just display "No Metrics available yet". + // + // To know if the .blob is an image, parse the MIME type from the .json. + // (See blob.ts for the encoding). + // + // The page will have the Blob ID at the top (as the title). + // Below the title, the thumbnail of the blob. + // + // Metrics will have 3 charts: + // (1) Daily hits count (last 30 days) + // (2) Daily unique visitors count (last 30 days) + // (3) Daily CDN Hits Precentage (last 30 days) + // + // All charts are date(daily) x-axis + // - Always display 30 days. + // - For missing days in .metrics, use a placeholder value + // (0 for counts, 100% for percentage) + // - Each chart should have a title. + // + // The time the page was generated should be shown at the bottom. + // (use UTC time format). + // + // The format of .metrics is JSON. Example: + // { + // "blobId": "qjwu732...", + // "sizes": { + // "blob": 123456, + // "meta": 123456, + // }, + // "daily": { + // "2024-02-03": { + // "blob": { + // "hits": 123, + // "visitors": 2, + // "hitsEdge": 0, + // }, + // "meta": { + // "hits": 123, + // "visitors": 2, + // "hitsEdge": 0, + // }, + // ... + // } + // } + // } + // + // All data is coming from the daily.{data}.blob object. + // + // The CDN Hits Percentage is calculated as: (hitsEdge / hits * 100) and is 0% when hits is 0 + // when stats are not present for a given day. + // + const homePath = process.env.HOME || ''; + const metricsPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.metrics`); + //const jsonPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.json`); + //const blobPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.blob`); + try { + // Read the metrics file. Can fail but won't throw an error. + const metrics = await loadMetricsFile(metricsPath, { verbose: false }); + const isValidMetrics = isValidMetricsType(metrics, { verbose: false }); + + if (!isValidMetrics) { + const html = ` + + + + ${id} + + +

No Metrics data available yet for blob [ ${id} ].

+ May take up to 24 hours for next update. + + + `; + // Send the HTML response + res.send(html); + return; + } + + const today = new Date(); + const last30Days = Array.from({ length: 30 }, (_, i) => format(subDays(today, i + 1), 'yyyy-MM-dd')).reverse(); + + // Calculate CDN Hits Percentage + const blobHits: number[] = []; + const blobVisitors: number[] = []; + const blobCdnHitsPercentage: number[] = []; + + last30Days.forEach(date => { + // If the date is not in the metrics, use a placeholder + const dayMetrics = (isValidMetrics && metrics.daily[date]?.blob) || { hits: 0, hitsEdge: 0, visitors: 0 }; + blobHits.push(dayMetrics.hits); + blobVisitors.push(dayMetrics.visitors); + const cdnHitsPercentage = dayMetrics.hits === 0 ? 0 : (dayMetrics.hitsEdge / dayMetrics.hits) * 100; + blobCdnHitsPercentage.push(cdnHitsPercentage); + }); + + // Remove the leading yyyy- in each element of last30Days + last30Days.forEach((date, i) => { + last30Days[i] = date.slice(5); + }); + + // Generate HTML with Charts + const html = ` + + + + ${id} + + + + +

Dashboard for ${id}

+
+ +
+
+ +
+
+ +
+ + + + `; + + // Send the HTML response + res.send(html); + } catch (error) { + console.error('Error reading metrics:', error); + res.status(500).send('Internal Server Error'); + } +}; diff --git a/backend/cache/src/routes/viewRoutes.ts b/backend/cache/src/routes/viewRoutes.ts new file mode 100644 index 0000000..e13e448 --- /dev/null +++ b/backend/cache/src/routes/viewRoutes.ts @@ -0,0 +1,12 @@ +import express from 'express'; + +import { getView } from '@/controllers/view'; + +export const viewRoutes = express.Router(); + +viewRoutes.get('/', (req, res) => { + res.set('Cache-Control', 'public, max-age=86400'); + res.send('Blob ID missing.

Useage: https://cdn.suiftly.io/view/<your_blob_id>'); +}); + +viewRoutes.get('/:id', getView); diff --git a/backend/cache/src/server.ts b/backend/cache/src/server.ts index 8704ef3..9454153 100644 --- a/backend/cache/src/server.ts +++ b/backend/cache/src/server.ts @@ -4,7 +4,9 @@ import cors from 'cors'; import express from 'express'; import { blobRoutes } from '@/routes/blobRoutes'; + import { metricsRoutes } from './routes/metricsRoutes'; +import { viewRoutes } from './routes/viewRoutes'; const port = process.env.PORT || 3000; @@ -33,6 +35,7 @@ app.get('/', (req, res) => { app.use('/blob', blobRoutes); app.use('/metrics', metricsRoutes); +app.use('/view', viewRoutes); app.listen(port, () => { console.log(`App listening on port: ${port}`); diff --git a/backend/cdn-metrics/src/common b/backend/cdn-metrics/src/common new file mode 120000 index 0000000..c2ea83d --- /dev/null +++ b/backend/cdn-metrics/src/common @@ -0,0 +1 @@ +../../common/src \ No newline at end of file diff --git a/backend/cdn-metrics/src/server.ts b/backend/cdn-metrics/src/server.ts index 112e7e2..c34d176 100644 --- a/backend/cdn-metrics/src/server.ts +++ b/backend/cdn-metrics/src/server.ts @@ -1,6 +1,17 @@ // Download, unzip, verify and maintain metrics in various directories // depending of its age and validity. // +// Summary +// ------- +// All metrics (up to one year) are kept in local file system. +// +// Data flows: +// DOWNLOAD_DIR -(unzip)-> UNZIP_DIR -(validated)-> RECENT_DIR +// DOWNLOAD_DIR -(old and present in RECENT_DIR)-> ARCHIVE_DIR +// RECENT_DIR files -(parsed into ~/cache)-> On success adds .done extension +// +// Specs +// ----- // The file is downloaded from the BunnyCDN logging endpoint like this: // https://logging.bunnycdn.com/{mm}-{dd}-{yy}/pull_zone_id.log // @@ -43,52 +54,15 @@ import { pipeline } from 'stream/promises'; import { promisify } from 'util'; import zlib from 'zlib'; +import * as metrics from './common/cdn-metrics'; +import { sortObjectKeys } from './common/utils'; + const EXPIRATION_RECENT_DIR = 40 * 24 * 60 * 60 * 1000; // 40 days in milliseconds const EXPIRATION_UNZIP_DIR = 40 * 24 * 60 * 60 * 1000; const EXPIRATION_DOWNLOAD_DIR = 37 * 24 * 60 * 60 * 1000; // 37 days in milliseconds const readdir = promisify(fs.readdir); -// Util function to sort object keys (in-place). -// Handles nested objects and arrays recursively. -// Useful to sort prior to JSON.stringify -function sortObjectKeys(obj: any): void { - if (typeof obj !== 'object' || obj === null) { - return; - } - - if (Array.isArray(obj)) { - obj.forEach(sortObjectKeys); - return; - } - - const keys = Object.keys(obj); - const sortedKeys = [...keys].sort(); - - // Check if the keys are already sorted - let isSorted = true; - for (let i = 0; i < keys.length; i++) { - if (keys[i] !== sortedKeys[i]) { - isSorted = false; - break; - } - } - - if (!isSorted) { - sortedKeys.forEach(key => { - const value = obj[key]; - delete obj[key]; - obj[key] = value; - sortObjectKeys(value); - }); - } else { - // If already sorted, still need to sort nested objects - keys.forEach(key => { - sortObjectKeys(obj[key]); - }); - } -} - // Load more .env specific to this app (and interpolate $HOME) const HOME_DIR = process.env.HOME || process.env.USERPROFILE || ''; dotenv.config({ path: path.join(HOME_DIR, 'suiftly-ops', 'dotenv', '.env.metrics') }); @@ -99,54 +73,17 @@ for (const key in process.env) { } } -// Type guard on string to supported Route conversion. -const ROUTES = ['blob', 'icon48x48', 'icon96x96', 'icon256x256', 'meta', 'metrics', 'view'] as const; -type Route = (typeof ROUTES)[number]; -const isValidRoute = (route: string): route is Route => { - return ROUTES.includes(route as Route); -}; - -// Defines how metrics are stored in-memory and on disk (JSON). -interface RouteDayMetrics { - hits: number; - hitsEdge: number; - visitors: number; - visitors_set?: Set; -} - -interface DailyMetrics { - [date: string]: Partial>; -} - -interface Metrics { - blobId: string; - daily: DailyMetrics; - sizes: Partial>; -} - -// In-memory map only for while parsing a log file. -interface MetricsMap { - [blobId: string]: Metrics; -} - -// All metrics (up to one year) are kept in local file system. -// -// The data flows are: -// DOWNLOAD_DIR -(unzip)-> UNZIP_DIR -(validated)-> RECENT_DIR -// DOWNLOAD_DIR -(present in RECENT_DIR)-> ARCHIVE_DIR -const CDN_METRICS_DIR = process.env.CDN_METRICS_DIR || `${HOME_DIR}/cdn-metrics`; - // Last 30 days downloaded files (compressed) -const DOWNLOAD_DIR = process.env.CDN_METRICS_DOWNLOAD_DIR || `${CDN_METRICS_DIR}/download`; +const DOWNLOAD_DIR = process.env.CDN_METRICS_DOWNLOAD_DIR || `${metrics.CDN_METRICS_DIR}/download`; // Intermediate directory to validate the unzip of a given day. -const UNZIP_DIR = process.env.CDN_METRICS_UNZIP_DIR || `${CDN_METRICS_DIR}/unzipped`; +const UNZIP_DIR = process.env.CDN_METRICS_UNZIP_DIR || `${metrics.CDN_METRICS_DIR}/unzipped`; // Last 30 days validated unzipped logs. -const RECENT_DIR = process.env.CDN_METRICS_RECENT_DIR || `${CDN_METRICS_DIR}/recent`; +const RECENT_DIR = process.env.CDN_METRICS_RECENT_DIR || `${metrics.CDN_METRICS_DIR}/recent`; // Last one year zipper logs. -const ARCHIVE_DIR = process.env.CDN_METRICS_ARCHIVE_DIR || `${CDN_METRICS_DIR}/archive`; +const ARCHIVE_DIR = process.env.CDN_METRICS_ARCHIVE_DIR || `${metrics.CDN_METRICS_DIR}/archive`; // URL fragments used to download daily metrics. const PREFIX_URL = process.env.CDN_METRICS_PREFIX_URL || ''; @@ -185,7 +122,7 @@ function prepareSetup(): boolean { return false; } - if (!CDN_METRICS_DIR) { + if (!metrics.CDN_METRICS_DIR) { console.error('Missing environment variables: CDN_METRICS_DIR'); return false; } @@ -533,7 +470,7 @@ async function updateMetrics() { input: fileStream, }); - const metricsMap: MetricsMap = {}; + const metricsMap: metrics.MetricsMap = {}; for await (const line of rl) { if (!line.trim()) continue; // Skip empty lines @@ -554,11 +491,11 @@ async function updateMetrics() { if (!route || !blobId) continue; // Skip lines with missing URL fragments // Convert route string to Route type. - if (!isValidRoute(route)) { + if (!metrics.isValidRoute(route)) { console.error('Invalid route:', route); continue; // Skip lines with non-supported route } - const routeType = route as Route; + const routeType = route as metrics.Route; // TODO Sanity check the blobId. @@ -652,8 +589,8 @@ async function updateMetrics() { // Remove elements in sizes that are zero. // At least one element should exists. for (const route in blobMetrics.sizes) { - if (blobMetrics.sizes[route as Route] === 0) { - delete blobMetrics.sizes[route as Route]; + if (blobMetrics.sizes[route as metrics.Route] === 0) { + delete blobMetrics.sizes[route as metrics.Route]; } } if (Object.keys(blobMetrics.sizes).length === 0) { @@ -665,7 +602,7 @@ async function updateMetrics() { const prefix2 = blobId.slice(2, 4); const metricsDir = path.join(HOME_DIR, 'cache', prefix1, prefix2); const metricsFilePath = path.join(metricsDir, `${blobId}.metrics`); - let fileMetrics: Metrics | undefined = undefined; + let fileMetrics: metrics.Metrics | undefined = undefined; if (!fs.existsSync(metricsDir)) { // That should never happen, but just in case... fs.mkdirSync(metricsDir, { recursive: true }); @@ -711,7 +648,7 @@ async function updateMetrics() { fileMetrics.sizes = blobMetrics.sizes; } else { for (const [route, size] of Object.entries(blobMetrics.sizes)) { - const routeType = route as Route; + const routeType = route as metrics.Route; if (size > 0 && size > (fileMetrics.sizes[routeType] || 0)) { fileMetrics.sizes[routeType] = size; } @@ -725,7 +662,7 @@ async function updateMetrics() { const dayMetrics = blobMetrics.daily[day]; for (const route in dayMetrics) { if (Object.prototype.hasOwnProperty.call(dayMetrics, route)) { - const routeMetrics = dayMetrics[route as Route]; + const routeMetrics = dayMetrics[route as metrics.Route]; if (routeMetrics) { delete routeMetrics.visitors_set; } @@ -747,7 +684,7 @@ async function updateMetrics() { fileMetrics.daily[day] = fileMetricsDay; } for (const route in dayMetrics) { - const routeType = route as Route; + const routeType = route as metrics.Route; fileMetricsDay[routeType] = dayMetrics[routeType]; } } diff --git a/backend/cache/src/utils/validation.ts b/backend/common/src/blob.ts similarity index 100% rename from backend/cache/src/utils/validation.ts rename to backend/common/src/blob.ts diff --git a/backend/common/src/cdn-metrics.ts b/backend/common/src/cdn-metrics.ts new file mode 100644 index 0000000..8835949 --- /dev/null +++ b/backend/common/src/cdn-metrics.ts @@ -0,0 +1,170 @@ +import dotenv from 'dotenv'; +import { promises as fs } from 'fs'; +import path from 'path'; + +const HOME_DIR = process.env.HOME || process.env.USERPROFILE || ''; +dotenv.config({ path: path.join(HOME_DIR, 'suiftly-ops', 'dotenv', '.env.metrics') }); + +// Type guard on string to supported Route conversion. +export const ROUTES = ['blob', 'icon48x48', 'icon96x96', 'icon256x256', 'meta', 'metrics', 'view'] as const; +export type Route = (typeof ROUTES)[number]; +export const isValidRoute = (route: string): route is Route => { + return ROUTES.includes(route as Route); +}; + +// Defines how metrics are stored in-memory and on disk (JSON). + +// Metrics for a specicic blob, day and route (e.g. blob, view, icon48x48, etc). +export interface RouteDay { + hits: number; + hitsEdge: number; + visitors: number; + visitors_set?: Set; +} + +// All metrics related to a specific blob and day. +export interface Daily { + [date: string]: Partial>; +} + +// Top object for metrics related to a single blob. +export interface Metrics { + blobId: string; + daily: Daily; + sizes: Partial>; +} + +// Map of metrics for blobs. +export interface MetricsMap { + [blobId: string]: Metrics; +} + +// Load Metrics from a file. +// +// Caller is responsible to further validate the object. Consider isValidMetricsType(). +export interface LoadOptions { + verbose?: boolean; +} + +export async function loadMetricsFile(metricsPath: string, options: LoadOptions = {}): Promise { + const { verbose = false } = options; + try { + const metricsData = await fs.readFile(metricsPath, 'utf-8'); + const metrics: Metrics = JSON.parse(metricsData); + if (verbose) { + console.log('Metrics loaded:', metrics); + } + return metrics; + } catch (err) { + if (verbose) { + console.error('Error reading metrics file:', err); + } + return undefined; + } +} + +// Validate that an object (ususally coming from JSON.parse) is a Metrics object. +// +// Check only expected property types, not their values. +// +// Returns is a 'Typescript type predicate' to inform the compiler that +// the Metrics object is valid (or not) in the caller scope. +export interface ValidationOptions { + verbose?: boolean; +} + +export function isValidMetricsType(metrics: unknown, options: ValidationOptions = {}): metrics is Metrics { + const { verbose = false } = options; + + if (verbose) { + console.log('Validating metrics:', metrics); + } + + if (!metrics || typeof metrics !== 'object') { + if (verbose) { + console.error('Metrics is not an object'); + } + return false; + } + + // Cast to Metrics (this does not validate anything). + // Must validate the object properties one by one. + const m = metrics as Metrics; + + // Check top properties + if (typeof m.blobId !== 'string') { + if (verbose) { + console.error('blobId is not a string'); + } + return false; + } + + if (typeof m.daily !== 'object' || m.daily === null) { + if (verbose) { + console.error('daily is not an object'); + } + return false; + } + + if (typeof m.sizes !== 'object' || m.sizes === null) { + if (verbose) { + console.error('sizes is not an object'); + } + return false; + } + + // Check each date in daily + for (const date in m.daily) { + if (typeof date !== 'string') { + if (verbose) { + console.error('date is not a string'); + } + return false; + } + + const routes = m.daily[date]; + if (typeof routes !== 'object' || routes === null) { + if (verbose) { + console.error('routes is not an object'); + } + return false; + } + + // Check each route in daily[date] + for (const route in routes) { + const routeDay = routes[route as Route]; + if (typeof routeDay !== 'object' || routeDay === null) { + if (verbose) { + console.error('routeDay is not an object'); + } + return false; + } + + if ( + typeof routeDay.hits !== 'number' || + typeof routeDay.hitsEdge !== 'number' || + typeof routeDay.visitors !== 'number' + ) { + if (verbose) { + console.error(`routeDay properties are not numbers for route [${route}]`); + } + return false; + } + } + } + + // Check each route in sizes + for (const route in m.sizes) { + const size = m.sizes[route as Route]; + if (size !== undefined && typeof size !== 'number') { + if (verbose) { + console.error(`optional size property is not a number for route [${route}]`); + } + return false; + } + } + + return true; +} + +export const CDN_METRICS_DIR = process.env.CDN_METRICS_DIR || `${HOME_DIR}/cdn-metrics`; diff --git a/backend/cache/src/utils/strings.ts b/backend/common/src/strings.ts similarity index 100% rename from backend/cache/src/utils/strings.ts rename to backend/common/src/strings.ts diff --git a/backend/common/src/utils.ts b/backend/common/src/utils.ts new file mode 100644 index 0000000..b138dab --- /dev/null +++ b/backend/common/src/utils.ts @@ -0,0 +1,39 @@ +// Util function to sort object keys (in-place). +// Handles nested objects and arrays recursively. +// Useful to sort prior to JSON.stringify +export function sortObjectKeys(obj: any): void { + if (typeof obj !== 'object' || obj === null) { + return; + } + + if (Array.isArray(obj)) { + obj.forEach(sortObjectKeys); + return; + } + + const keys = Object.keys(obj); + const sortedKeys = [...keys].sort(); + + // Check if the keys are already sorted + let isSorted = true; + for (let i = 0; i < keys.length; i++) { + if (keys[i] !== sortedKeys[i]) { + isSorted = false; + break; + } + } + + if (!isSorted) { + sortedKeys.forEach(key => { + const value = obj[key]; + delete obj[key]; + obj[key] = value; + sortObjectKeys(value); + }); + } else { + // If already sorted, still need to sort nested objects + keys.forEach(key => { + sortObjectKeys(obj[key]); + }); + } +}