feat: show index stats as a table

Kukei-eu · Dec 25, 2023 · 00e2e0a · 00e2e0a
1 parent a079fa2
commit 00e2e0a
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 3 deletions.
diff --git a/dist/static/layout.css b/dist/static/layout.css
@@ -307,3 +307,19 @@ body.stats {
 	background: var(--color-background);
 	box-shadow: var(--box-shadow-bottom);
 }
+
+table {
+	border-collapse: collapse;
+	& th, & td  {
+		padding: 0.5rem;
+		text-align: center;
+	}
+
+	& th:first-child, & td:first-child {
+		text-align: left;
+	}
+
+	& td {
+		border-bottom: 1px solid var(--color-base-muted);
+	}
+}
diff --git a/dist/static/modules.js b/dist/static/modules.js
@@ -14,10 +14,10 @@ class FormatNumber extends HTMLElement {
 
 class FormatDate extends HTMLElement {
 	connectedCallback() {
-		this.innerHTML = this.formatNumber(this.innerHTML);
+		this.innerHTML = this.formatDate(this.innerHTML);
 	}
 
-	formatNumber(number) {
+	formatDate(number) {
 		try {
 			const parsed = new Date(number);
 			if (!isFinite(parsed)) {

diff --git a/dist/static/variables.css b/dist/static/variables.css
@@ -4,6 +4,7 @@
 
 	--color-background: oklch(1 0 0);
 	--color-base: oklch(0 0 0);
+	--color-base-muted: oklch(0.521 0 89.876);
 	--color-contrast-bg: oklch(0 0 0);
 	--color-contrast-fg: oklch(1 0 0);
 	--color-link: unset;
@@ -16,6 +17,7 @@
 	:root {
 		--color-background: oklch(0.25 0 0);
 		--color-base: oklch(0.84 0.06 152.17);
+		--color-base-muted: oklch(0.869 0.026 149.045);
 		--color-contrast-bg: oklch(0.84 0.06 152.17);
 		--color-contrast-fg: oklch(0.25 0 0);
 		--border-size: 4px;

diff --git a/functions/status/index.js b/functions/status/index.js
@@ -2,7 +2,7 @@ import Mustache from 'mustache';
 import template from './template.html';
 import { getDefaultViewData } from '../../lib/view.js';
 import {emitPageView} from '../../lib/plausible.js';
-import { getUnchecked, getIndexStats } from '../../lib/mongo.js';
+import {getUnchecked, getIndexStats, getCrawlHistory} from '../../lib/mongo.js';
 
 export const onRequestGet = async (context) => {
 	emitPageView(context);
@@ -11,6 +11,7 @@ export const onRequestGet = async (context) => {
 	const unchecked = await getUnchecked(env);
 	const uncheckedLang = unchecked === null ? 'unknown' : unchecked;
 	const viewDefaults = await getDefaultViewData(env);
+	const history = await getCrawlHistory(env);
 
 	const indexMap = new Map();
 	const finalStats = indexStats
@@ -42,6 +43,7 @@ export const onRequestGet = async (context) => {
 	const view = {
 		...viewDefaults,
 		title: 'Index statistics - kukei.eu',
+		history,
 		finalStats,
 		uncheckedLang,
 	};

diff --git a/functions/status/template.html b/functions/status/template.html
@@ -23,6 +23,33 @@ <h2>Index statistics</h2>
 		<li><format-number>{{magazinesPages}}</format-number> magazines pages</li>
 	</ul>
 	<p>Pages known but yet to be crawled: <format-number>{{ uncheckedLang }}</format-number></p>
+	<h2>Index over time</h2>
+	<table>
+		<thead>
+			<tr>
+				<th>Date</th>
+				<th>Crawled count</th>
+				<th>Known, pending</th>
+				<th>Indexed</th>
+				<th>Indexed: blogs</th>
+				<th>Index: docs</th>
+				<th>Index: magazines</th>
+			</tr>
+		</thead>
+		<tbody>
+			{{#history}}
+			<tr>
+				<td><format-date>{{ date }}</format-date></td>
+				<td><format-number>{{ crawledCount }}</format-number></td>
+				<td><format-number>{{ uncheckedLang }}</format-number></td>
+				<td><format-number>{{ totalPages }}</format-number></td>
+				<td><format-number>{{ blogPages }}</format-number></td>
+				<td><format-number>{{ docsPages }}</format-number></td>
+				<td><format-number>{{ magazinesPages }}</format-number></td>
+			</tr>
+			{{/history}}
+		</tbody>
+	</table>
 	<h2>Index sources list</h2>
 	<p>Index / url - last crawled at</p>
 	<ul class="list">

diff --git a/lib/mongo.js b/lib/mongo.js
@@ -28,6 +28,31 @@ export const getIndexStats = async (envs) => {
 	return response?.documents ?? [];
 };
 
+export const getCrawlHistory = async (envs) => {
+	const response = await callMongo(envs, 'aggregate', {
+		collection: `${envs.ATLAS_COLLECTION}-stats`,
+		database: envs.ATLAS_DB,
+		dataSource: envs.ATLAS_SOURCE,
+		pipeline: [
+			{
+				$sort: {
+					created: -1
+				}
+			},
+			{
+				$limit: 100
+			}
+		]
+	});
+
+	const documents = (response?.documents ?? []).map((el) => ({
+		...el,
+		date: (new Date(el.created)).toISOString()
+	}));
+
+	return documents;
+};
+
 export const getUnchecked = async (envs) => {
 	const response = await callMongo(envs, 'aggregate', {
 		collection: `${envs.ATLAS_COLLECTION}-links`,