Tracking pageviews with middleware

app/api/track/route.ts

@@ -0,0 +1,13 @@
+import { PageViewSchema, trackPageView } from 'lib/databases/analytics';
+
+export const POST = async (req: Request) => {
+  const _body = await req.json();
+
+  const data = await PageViewSchema.validate(_body, { stripUnknown: true }).catch(() => {});
+
+  if (data) {
+    await trackPageView(data).catch(() => {});
+  }
+
+  return new Response('OK');
+};

lib/databases/analytics.ts

@@ -0,0 +1,24 @@
+import { InferType, object, string } from 'yup';
+import { getClickHouse } from './clickhouse';
+
+const client = getClickHouse();
+
+export const PageViewSchema = object({
+  path: string().required(),
+  affiliate: string(),
+  referrer: string(),
+  agent: string(),
+  hostname: string().required(),
+});
+
+export type PageViewData = InferType<typeof PageViewSchema>;
+
+const tableName = 'page_view';
+
+export const trackPageView = async (data: PageViewData) => {
+  await client.insert({
+    table: tableName,
+    values: [data],
+    format: 'JSONEachRow',
+  });
+};

lib/databases/clickhouse.ts

@@ -0,0 +1,12 @@
+import { createClient } from '@clickhouse/client';
+
+let client: undefined | ReturnType<typeof createClient>;
+
+export const getClickHouse = () => {
+  if (!client) {
+    client = createClient({
+      url: process.env.CLICKHOUSE_URL,
+    });
+  }
+  return client;
+};

lib/databases/table.md

@@ -0,0 +1,15 @@
+Clickhousedb table for tracking page views
+
+```sql
+CREATE TABLE page_view (
+    event_date Date DEFAULT today(),                -- Stores the date of the event, defaulting to the current date
+    event_date_time DateTime DEFAULT now(),         -- Timestamp of the event
+    path String,                                    -- Path of the page
+    affiliate String,                                -- Affiliate identifier
+    referrer String,                                -- URL of the referrer
+    agent String,                                   -- User agent of the client
+    hostname String                                 -- Hostname where the event was recorded
+) ENGINE = MergeTree()
+PARTITION BY toYYYYMM(event_date)
+ORDER BY (event_date, event, customer, source);     -- Adjust the ORDER BY clause based on your query patterns
+```

middleware.ts

@@ -1,7 +1,107 @@
+import { PageViewData } from 'lib/databases/analytics';
 import { defaultLocale, localePrefix, locales } from 'lib/i18n/config';
 import createMiddleware from 'next-intl/middleware';
+import { NextRequest } from 'next/server';
 
-export default createMiddleware({ locales, localePrefix, defaultLocale });
+export const middleware = (request: NextRequest) => {
+  const isPageView = isValidPageView(request);
+
+  if (isPageView) {
+    trackPageView(request);
+  }
+
+  return createMiddleware({ locales, localePrefix, defaultLocale })(request);
+};
+
+const trackPageView = async (request: NextRequest) => {
+  const server = getURL('/api/track');
+
+  const url = new URL(request.url);
+  const path = url.pathname;
+  const referrer = request.headers.get('referer');
+  const agent = request.headers.get('user-agent');
+
+  const data: PageViewData = {
+    path,
+    ...(referrer && { referrer }),
+    ...(referrer && { referrer }),
+    ...(agent && { agent }),
+    hostname: server.hostname,
+  };
+
+  // Send data to /api/track
+  fetch(server.href, {
+    method: 'POST',
+    headers: {
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify({
+      name: 'page-view',
+      ...data,
+    }),
+  }).catch(() => {});
+};
+
+/**
+ * Get the URL of the current (vercel) deployment
+ */
+const getURL = (path?: `/${string}`) => {
+  const hostname =
+    process.env.NEXT_PUBLIC_VERCEL_PROJECT_PRODUCTION_URL ||
+    process.env.VERCEL_PROJECT_PRODUCTION_URL ||
+    process.env.VERCEL_URL ||
+    'localhost:3000';
+  const isDev = process.env.NODE_ENV !== 'production';
+
+  const protocol = isDev ? 'http' : 'https';
+
+  return new URL(`${protocol}://${hostname}${path || ''}`);
+};
+
+const isValidPageView = (request: NextRequest) => {
+  // List of regex patterns to match against the user-agent header
+  const botUserAgents = [
+    '.*bot.*',
+    '.*preview.*',
+    '.*crawler.*',
+    '.*google.*',
+    '.*bing.*',
+    '.*yahoo.*',
+    '.*baidu.*',
+    '.*yandex.*',
+    '.*duckduckgo.*',
+    '.*facebook.*',
+    '.*twitter.*',
+    '.*instagram.*',
+    '.*pinterest.*',
+    '.*linkedin.*',
+  ];
+
+  // Check if the user-agent header matches any of the bot patterns
+  const isBot = botUserAgents.some((pattern) => {
+    const regex = new RegExp(pattern, 'i');
+    return regex.test(request.headers.get('user-agent'));
+  });
+  if (isBot) return false;
+
+  const ref = new URL(request.headers.get('referer'));
+  const url = new URL(request.url);
+
+  const isNavigating = request.headers.has('next-url');
+  if (isNavigating) return false;
+
+  // Check if the request is a navigation
+  const isNavigation = ref.pathname !== url.pathname;
+
+  if (isNavigation) return true;
+
+  // Check if the request is a page view
+  const isPageView = request.headers.get('sec-fetch-mode') === 'navigate';
+
+  if (!isPageView) return false;
+
+  return true;
+};
 
 export const config = {
   // Allow all paths starting with /address and apply exclusions to other paths

package.json

@@ -21,6 +21,7 @@
   },
   "private": true,
   "dependencies": {
+    "@clickhouse/client": "^1.4.1",
     "@headlessui/react": "^1.7.19",
     "@heroicons/react": "^2.1.3",
     "@revoke.cash/chains": "^47.0.0",
@@ -64,7 +65,8 @@
     "timeago.js": "^4.0.2",
     "use-local-storage": "^3.0.0",
     "viem": "^2.9.5",
-    "wagmi": "^2.5.13"
+    "wagmi": "^2.5.13",
+    "yup": "^1.4.0"
   },
   "devDependencies": {
     "@cypress/grep": "^4.0.1",

yarn.lock

@@ -143,6 +143,22 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@clickhouse/client-common@npm:1.4.1":
+  version: 1.4.1
+  resolution: "@clickhouse/client-common@npm:1.4.1"
+  checksum: 69bd0c9e3d4ab2135f73fa4a8e60bc5c1ee7ca4e7cd036e83b849d3a36166401c1d6da114da43e31c63addb5f9dabfe157b7d77a236644275a6b319a9c9d1e3b
+  languageName: node
+  linkType: hard
+
+"@clickhouse/client@npm:^1.4.1":
+  version: 1.4.1
+  resolution: "@clickhouse/client@npm:1.4.1"
+  dependencies:
+    "@clickhouse/client-common": 1.4.1
+  checksum: 462e37289f5d57fbb7d49aa1f1d1e4b2f05b76a8a19080e5bcd9b16b596472cc542fcd61da6eeab05b02f38b677e5d2e8a01564163a589b6a0351714323ab382
+  languageName: node
+  linkType: hard
+
 "@coinbase/wallet-sdk@npm:3.9.1":
   version: 3.9.1
   resolution: "@coinbase/wallet-sdk@npm:3.9.1"
@@ -9852,6 +9868,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"property-expr@npm:^2.0.5":
+  version: 2.0.6
+  resolution: "property-expr@npm:2.0.6"
+  checksum: 89977f4bb230736c1876f460dd7ca9328034502fd92e738deb40516d16564b850c0bbc4e052c3df88b5b8cd58e51c93b46a94bea049a3f23f4a022c038864cab
+  languageName: node
+  linkType: hard
+
 "property-information@npm:^5.0.0":
   version: 5.6.0
   resolution: "property-information@npm:5.6.0"
@@ -10626,6 +10649,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "revoke.cash@workspace:."
   dependencies:
+    "@clickhouse/client": ^1.4.1
     "@cypress/grep": ^4.0.1
     "@headlessui/react": ^1.7.19
     "@heroicons/react": ^2.1.3
@@ -10701,6 +10725,7 @@ __metadata:
     viem: ^2.9.5
     wagmi: ^2.5.13
     walkdir: ^0.4.1
+    yup: ^1.4.0
   languageName: unknown
   linkType: soft
 
@@ -11761,6 +11786,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"tiny-case@npm:^1.0.3":
+  version: 1.0.3
+  resolution: "tiny-case@npm:1.0.3"
+  checksum: 3f7a30c39d5b0e1bc097b0b271bec14eb5b836093db034f35a0de26c14422380b50dc12bfd37498cf35b192f5df06f28a710712c87ead68872a9e37ad6f6049d
+  languageName: node
+  linkType: hard
+
 "tippy.js@npm:^6.3.1":
   version: 6.3.7
   resolution: "tippy.js@npm:6.3.7"
@@ -11802,6 +11834,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"toposort@npm:^2.0.2":
+  version: 2.0.2
+  resolution: "toposort@npm:2.0.2"
+  checksum: d64c74b570391c9432873f48e231b439ee56bc49f7cb9780b505cfdf5cb832f808d0bae072515d93834dd6bceca5bb34448b5b4b408335e4d4716eaf68195dcb
+  languageName: node
+  linkType: hard
+
 "totalist@npm:^3.0.0":
   version: 3.0.1
   resolution: "totalist@npm:3.0.1"
@@ -11923,6 +11962,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"type-fest@npm:^2.19.0":
+  version: 2.19.0
+  resolution: "type-fest@npm:2.19.0"
+  checksum: a4ef07ece297c9fba78fc1bd6d85dff4472fe043ede98bd4710d2615d15776902b595abf62bd78339ed6278f021235fb28a96361f8be86ed754f778973a0d278
+  languageName: node
+  linkType: hard
+
 "type-fest@npm:^3.0.0":
   version: 3.13.1
   resolution: "type-fest@npm:3.13.1"
@@ -12998,6 +13044,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"yup@npm:^1.4.0":
+  version: 1.4.0
+  resolution: "yup@npm:1.4.0"
+  dependencies:
+    property-expr: ^2.0.5
+    tiny-case: ^1.0.3
+    toposort: ^2.0.2
+    type-fest: ^2.19.0
+  checksum: 20a2ee0c1e891979ca16b34805b3a3be9ab4bea6ea3d2f9005b998b4dc992d0e4d7b53e5f4d8d9423420046630fb44fdf0ecf7e83bc34dd83392bca046c5229d
+  languageName: node
+  linkType: hard
+
 "zustand@npm:4.4.1":
   version: 4.4.1
   resolution: "zustand@npm:4.4.1"