Merge pull request #150 from Whats-Cookin/refactor/feed

feat(claim_feed): create new endpoint to list using a cursor and remove useless table joins

docs/swagger/swagger.yaml

@@ -123,6 +123,46 @@ paths:
               schema:
                 "$ref": "#/components/schemas/Claims"
 
+  "/api/claims/v3":
+    get:
+      tags:
+        - Claims
+      description: List the claims using a cursor instead of page number (more performant)
+      summary: List the claims
+      parameters:
+        - name: limit
+          in: query
+          schema:
+            type: integer
+            example: 100
+            minimum: 1
+            maximum: 10000
+            nullable: true
+        - name: search
+          in: query
+          schema:
+            type: string
+            nullable: true
+        - name: nextPage
+          in: query
+          schema:
+            type: string
+            format: byte
+            nullable: true
+      responses:
+        "200":
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  nextPage:
+                    type: string
+                    format: byte
+                    nullable: true
+                  claims:
+                    "$ref": "#/components/schemas/FeedClaimV3"
+
   "/api/claim":
     post:
       tags:
@@ -235,6 +275,12 @@ components:
       type: array
       items:
         "$ref": "#/components/schemas/Claim"
+
+    FeedClaimsV3:
+      type: array
+      items:
+        "$ref": "#/components/schemas/FeedClaimV3"
+
     Claim:
       type: object
       required:
@@ -290,6 +336,32 @@ components:
           type: string
         source_description:
           type: string
+
+    FeedClaimV3:
+      type: object
+      required:
+        - name
+        - link
+        - claim_id
+        - statement
+        - effective_date
+      properties:
+        name:
+          type: string
+        link:
+          type: string
+        claim_id:
+          type: integer
+          format: int32
+        statement:
+          type: string
+        stars:
+          type: integer
+          maximum: 5
+        effective_date:
+          type: string
+          format: date-time
+
     Image:
       type: object
       properties:

src/controllers/api.controller.ts

@@ -13,6 +13,8 @@ import { uploadImageToS3 } from "../utils/aws-s3";
 import { calculateBufferHash } from "../utils/hash";
 import { config } from "../config";
 
+const DEFAULT_LIMIT = 100;
+
 const claimDao = new ClaimDao();
 const nodeDao = new NodeDao();
 
@@ -214,3 +216,31 @@ export const claimsFeed = async (req: Request, res: Response, next: NextFunction
     passToExpressErrorHandler(err, next);
   }
 };
+
+export async function claimsFeedV3(req: Request, res: Response, next: NextFunction) {
+  try {
+    const { search, limit, nextPage } = parseAndValidateClaimsFeedV3Query(req.query);
+    const feedEntries = await nodeDao.getFeedEntriesV3(limit, nextPage, search);
+    return res.status(200).json(feedEntries);
+  } catch (err) {
+    passToExpressErrorHandler(err, next);
+  }
+}
+
+function parseAndValidateClaimsFeedV3Query(query: Request["query"]): {
+  limit: number;
+  nextPage: string | null;
+  search: string | null;
+} {
+  const limit = parseInt((query.limit || DEFAULT_LIMIT).toString());
+
+  if (Number.isNaN(limit) || limit <= 0 || limit > 10000) {
+    throw new createError.UnprocessableEntity("Invalid limit value");
+  }
+
+  const search = query.search ? decodeURIComponent(query.search.toString()) : null;
+
+  const nextPage = query.nextPage?.toString() || null;
+
+  return { limit, search, nextPage };
+}

src/dao/api.dao.ts

@@ -5,6 +5,8 @@ import { makeClaimSubjectURL } from "../utils";
 import { CreateClaimV2Dto } from "../middlewares/validators";
 import { ImageDto } from "../middlewares/validators/claim.validator";
 
+const MAX_POSSIBLE_CURSOR = "999999999999999999999999999999";
+
 interface ReportI {
   name: string;
   thumbnail: string;
@@ -254,6 +256,14 @@ interface FeedEntry {
   image_metadata: any | null;
 }
 
+interface FeedEntryV3 {
+  name: string;
+  link: string;
+  claim_id: number;
+  statement: string | null;
+  stars: number | null;
+  effective_date: Date | null;
+}
 // Node Dao is a Class to hold all the Prisma queries related to the Node model
 export class NodeDao {
   getNodes = async (page: number, limit: number) => {
@@ -389,6 +399,73 @@ export class NodeDao {
     }
   };
 
+  async getFeedEntriesV3(limit: number, cursor: string | null, query: string | null) {
+    try {
+      query = query ? `%${query}%` : null;
+      cursor = cursor ? Buffer.from(cursor, "base64").toString() : null;
+
+      const rawQ = Prisma.sql`
+        WITH RankedClaims AS (
+          SELECT
+            n.name AS name,
+            n."nodeUri" AS link,
+            c.id AS claim_id,
+            c.statement AS statement,
+            c.stars AS stars,
+            c."effectiveDate" AS effective_date,
+            ROW_NUMBER() OVER (PARTITION BY c.id) AS row_num,
+            CONCAT(COALESCE(to_char(c."effectiveDate", 'YYYYMMDDHH24MISS'), ''), c.id::TEXT) AS cursor
+          FROM "Claim" c
+          INNER JOIN "Edge" AS e ON c.id = e."claimId"
+          INNER JOIN "Node" AS n ON e."startNodeId" = n.id
+          WHERE
+            n."entType" != 'CLAIM'
+            AND e.label != 'source'
+            AND c."effectiveDate" IS NOT NULL
+            AND c.statement IS NOT NULL
+            AND n.name IS NOT NULL
+            AND n.name != ''
+            AND (
+              c.subject ILIKE COALESCE(${query}, '%') OR
+              c.statement ILIKE COALESCE(${query}, '%') OR
+              n.name ILIKE COALESCE(${query}, '%')
+            )
+          ORDER BY c."effectiveDate" DESC, c.id DESC
+        )
+        SELECT 
+          name,
+          link,
+          claim_id,
+          statement,
+          stars,
+          effective_date,
+          cursor
+        FROM RankedClaims
+        WHERE
+          row_num = 1
+          AND cursor < COALESCE(${cursor}, ${MAX_POSSIBLE_CURSOR})
+        LIMIT ${limit}
+      `;
+
+      const claims = await prisma.$queryRaw<(FeedEntryV3 & { cursor?: string })[]>(rawQ);
+
+      const lastCursor = claims.at(-1)?.cursor;
+      const nextPage = lastCursor && claims.length >= limit ? Buffer.from(lastCursor).toString("base64") : null;
+
+      for (let i = 0; i < claims.length; i++) {
+        delete claims[i].cursor;
+      }
+
+      return {
+        nextPage,
+        claims: claims as FeedEntryV3[],
+      };
+    } catch (error) {
+      console.error("Error fetching feed entries:", error);
+      throw new Error("Failed to fetch feed entries");
+    }
+  }
+
   getNodeById = async (nodeId: number) => {
     return await prisma.node.findUnique({
       where: {

src/routes/apiRoutes/api.route.ts

@@ -13,7 +13,7 @@ import {
   createClaimV2,
 } from "../../controllers";
 import { jwtVerify } from "../../middlewares";
-import { getAllClaims } from "../../controllers/api.controller";
+import { claimsFeedV3, getAllClaims } from "../../controllers/api.controller";
 import { claimPostSchema, joiValidator } from "../../middlewares/validators/claim.validator";
 import { upload } from "../../middlewares/upload/multer.upload";
 
@@ -26,6 +26,7 @@ router.get("/claim/:claimId?", claimGetById);
 router.get("/claims-all", getAllClaims);
 router.get("/claimsfeed", claimsGet);
 router.get("/claimsfeed2", claimsFeed);
+router.get("/claims/v3", claimsFeedV3);
 router.get("/node/search", searchNodes);
 router.get("/node/:nodeId?", getNodeById);
 router.get("/my-node", getNodeForLoggedInUser);