Feature/clustering dashboard (#398)

* First draft of Insights dashboard * Add backend router + models for getting raw queries * Add backend router + models for getting raw queries * Add retrieve_topics endpoint + dummy classification endpoint * Let frontend read backend dummy data * Modify return type on /topics endpoint * Add first draft of front-end/ backend topic modelling * Switch 4 -> 4o * Add loading wheel + pagination * Remove styled-components for MUI, fix spacing * Remove styled-components from package.json * Add example sentence to whole topic_df upfront since it's a quick operation * Remove debug line * Check for token before call * Make fetchTopicsData conform to other functions * Calculate correct number of examples * refactor backend * Expand synthetic query generation script * Fix pd.cut() bucketing issue * Add logic for process all time periods, return unclustered queries + modifies models * endpoints work; using litellm proxy * removed utils; fixed mypy * Add core backend logic for pulling Redis results * Frontend pulls data from Redis + working pop up to generate new data * Add regenerate button to top of each page + headers * made calls async * Add timestamps for new generation on frontend reading from Redis * Render new topics when they are freshly generated * Remove dummy api call, make time updated refresh live * New styles * minor colors and fonts fix * from co-working * Add timestamp to topicmodel backend logic * Convert query_datetime to strings for unclustered queries * Hook up backend to frontend, fix types + return lists of dicts * fixes * add pagination * Fixed render when no content avaialble + pagination issues * Removing old files + tempData * Fix typing errors * Fix typing errors * Ensure dicts contains strings for MyPy * fixed after merge * Added AI summary * fixes from review --------- Co-authored-by: Sid Ravinutala <[email protected]>
IDinsight · Aug 30, 2024 · e76dbf3 · e76dbf3
1 parent a26a9a8
commit e76dbf3
Show file tree

Hide file tree

Showing 21 changed files with 1,012 additions and 11 deletions.
diff --git a/admin_app/package-lock.json b/admin_app/package-lock.json
diff --git a/admin_app/src/app/dashboard/api.ts b/admin_app/src/app/dashboard/api.ts
@@ -21,6 +21,40 @@ const getOverviewPageData = async (period: Period, token: string) => {
   });
 };
 
+const fetchTopicsData = async (period: Period, token: string) => {
+  return fetch(`${NEXT_PUBLIC_BACKEND_URL}/dashboard/insights/${period}`, {
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${token}`,
+    },
+  }).then((response) => {
+    if (response.ok) {
+      let resp = response.json();
+      return resp;
+    } else {
+      throw new Error("Error fetching Topics data");
+    }
+  });
+};
+
+const generateNewTopics = async (period: Period, token: string) => {
+  return fetch(`${NEXT_PUBLIC_BACKEND_URL}/dashboard/insights/${period}/refresh`, {
+    method: "GET",
+    headers: {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${token}`,
+    },
+  }).then((response) => {
+    if (response.ok) {
+      let resp = response.json();
+      return resp;
+    } else {
+      throw new Error("Error generating Topics data");
+    }
+  });
+};
+
 const getPerformancePageData = async (period: Period, token: string) => {
   return fetch(`${NEXT_PUBLIC_BACKEND_URL}/dashboard/performance/${period}`, {
     method: "GET",
@@ -90,4 +124,6 @@ export {
   getPerformancePageData,
   getPerformanceDrawerData,
   getPerformanceDrawerAISummary,
+  fetchTopicsData,
+  generateNewTopics,
 };
diff --git a/admin_app/src/app/dashboard/components/Insights.tsx b/admin_app/src/app/dashboard/components/Insights.tsx
@@ -0,0 +1,135 @@
+import React from "react";
+import Grid from "@mui/material/Unstable_Grid2";
+import Topics from "./insights/Topics";
+import Queries from "./insights/Queries";
+import Box from "@mui/material/Box";
+import { useState } from "react";
+import { QueryData, Period, TopicModelingResponse } from "../types";
+import { generateNewTopics, fetchTopicsData } from "../api";
+import { useAuth } from "@/utils/auth";
+
+interface InsightProps {
+  timePeriod: Period;
+}
+
+const Insight: React.FC<InsightProps> = ({ timePeriod }) => {
+  const { token } = useAuth();
+  const [selectedTopicId, setSelectedTopicId] = useState<number | null>(null);
+  const [topicQueries, setTopicQueries] = useState<QueryData[]>([]);
+  const [refreshTimestamp, setRefreshTimestamp] = useState<string>("");
+  const [refreshing, setRefreshing] = useState<boolean>(false);
+  const [aiSummary, setAiSummary] = useState<string>("");
+
+  const [dataFromBackend, setDataFromBackend] = useState<TopicModelingResponse>({
+    data: [],
+    refreshTimeStamp: "",
+    unclustered_queries: [],
+  });
+
+  const runRefresh = () => {
+    setRefreshing(true);
+    generateNewTopics(timePeriod, token!).then((_) => {
+      const date = new Date();
+      setRefreshTimestamp(date.toLocaleString());
+      setRefreshing(false);
+    });
+  };
+
+  React.useEffect(() => {
+    if (token) {
+      fetchTopicsData(timePeriod, token).then((dataFromBackend) => {
+        setDataFromBackend(dataFromBackend);
+        if (dataFromBackend.data.length > 0) {
+          setSelectedTopicId(dataFromBackend.data[0].topic_id);
+        }
+      });
+    } else {
+      console.log("No token found");
+    }
+  }, [token, refreshTimestamp, timePeriod]);
+
+  React.useEffect(() => {
+    if (selectedTopicId !== null) {
+      const filterQueries = dataFromBackend.data.find(
+        (topic) => topic.topic_id === selectedTopicId,
+      );
+
+      if (filterQueries) {
+        setTopicQueries(filterQueries.topic_samples);
+        setAiSummary(filterQueries.topic_summary);
+      } else {
+        setTopicQueries([]);
+        setAiSummary("Not available.");
+      }
+    } else {
+      setTopicQueries([]);
+      setAiSummary("Not available.");
+    }
+  }, [dataFromBackend, selectedTopicId, refreshTimestamp, timePeriod]);
+
+  const topics = dataFromBackend.data.map(
+    ({ topic_id, topic_name, topic_popularity }) => ({
+      topic_id,
+      topic_name,
+      topic_popularity,
+    }),
+  );
+
+  return (
+    <Grid container sx={{ bgcolor: "grey.100", borderRadius: 2, mx: 0.5 }}>
+      <Grid
+        container
+        md={12}
+        columnSpacing={{ xs: 2 }}
+        sx={{ bgcolor: "white", borderRadius: 2, mx: 0.5, mt: 2, height: 400 }}
+      >
+        <Grid
+          md={3}
+          sx={{ p: 2, borderRight: 1, borderColor: "grey.300", borderWidth: 2 }}
+        >
+          <Topics
+            data={topics}
+            selectedTopicId={selectedTopicId}
+            onClick={setSelectedTopicId}
+            topicsPerPage={4}
+          />
+        </Grid>
+        <Grid md={9} sx={{ p: 2 }}>
+          <Queries
+            data={topicQueries}
+            onRefreshClick={runRefresh}
+            aiSummary={aiSummary}
+            lastRefreshed={dataFromBackend.refreshTimeStamp}
+            refreshing={refreshing}
+          />
+        </Grid>
+      </Grid>
+      <Grid
+        md={12}
+        height={400}
+        sx={{
+          bgcolor: "white",
+          borderRadius: 2,
+          mx: 0.5,
+          mt: 2,
+          justifyItems: "center",
+          justifySelf: "stretch",
+        }}
+      >
+        <Box
+          textAlign="center"
+          height="100%"
+          sx={{
+            display: "flex",
+            alignItems: "center",
+            justifyContent: "center",
+          }}
+        >
+          -- Chart - Coming Soon! --
+        </Box>
+      </Grid>
+    </Grid>
+  );
+};
+
+export default Insight;
diff --git a/admin_app/src/app/dashboard/components/insights/Queries.tsx b/admin_app/src/app/dashboard/components/insights/Queries.tsx
@@ -0,0 +1,171 @@
+import React from "react";
+import { Box } from "@mui/material";
+import Table from "@mui/material/Table";
+import TableBody from "@mui/material/TableBody";
+import TableCell from "@mui/material/TableCell";
+import TableContainer from "@mui/material/TableContainer";
+import TableHead from "@mui/material/TableHead";
+import TableRow from "@mui/material/TableRow";
+import { grey, orange } from "@mui/material/colors";
+import Typography from "@mui/material/Typography";
+import AutoAwesomeIcon from "@mui/icons-material/AutoAwesome";
+import Button from "@mui/material/Button";
+import { QueryData } from "../../types";
+import CircularProgress from "@mui/material/CircularProgress";
+
+interface QueriesProps {
+  data: QueryData[];
+  onRefreshClick: () => void;
+  lastRefreshed: string;
+  refreshing: boolean;
+  aiSummary: string;
+}
+
+interface AISummaryProps {
+  aiSummary: string;
+}
+const AISummary: React.FC<AISummaryProps> = ({ aiSummary }) => {
+  return (
+    <Box
+      sx={{
+        display: "flex",
+        flexGrow: 1,
+        flexDirection: "column",
+        maxheight: 400,
+        border: 1,
+        borderColor: "secondary.main",
+        borderRadius: 2,
+        background: "linear-gradient(to bottom, rgba(176,198,255,0.5), #ffffff)",
+        p: 2,
+        mb: 3,
+      }}
+    >
+      <Box sx={{ display: "flex", flexDirection: "row", mb: 2 }}>
+        <AutoAwesomeIcon sx={{ fontSize: 25, mr: 1, color: "#9eb2e5" }} />
+        <Typography
+          sx={{
+            lineHeight: "24px",
+            fontWeight: 600,
+            fontColor: "black",
+            textAlign: "center",
+          }}
+        >
+          AI Overview
+        </Typography>
+      </Box>
+      <Typography
+        sx={{
+          lineHeight: "15px",
+          fontWeight: 300,
+          fontSize: "small",
+          fontColor: "black",
+          textAlign: "left",
+        }}
+      >
+        {aiSummary}
+      </Typography>
+    </Box>
+  );
+};
+
+const Queries: React.FC<QueriesProps> = ({
+  data,
+  onRefreshClick,
+  lastRefreshed,
+  refreshing,
+  aiSummary,
+}) => {
+  const formattedLastRefreshed =
+    lastRefreshed.length > 0
+      ? Intl.DateTimeFormat("en-ZA", {
+          dateStyle: "short",
+          timeStyle: "short",
+        }).format(new Date(lastRefreshed))
+      : "Never";
+
+  return (
+    <Box sx={{ display: "flex", flexDirection: "column" }}>
+      <Box
+        sx={{
+          display: "flex",
+          flexDirection: "row",
+          justifyContent: "space-between",
+          mb: 2,
+        }}
+      >
+        <Box sx={{ fontSize: 22, fontWeight: 700 }}>Example Queries</Box>
+        <Box sx={{ display: "flex", flexDirection: "row" }}>
+          <Box
+            sx={{
+              display: "flex",
+              mr: 2,
+              fontSize: "small",
+              alignItems: "center",
+              color: grey[600],
+            }}
+          >
+            Last run: {formattedLastRefreshed}
+          </Box>
+          <Button
+            disabled={refreshing}
+            variant="contained"
+            sx={{
+              bgcolor: orange[500],
+              width: 180,
+              "&:hover": {
+                bgcolor: orange[700],
+              },
+            }}
+            onClick={onRefreshClick}
+          >
+            {refreshing ? <CircularProgress size={24} /> : "Re-run Discovery"}
+          </Button>
+        </Box>
+      </Box>
+      <AISummary aiSummary={aiSummary} />
+      <Box
+        sx={{
+          display: "flex",
+          flexDirection: "column",
+          overflow: "hidden",
+          overflowY: "scroll",
+          maxHeight: 200,
+        }}
+      >
+        {data.length > 0 ? (
+          <TableContainer sx={{ border: 1, borderColor: grey[300], borderRadius: 1 }}>
+            <Table size="small">
+              <TableHead>
+                <TableRow
+                  sx={{ bgcolor: grey[100], position: "sticky", top: 0, zIndex: 1 }}
+                >
+                  <TableCell sx={{ fontWeight: 800 }}>Timestamp</TableCell>
+                  <TableCell sx={{ fontWeight: 800 }}>User Question</TableCell>
+                </TableRow>
+              </TableHead>
+              <TableBody>
+                {data.map((row, index) => (
+                  <TableRow key={index}>
+                    <TableCell width="20%">
+                      {Intl.DateTimeFormat("en-ZA", {
+                        dateStyle: "short",
+                        timeStyle: "short",
+                      }).format(new Date(row.query_datetime_utc))}
+                    </TableCell>
+                    <TableCell>{row.query_text}</TableCell>
+                  </TableRow>
+                ))}
+              </TableBody>
+            </Table>
+          </TableContainer>
+        ) : (
+          <Box sx={{ fontSize: "small" }}>
+            No queries found. Please re-run discovery
+          </Box>
+        )}
+      </Box>
+    </Box>
+  );
+};
+
+export default Queries;