Merge branch 'main' into feature_flags

backend/pyproject.toml

@@ -10,7 +10,7 @@ homepage = "https://zenoml.com"
 [tool.poetry.dependencies]
 aiofiles = "^23.2.1"
 cognitojwt = "^1.4.1"
-fastapi = "0.103.1"
+fastapi = "0.100.1"
 fastapi-cloudauth = "^0.4.3"
 inquirer = "^3.1.2"
 nest-asyncio = "^1.5.6"
@@ -25,6 +25,7 @@ uvicorn = "^0.23.2"
 zeno-sliceline = "^0.0.1"
 pyarrow = "^13.0.0"
 sqlalchemy = "^2.0.20"
+amplitude-analytics = "^1.1.3"
 
 [tool.poetry.dev-dependencies]
 black = "^23.3.0"

backend/zeno_backend/classes/table.py

@@ -8,6 +8,7 @@ class TableRequest(CamelModel):
     """A request specification for table data."""
 
     columns: list[ZenoColumn]
+    model: str | None = None
     diff_column_1: ZenoColumn | None = None
     diff_column_2: ZenoColumn | None = None
     offset: int

backend/zeno_backend/database/insert.py

@@ -418,17 +418,19 @@ def tag(project: str, tag: Tag) -> int | None:
         return id[0][0]
 
 
-def user(user: User):
+def user(user: User) -> int | None:
     """Add a new user to the database.
 
     Args:
         user (User): the user to be added.
     """
     db = Database()
-    db.connect_execute(
-        'INSERT INTO users ("name") values(%s)',
+    id = db.connect_execute_return(
+        'INSERT INTO users ("name") values(%s) RETURNING id;',
         [user.name],
     )
+    if id is not None:
+        return id[0][0]
 
 
 def organization(user: User, organization: Organization):

backend/zeno_backend/processing/filtering.py

@@ -46,7 +46,7 @@ def filter_to_sql(
                     val = f.value
             column_id = (
                 f.column.id
-                if model is None
+                if f.column.model is None or model is None
                 else column_id_from_name_and_model(project, f.column.name, model)
             )
             filt = (

backend/zeno_backend/routers/sdk.py

@@ -1,8 +1,10 @@
 """FastAPI server endpoints for the Zeno SDK."""
 import io
+import os
 import uuid
 
 import pandas as pd
+from amplitude import Amplitude, BaseEvent
 from fastapi import (
     APIRouter,
     Depends,
@@ -18,6 +20,8 @@
 from zeno_backend.classes.project import Project
 from zeno_backend.database import insert, select
 
+amplitude_client = Amplitude(os.environ["AMPLITUDE_API_KEY"])
+
 
 class APIKeyBearer(HTTPBearer):
     """API key bearer authentication scheme."""
@@ -93,6 +97,13 @@ def create_project(project: Project, api_key=Depends(APIKeyBearer())):
         )
 
     project.uuid = str(uuid.uuid4())
+    amplitude_client.track(
+        BaseEvent(
+            event_type="Project Created",
+            user_id="00000" + str(user_id),
+            event_properties={"project_uuid": project.uuid},
+        )
+    )
     insert.project(project, user_id)
     return project.uuid
 
@@ -104,6 +115,7 @@ def upload_dataset(
     label_column: str = Form(None),
     data_column: str = Form(None),
     file: UploadFile = File(...),
+    api_key=Depends(APIKeyBearer()),
 ):
     """Upload a dataset to a Zeno project.
 
@@ -115,7 +127,15 @@ def upload_dataset(
         data_column (str | None, optional): the name of the column containing the
             raw data. Only works for small text data. Defaults to None.
         file (UploadFile): the dataset to upload.
+        api_key (str, optional): API key.
     """
+    user_id = select.user_id_by_api_key(api_key)
+    if user_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=("ERROR: Invalid API key."),
+        )
+
     try:
         bytes = file.file.read()
         dataset_df = pd.read_feather(io.BytesIO(bytes))
@@ -141,6 +161,7 @@ def upload_system(
     output_column: str = Form(...),
     id_column: str = Form(...),
     file: UploadFile = File(...),
+    api_key=Depends(APIKeyBearer()),
 ):
     """Upload a system to a Zeno project.
 
@@ -150,7 +171,15 @@ def upload_system(
         output_column (str): the name of the column containing the system output.
         id_column (str): the name of the column containing the instance IDs.
         file (UploadFile): the dataset to upload.
+        api_key (str, optional): API key.
     """
+    user_id = select.user_id_by_api_key(api_key)
+    if user_id is None:
+        raise HTTPException(
+            status_code=status.HTTP_401_UNAUTHORIZED,
+            detail=("ERROR: Invalid API key."),
+        )
+
     try:
         bytes = file.file.read()
         system_df = pd.read_feather(io.BytesIO(bytes))
@@ -167,3 +196,11 @@ def upload_system(
             status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
             detail=("ERROR: Unable to create system table: " + str(e)),
         ) from e
+
+    amplitude_client.track(
+        BaseEvent(
+            event_type="System Uploaded",
+            user_id="00000" + str(user_id),
+            event_properties={"project_uuid": project},
+        )
+    )

backend/zeno_backend/server.py

@@ -5,6 +5,7 @@
 
 import pandas as pd
 import uvicorn
+from amplitude import Amplitude, BaseEvent
 from dotenv import load_dotenv
 from fastapi import Depends, FastAPI, HTTPException, Request, Response, status
 from fastapi.middleware.cors import CORSMiddleware
@@ -43,6 +44,8 @@
 
 from .routers import sdk
 
+amplitude_client = Amplitude(os.environ["AMPLITUDE_API_KEY"])
+
 
 def get_server() -> FastAPI:
     """Provide the FastAPI server and specifies its inputs.
@@ -238,7 +241,7 @@ def get_filtered_table(req: TableRequest, project_uuid: str, request: Request):
         if not util.access_valid(project_uuid, request):
             return Response(status_code=401)
         filter_sql = table_filter(
-            project_uuid, None, req.filter_predicates, req.data_ids
+            project_uuid, req.model, req.filter_predicates, req.data_ids
         )
         sql_table = select.table_data_paginated(
             project_uuid, filter_sql, req.offset, req.limit, req.sort
@@ -284,6 +287,13 @@ def get_project(owner_name: str, project_name: str, request: Request):
             return Response(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
         if not util.access_valid(uuid, request):
             return Response(status_code=401)
+        amplitude_client.track(
+            BaseEvent(
+                event_type="Project Viewed",
+                user_id="ProjectViewedUser",
+                event_properties={"project_uuid": uuid},
+            )
+        )
         return select.project(
             owner_name, project_name, util.get_user_from_token(request)
         )
@@ -327,6 +337,12 @@ def get_projects(current_user=Depends(auth.claim())):
         tags=["zeno"],
     )
     def get_public_projects():
+        amplitude_client.track(
+            BaseEvent(
+                event_type="Home Viewed",
+                user_id="HomeViewedUser",
+            )
+        )
         return select.public_projects()
 
     @api_app.post(
@@ -444,7 +460,13 @@ def login(name: str):
         if fetched_user is None:
             try:
                 user = User(id=-1, name=name, admin=None)
-                insert.user(user)
+                user_id = insert.user(user)
+                amplitude_client.track(
+                    BaseEvent(
+                        event_type="User Registered",
+                        user_id="00000" + str(user_id) if user_id else "",
+                    )
+                )
                 insert.api_key(user)
                 return select.user(name)
             except Exception as exc:
@@ -453,6 +475,12 @@ def login(name: str):
                     detail=str(exc),
                 ) from exc
         else:
+            amplitude_client.track(
+                BaseEvent(
+                    event_type="User Logged In",
+                    user_id="00000" + str(fetched_user.id),
+                )
+            )
             return fetched_user
 
     @api_app.post(

frontend/.env.example

@@ -2,4 +2,5 @@ PUBLIC_BACKEND_ENDPOINT = "the url and port at which your backend is running, th
 ZENO_USER_POOL_AUTH_REGION = "cognito user pool auth region"
 ZENO_USER_POOL_CLIENT_ID = "cognito user pool client id"
 ZENO_USER_POOL_ID = "cognito user pool id"
-ALLOW_INSECURE_HTTP = "true or false, whether to allow http or not"
+ALLOW_INSECURE_HTTP = "true or false, whether to allow http or not"
+AMPLITUDE_API_KEY = "amplitude api key"

frontend/src/lib/api/metadata.ts

@@ -24,6 +24,26 @@ export interface HistogramEntry {
 	metric?: number;
 }
 
+export async function loadHistogramData(
+	project_uuid: string,
+	tagIds: string[] | undefined,
+	selectionIds: string[] | undefined,
+	model: string | undefined,
+	columns: ZenoColumn[]
+) {
+	const dataIds =
+		tagIds !== undefined && selectionIds !== undefined
+			? [...new Set([...tagIds, ...selectionIds])]
+			: tagIds !== undefined
+			? tagIds
+			: selectionIds;
+	const histograms = await getHistograms(project_uuid, columns, model);
+	const counts = await getHistogramCounts(project_uuid, columns, histograms, undefined, dataIds);
+	if (counts === undefined) {
+		return;
+	}
+	return counts;
+}
 /**
  * Fetch metadata columns buckets for histograms.
  *
@@ -33,18 +53,15 @@ export interface HistogramEntry {
  * @returns Histogram buckets for each column.
  */
 export async function getHistograms(
+	project_uuid: string,
 	completeColumns: ZenoColumn[],
 	model: string | undefined
 ): Promise<Map<string, HistogramEntry[]>> {
 	const requestedHistograms = completeColumns.filter(
 		(c) => (c.model === null || c.model === model) && c.columnType !== ZenoColumnType.DATA
 	);
 	requestingHistogramCounts.set(true);
-	const config = get(project);
-	if (!config) {
-		return Promise.reject('No project selected.');
-	}
-	const res = await ZenoService.getHistogramBuckets(config.uuid, requestedHistograms);
+	const res = await ZenoService.getHistogramBuckets(project_uuid, requestedHistograms);
 	requestingHistogramCounts.set(false);
 	const histograms = new Map<string, HistogramEntry[]>(
 		requestedHistograms.map((col, i) => [col.id, res[i]])
@@ -63,24 +80,22 @@ let histogramCountRequest: CancelablePromise<Array<Array<number>>>;
  * @returns Histogram counts for each column.
  */
 export async function getHistogramCounts(
+	project_uuid: string,
+	columns: ZenoColumn[],
 	histograms: Map<string, HistogramEntry[]>,
 	filterPredicates?: FilterPredicateGroup,
 	dataIds?: string[]
 ): Promise<Map<string, HistogramEntry[]> | undefined> {
 	const columnRequests = [...histograms.entries()].map(([k, v]) => ({
-		column: get(columns).find((col) => col.id === k) ?? get(columns)[0],
+		column: columns.find((col) => col.id === k) ?? columns[0],
 		buckets: v
 	}));
 	if (histogramCountRequest) {
 		histogramCountRequest.cancel();
 	}
 	try {
-		const config = get(project);
-		if (!config) {
-			return Promise.reject('No project selected.');
-		}
 		requestingHistogramCounts.set(true);
-		histogramCountRequest = ZenoService.calculateHistogramCounts(config.uuid, {
+		histogramCountRequest = ZenoService.calculateHistogramCounts(project_uuid, {
 			columnRequests,
 			filterPredicates,
 			dataIds