Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: move cache to chart data #173

Merged
merged 3 commits into from
Sep 15, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
87 changes: 87 additions & 0 deletions backend/zeno_backend/classes/chart.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,20 @@ class XCParameters(CamelModel):
color_channel: SlicesOrModels
x_channel: SlicesOrModels

def __hash__(self) -> int:
"""Hash the chart parameters.

Returns:
int: The hash of the chart parameters.
"""
return hash(
hash(tuple(self.slices))
+ hash(self.metric)
+ hash(tuple(self.models))
+ hash(self.color_channel)
+ hash(self.x_channel)
)


class TableParameters(CamelModel):
"""Parameter specification for a tabular visualization."""
Expand All @@ -51,6 +65,21 @@ class TableParameters(CamelModel):
x_channel: SlicesMetricsOrModels
fixed_channel: SlicesMetricsOrModels

def __hash__(self) -> int:
"""Hash the chart parameters.

Returns:
int: The hash of the chart parameters.
"""
return hash(
hash(tuple(self.metrics))
+ hash(tuple(self.slices))
+ hash(tuple(self.models))
+ hash(self.y_channel)
+ hash(self.x_channel)
+ hash(self.fixed_channel)
)


class BeeswarmParameters(CamelModel):
"""Parameter specification for a beeswarm chart."""
Expand All @@ -62,6 +91,21 @@ class BeeswarmParameters(CamelModel):
color_channel: SlicesOrModels
fixed_dimension: str

def __hash__(self):
"""Hash the chart parameters.

Returns:
int: The hash of the chart parameters.
"""
return hash(
hash(tuple(self.metrics))
+ hash(tuple(self.slices))
+ hash(tuple(self.models))
+ hash(self.y_channel)
+ hash(self.color_channel)
+ hash(self.fixed_dimension)
)


class RadarParameters(CamelModel):
"""Parameter specification for a radar chart."""
Expand All @@ -73,6 +117,21 @@ class RadarParameters(CamelModel):
layer_channel: SlicesOrModels
fixed_channel: SlicesMetricsOrModels

def __hash__(self):
"""Hash the chart parameters.

Returns:
int: The hash of the chart parameters.
"""
return hash(
hash(tuple(self.metrics))
+ hash(tuple(self.slices))
+ hash(tuple(self.models))
+ hash(self.axis_channel)
+ hash(self.layer_channel)
+ hash(self.fixed_channel)
)


class HeatmapParameters(CamelModel):
"""Parameter specirication for a heatmap chart."""
Expand All @@ -84,6 +143,21 @@ class HeatmapParameters(CamelModel):
y_channel: SlicesOrModels
x_channel: SlicesOrModels

def __hash__(self) -> int:
"""Hash the chart parameters.

Returns:
int: The hash of the chart parameters.
"""
return hash(
hash(self.metric)
+ hash(tuple(self.x_values))
+ hash(tuple(self.y_values))
+ hash(self.model)
+ hash(self.y_channel)
+ hash(self.x_channel)
)


class Chart(CamelModel):
"""Generic chart specification with parameters for specific chart types."""
Expand All @@ -100,6 +174,19 @@ class Chart(CamelModel):
)
project_uuid: str | None = None

def __hash__(self) -> int:
"""Hash the chart parameters.

Returns:
int: The hash of the chart parameters.
"""
return hash(
hash(self.name)
+ hash(self.type)
+ hash(self.parameters)
+ hash(self.project_uuid)
)


class ParametersEncoder(json.JSONEncoder):
"""JSON encoder for chart parameter data."""
Expand Down
4 changes: 3 additions & 1 deletion backend/zeno_backend/processing/chart.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
"""Functions for extracting chart data from SQL."""
import json
from functools import lru_cache
from typing import Any

from zeno_backend.classes.chart import (
Expand Down Expand Up @@ -335,11 +336,12 @@ def heatmap_data(chart: Chart, project: str) -> str:
return json.dumps({"table": elements})


@lru_cache(4096)
def chart_data(chart: Chart, project: str) -> str:
"""Extract the chart data for a specific chart that the user created.

Args:
chart (Chart): the chart for which to extract the data.
chart (Chart): the chart for which to generate data.
project (str): the project the user is currently working with

Returns:
Expand Down
29 changes: 20 additions & 9 deletions backend/zeno_backend/processing/histogram_processing.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
"""Functions for creating the frontend metadata histograms."""


import asyncio

import numpy as np
import pandas as pd
from psycopg import sql
Expand Down Expand Up @@ -228,7 +230,7 @@ def histogram_metric_task(
return []


def histogram_count(
async def histogram_count(
request: HistogramRequest,
col_request: HistogramColumnRequest,
project_uuid: str,
Expand Down Expand Up @@ -325,16 +327,25 @@ def histogram_counts(
filter_sql = table_filter(
project_uuid, req.model, req.filter_predicates, req.data_ids
)
res = []

loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)

tasks = []
for r in req.column_requests:
res.append(
histogram_count(
req,
r,
project_uuid,
filter_sql,
project_obj.calculate_histogram_metrics,
tasks.append(
loop.create_task(
histogram_count(
req,
r,
project_uuid,
filter_sql,
project_obj.calculate_histogram_metrics,
)
)
)

res = loop.run_until_complete(asyncio.gather(*tasks))
loop.close()

return res
3 changes: 0 additions & 3 deletions backend/zeno_backend/processing/metrics/map.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
"""Map metric names to metric functions."""

from functools import lru_cache

from psycopg import sql

from zeno_backend.classes.base import GroupMetric
Expand Down Expand Up @@ -53,7 +51,6 @@ def count(
)


@lru_cache(maxsize=4096)
def metric_map(
metric: Metric | None,
project: str,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
size: number;
}>;
};
export let width = 1000;
export let height = 400;
export let width = 700;
export let height = 600;
Comment on lines +16 to +17
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: should not do changes like these in PRs about something completely different probably.


$: parameters = chart.parameters as RadarParameters;

Expand Down
13 changes: 7 additions & 6 deletions frontend/src/routes/(app)/+layout.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,19 @@ import { OpenAPI, ZenoService } from '$lib/zenoapi';
import { redirect } from '@sveltejs/kit';

export const load = async ({ cookies, url }) => {
OpenAPI.BASE = getEndpoint() + '/api';
const userCookie = cookies.get('loggedIn');

if (userCookie) {
const cognitoUser = JSON.parse(userCookie) as AuthUser;
let cognitoUser = JSON.parse(userCookie) as AuthUser;
if (new Date() > new Date(cognitoUser.accessTokenExpires * 1000)) {
try {
const res = await refreshAccessToken(cognitoUser.refreshToken);
const user = extractUserFromSession(res);
cognitoUser = extractUserFromSession(res);
OpenAPI.HEADERS = {
Authorization: 'Bearer ' + user.accessToken
Authorization: 'Bearer ' + cognitoUser.accessToken
};
cookies.set('loggedIn', JSON.stringify(user), {
cookies.set('loggedIn', JSON.stringify(cognitoUser), {
path: '/',
httpOnly: true,
sameSite: 'strict',
Expand All @@ -30,11 +32,10 @@ export const load = async ({ cookies, url }) => {
}
}
// If the user is not authenticated, redirect to the login page
if (!cognitoUser.id || !cognitoUser.accessToken) {
if (!cognitoUser || !cognitoUser.id || !cognitoUser.accessToken) {
throw redirect(303, `/login?redirectTo=${url.pathname}`);
}

OpenAPI.BASE = getEndpoint() + '/api';
OpenAPI.HEADERS = {
Authorization: 'Bearer ' + cognitoUser.accessToken
};
Expand Down