diff --git a/common/api/hooks/service.ts b/common/api/hooks/service.ts index 9c3e4ace2..a422d2ad5 100644 --- a/common/api/hooks/service.ts +++ b/common/api/hooks/service.ts @@ -1,7 +1,7 @@ import { useQuery } from '@tanstack/react-query'; -import type { FetchScheduledServiceOptions } from '../../types/api'; +import type { FetchScheduledServiceOptions, FetchServiceHoursOptions } from '../../types/api'; import { ONE_HOUR } from '../../constants/time'; -import { fetchScheduledService } from '../service'; +import { fetchScheduledService, fetchServiceHours } from '../service'; export const useScheduledService = (options: FetchScheduledServiceOptions, enabled?: boolean) => { return useQuery(['scheduledservice', options], () => fetchScheduledService(options), { @@ -9,3 +9,10 @@ export const useScheduledService = (options: FetchScheduledServiceOptions, enabl staleTime: ONE_HOUR, }); }; + +export const useServiceHours = (params: FetchServiceHoursOptions, enabled?: boolean) => { + return useQuery(['service_hours', params], () => fetchServiceHours(params), { + enabled: enabled, + staleTime: 0, + }); +}; diff --git a/common/api/service.ts b/common/api/service.ts index ea97b752a..acdd63e7a 100644 --- a/common/api/service.ts +++ b/common/api/service.ts @@ -1,6 +1,11 @@ -import type { FetchScheduledServiceOptions } from '../types/api'; +import type { + FetchScheduledServiceOptions, + FetchServiceHoursOptions, + FetchServiceHoursResponse, +} from '../types/api'; import { FetchScheduledServiceParams } from '../types/api'; import type { ScheduledService } from '../types/dataPoints'; +import { APP_DATA_BASE_PATH } from '../utils/constants'; import { apiFetch } from './utils/fetch'; export const fetchScheduledService = async ( @@ -14,3 +19,16 @@ export const fetchScheduledService = async ( errorMessage: 'Failed to fetch trip counts', }); }; + +export const fetchServiceHours = async ( + params: FetchServiceHoursOptions +): Promise => { + if (!params.start_date) return undefined; + const url = new URL(`${APP_DATA_BASE_PATH}/api/service_hours`, window.location.origin); + Object.keys(params).forEach((paramKey) => { + url.searchParams.append(paramKey, params[paramKey]); + }); + const response = await fetch(url.toString()); + if (!response.ok) throw new Error('Failed to fetch service hours'); + return await response.json(); +}; diff --git a/common/components/charts/TimeSeriesChart/TimeSeriesChart.tsx b/common/components/charts/TimeSeriesChart/TimeSeriesChart.tsx index 575a077d2..5c296e9e4 100644 --- a/common/components/charts/TimeSeriesChart/TimeSeriesChart.tsx +++ b/common/components/charts/TimeSeriesChart/TimeSeriesChart.tsx @@ -154,6 +154,8 @@ export const TimeSeriesChart = (props: Props) => { return { x: { + min: timeAxis.from, + max: timeAxis.to, display: true, type: 'time' as const, adapters: { diff --git a/common/components/charts/TimeSeriesChart/defaults.ts b/common/components/charts/TimeSeriesChart/defaults.ts index 0f51f540a..aed650312 100644 --- a/common/components/charts/TimeSeriesChart/defaults.ts +++ b/common/components/charts/TimeSeriesChart/defaults.ts @@ -38,7 +38,7 @@ export const defaultWeekAxis: ResolvedTimeAxis = { export const defaultMonthAxis: ResolvedTimeAxis = { granularity: 'month', - axisUnit: 'month', + axisUnit: 'year', label: 'Month', format: 'yyyy', tooltipFormat: 'MMM yyyy', diff --git a/common/components/charts/TimeSeriesChart/types.ts b/common/components/charts/TimeSeriesChart/types.ts index ad1835d07..0950a7a2b 100644 --- a/common/components/charts/TimeSeriesChart/types.ts +++ b/common/components/charts/TimeSeriesChart/types.ts @@ -35,13 +35,15 @@ export type DisplayStyle = DisplayStyle; export type Granularity = 'time' | 'day' | 'week' | 'month'; -export type AxisUnit = 'day' | 'month'; +export type AxisUnit = 'day' | 'month' | 'year'; export type ProvidedTimeAxis = { label: string; format?: string; tooltipFormat?: string; axisUnit?: AxisUnit; + from?: string; + to?: string; } & ({ granularity: Granularity } | { agg: AggType }); export type ResolvedTimeAxis = { @@ -50,6 +52,8 @@ export type ResolvedTimeAxis = { axisUnit?: AxisUnit; format?: string; tooltipFormat?: string; + from?: string; + to?: string; }; export type ValueAxis = { diff --git a/common/types/api.ts b/common/types/api.ts index 745151526..1435a9c7d 100644 --- a/common/types/api.ts +++ b/common/types/api.ts @@ -1,5 +1,5 @@ import type { AggType } from '../../modules/speed/constants/speeds'; -import type { SpeedRestriction } from './dataPoints'; +import type { ServiceHours, SpeedRestriction } from './dataPoints'; import type { Line, LineRouteId } from './lines'; export enum QueryNameKeys { @@ -99,3 +99,12 @@ export type FetchSpeedRestrictionsResponse = { date: string; zones: SpeedRestriction[]; }; + +export type FetchServiceHoursOptions = Partial<{ + start_date: string; + end_date: string; + line_id: string; + agg: AggType; +}>; + +export type FetchServiceHoursResponse = ServiceHours[]; diff --git a/common/types/dataPoints.ts b/common/types/dataPoints.ts index 686120c88..5c15a7b99 100644 --- a/common/types/dataPoints.ts +++ b/common/types/dataPoints.ts @@ -142,11 +142,17 @@ export type ServiceLevels = { }; export type ScheduledService = { - counts: number[]; start_date: string; end_date: string; start_date_service_levels: ServiceLevels; end_date_service_levels: ServiceLevels; + counts: { date: string; count: number }[]; +}; + +export type ServiceHours = { + date: string; + scheduled: number; + delivered: number; }; export type RidershipCount = { diff --git a/modules/service/PercentageServiceGraphWrapper.tsx b/modules/service/PercentageServiceGraphWrapper.tsx index e22739c10..2dad36c18 100644 --- a/modules/service/PercentageServiceGraphWrapper.tsx +++ b/modules/service/PercentageServiceGraphWrapper.tsx @@ -1,4 +1,4 @@ -import React from 'react'; +import React, { useMemo } from 'react'; import type { SetStateAction } from 'react'; import type { DeliveredTripMetrics, ScheduledService } from '../../common/types/dataPoints'; import { WidgetCarousel } from '../../common/components/general/WidgetCarousel'; @@ -33,11 +33,20 @@ export const PercentageServiceGraphWrapper: React.FC { // TODO: Add 1 or 2 widgets to percentage service graph. const { line } = useDelimitatedRoute(); + + const { scheduled, peak } = useMemo( + () => getPercentageData(data, predictedData, line), + [data, predictedData, line] + ); + + const { scheduledAverage, peakAverage } = useMemo(() => { + const scheduledAverage = getAverageWithNaNs(scheduled); + const peakAverage = getAverageWithNaNs(peak); + return { scheduledAverage, peakAverage }; + }, [scheduled, peak]); + if (!data.some((datapoint) => datapoint.miles_covered)) return ; - const { scheduled, peak } = getPercentageData(data, predictedData, line); - const scheduledAverage = getAverageWithNaNs(scheduled); - const peakAverage = getAverageWithNaNs(peak); return ( <> diff --git a/modules/service/ScheduledAndDeliveredGraph.tsx b/modules/service/ScheduledAndDeliveredGraph.tsx new file mode 100644 index 000000000..cd35d0a80 --- /dev/null +++ b/modules/service/ScheduledAndDeliveredGraph.tsx @@ -0,0 +1,40 @@ +/* eslint-disable import/no-unused-modules */ +import React, { useMemo } from 'react'; + +import type { Benchmark, Block, Dataset } from '../../common/components/charts/TimeSeriesChart'; +import { TimeSeriesChart } from '../../common/components/charts/TimeSeriesChart'; +import { useDelimitatedRoute } from '../../common/utils/router'; +import { LINE_COLORS } from '../../common/constants/colors'; +import type { AggType } from '../speed/constants/speeds'; + +interface ScheduledAndDeliveredGraphProps { + scheduled: Dataset; + delivered: Dataset; + startDate: string; + endDate: string; + valueAxisLabel: string; + agg: AggType; + benchmarks?: Benchmark[]; + blocks?: Block[]; +} + +export const ScheduledAndDeliveredGraph: React.FC = ( + props: ScheduledAndDeliveredGraphProps +) => { + const { scheduled, delivered, valueAxisLabel, agg, startDate, endDate, benchmarks, blocks } = + props; + const data = useMemo(() => [scheduled, delivered], [scheduled, delivered]); + const { line } = useDelimitatedRoute(); + const color = LINE_COLORS[line ?? 'default']; + + return ( + + ); +}; diff --git a/modules/service/ServiceDetails.tsx b/modules/service/ServiceDetails.tsx index 91c40d35b..532896918 100644 --- a/modules/service/ServiceDetails.tsx +++ b/modules/service/ServiceDetails.tsx @@ -4,7 +4,7 @@ import dayjs from 'dayjs'; import utc from 'dayjs/plugin/utc'; import { useDelimitatedRoute } from '../../common/utils/router'; import { ChartPlaceHolder } from '../../common/components/graphics/ChartPlaceHolder'; -import { useScheduledService } from '../../common/api/hooks/service'; +import { useScheduledService, useServiceHours } from '../../common/api/hooks/service'; import { Layout } from '../../common/layouts/layoutTypes'; import { PageWrapper } from '../../common/layouts/PageWrapper'; import { getSpeedGraphConfig } from '../speed/constants/speeds'; @@ -14,6 +14,7 @@ import { useDeliveredTripMetrics } from '../../common/api/hooks/tripmetrics'; import { WidgetTitle } from '../dashboard/WidgetTitle'; import { ServiceGraphWrapper } from './ServiceGraphWrapper'; import { PercentageServiceGraphWrapper } from './PercentageServiceGraphWrapper'; +import { ServiceHoursGraph } from './ServiceHoursGraph'; dayjs.extend(utc); export function ServiceDetails() { @@ -45,7 +46,18 @@ export function ServiceDetails() { enabled ).data; + const serviceHoursData = useServiceHours( + { + start_date: startDate, + end_date: endDate, + line_id: line, + agg: config.agg, + }, + enabled + ); + const serviceDataReady = !tripsData.isError && tripsData.data && line && config && predictedData; + const serviceHoursDataReady = !serviceHoursData.isError && serviceHoursData.data; if (!startDate || !endDate) { return

Select a date range to load graphs.

; @@ -88,7 +100,22 @@ export function ServiceDetails() { )} - {' '} + + + {serviceHoursDataReady ? ( + + ) : ( +
+ +
+ )} + + ); } diff --git a/modules/service/ServiceGraph.tsx b/modules/service/ServiceGraph.tsx index 5921e9098..567e309cd 100644 --- a/modules/service/ServiceGraph.tsx +++ b/modules/service/ServiceGraph.tsx @@ -1,234 +1,82 @@ -import React, { useMemo, useRef } from 'react'; -import { Line } from 'react-chartjs-2'; -import 'chartjs-adapter-date-fns'; -import { enUS } from 'date-fns/locale'; -import pattern from 'patternomaly'; -import Annotation from 'chartjs-plugin-annotation'; -import ChartjsPluginWatermark from 'chartjs-plugin-watermark'; +import React, { useMemo } from 'react'; import { useDelimitatedRoute } from '../../common/utils/router'; -import { CHART_COLORS, COLORS, LINE_COLORS } from '../../common/constants/colors'; -import { drawSimpleTitle } from '../../common/components/charts/Title'; -import { hexWithAlpha } from '../../common/utils/general'; -import type { ParamsType } from '../speed/constants/speeds'; import { PEAK_SCHEDULED_SERVICE } from '../../common/constants/baselines'; -import { useBreakpoint } from '../../common/hooks/useBreakpoint'; -import { watermarkLayout } from '../../common/constants/charts'; -import { ChartBorder } from '../../common/components/charts/ChartBorder'; -import { ChartDiv } from '../../common/components/charts/ChartDiv'; import type { DeliveredTripMetrics, ScheduledService } from '../../common/types/dataPoints'; +import type { ParamsType } from '../speed/constants/speeds'; import { getShuttlingBlockAnnotations } from './utils/graphUtils'; +import { ScheduledAndDeliveredGraph } from './ScheduledAndDeliveredGraph'; interface ServiceGraphProps { + config: ParamsType; data: DeliveredTripMetrics[]; predictedData: ScheduledService; - config: ParamsType; startDate: string; endDate: string; - showTitle?: boolean; } -export const ServiceGraph: React.FC = ({ - data, - predictedData, - config, - startDate, - endDate, - showTitle = false, -}) => { - const { line, linePath } = useDelimitatedRoute(); +export const ServiceGraph: React.FC = (props: ServiceGraphProps) => { + const { data, predictedData, startDate, endDate, config } = props; + const { line } = useDelimitatedRoute(); + const peak = PEAK_SCHEDULED_SERVICE[line ?? 'DEFAULT']; - const { tooltipFormat, unit, callbacks } = config; - const isMobile = !useBreakpoint('md'); - const ref = useRef(); + const benchmarks = useMemo(() => { + const label = `Historical maximum (${peak} round trips)`; + return [{ label, value: peak }]; + }, [peak]); - const chart = useMemo(() => { - const labels = data.map((point) => point.date); - const lineColor = LINE_COLORS[line ?? 'default']; + const blocks = useMemo(() => { const shuttlingBlocks = getShuttlingBlockAnnotations(data); - return ( - - - - datapoint.miles_covered ? Math.round(datapoint.count) : Number.NaN - ), - }, - { - label: `Scheduled round trips`, - stepped: true, - fill: true, - pointBorderWidth: 0, - pointRadius: 0, - pointHoverRadius: 6, + return shuttlingBlocks.map((block) => ({ + from: block.xMin as string, + to: block.xMax as string, + })); + }, [data]); + + const scheduled = useMemo(() => { + return { + label: 'Scheduled round trips', + data: predictedData.counts.map(({ date, count }, index) => { + const value = data[index]?.miles_covered > 0 && count ? count / 2 : 0; + return { date, value }; + }), + style: { + fillPattern: 'striped' as const, + tooltipLabel: (point) => { + const percentOfPeak = `${((100 * point.value) / peak).toFixed(1)}%`; + return `Scheduled: ${point.value} (${percentOfPeak} of peak)`; + }, + }, + }; + }, [data, predictedData, peak]); - borderColor: lineColor, - spanGaps: false, - data: predictedData.counts.map((count, index) => - data[index]?.miles_covered > 0 && count ? count / 2 : Number.NaN - ), - backgroundColor: pattern.draw('diagonal', 'transparent', lineColor, 5), - }, - { - // This null dataset produces the entry in the legend for the baseline annotation. - label: `Historical Maximum (${peak.toLocaleString('en-us')} round trips)`, - backgroundColor: CHART_COLORS.ANNOTATIONS, - data: null, - }, - ], - }} - options={{ - responsive: true, - maintainAspectRatio: false, - layout: { - padding: { - top: showTitle ? 25 : 0, - }, - }, - interaction: { - intersect: false, - }, - // @ts-expect-error The watermark plugin doesn't have typescript support - watermark: watermarkLayout(isMobile), - plugins: { - tooltip: { - mode: 'index', - position: 'nearest', - callbacks: { - ...callbacks, - label: (context) => { - return `${context.datasetIndex === 0 ? 'Actual:' : 'Scheduled:'} ${ - context.parsed.y - } (${((100 * context.parsed.y) / peak).toFixed(1)}% of historical maximum)`; - }, - }, - }, - legend: { - position: 'bottom', - labels: { - boxWidth: 15, - }, - }, - title: { - // empty title to set font and leave room for drawTitle fn - display: showTitle, - text: '', - }, - annotation: { - // Add your annotations here - annotations: [ - { - type: 'line', - yMin: peak, - yMax: peak, - borderColor: CHART_COLORS.ANNOTATIONS, - // corresponds to null dataset index. - display: (ctx) => ctx.chart.isDatasetVisible(2), - borderWidth: 2, - }, - ...shuttlingBlocks, - ], - }, - }, - scales: { - y: { - min: 0, - display: true, - ticks: { - color: COLORS.design.subtitleGrey, - }, - title: { - display: true, - text: 'Round trips', - color: COLORS.design.subtitleGrey, - }, - }, - x: { - min: startDate, - max: endDate, - type: 'time', - time: { - unit: unit, - tooltipFormat: tooltipFormat, - displayFormats: { - month: 'MMM', - }, - }, - ticks: { - color: COLORS.design.subtitleGrey, - }, - adapters: { - date: { - locale: enUS, - }, - }, - display: true, - title: { - display: false, - text: ``, - }, - }, - }, - }} - plugins={[ - { - id: 'customTitle', - afterDraw: (chart) => { - if (!data) { - // No data is present - const { ctx } = chart; - const { width } = chart; - const { height } = chart; - chart.clear(); + const delivered = useMemo(() => { + return { + label: 'Daily round trips', + data: data.map((datapoint) => { + const value = datapoint.miles_covered ? Math.round(datapoint.count) : 0; + return { date: datapoint.date, value }; + }), + style: { + tooltipLabel: (point) => { + const percentOfPeak = `${((100 * point.value) / peak).toFixed(1)}%`; + return `Actual: ${point.value} (${percentOfPeak} of peak)`; + }, + }, + }; + }, [data, peak]); - ctx.save(); - ctx.textAlign = 'center'; - ctx.textBaseline = 'middle'; - ctx.font = "16px normal 'Helvetica Nueue'"; - ctx.fillText('No data to display', width / 2, height / 2); - ctx.restore(); - } - if (showTitle) drawSimpleTitle(`Daily round trips`, chart); - }, - }, - Annotation, - ChartjsPluginWatermark, - ]} - /> - - - ); - }, [ - data, - line, - isMobile, - linePath, - predictedData.counts, - showTitle, - callbacks, - peak, - startDate, - endDate, - unit, - tooltipFormat, - ]); - return chart; + return ( + + ); }; diff --git a/modules/service/ServiceGraphWrapper.tsx b/modules/service/ServiceGraphWrapper.tsx index 08ad3c1a1..7a9df6b46 100644 --- a/modules/service/ServiceGraphWrapper.tsx +++ b/modules/service/ServiceGraphWrapper.tsx @@ -24,7 +24,7 @@ export const ServiceGraphWrapper: React.FC = ({ endDate, }) => { if (!data.some((datapoint) => datapoint.miles_covered)) return ; - const { average, peak } = getServiceWidgetValues(data, predictedData.counts); + const { average, peak } = getServiceWidgetValues(data, predictedData); return ( diff --git a/modules/service/ServiceHoursGraph.tsx b/modules/service/ServiceHoursGraph.tsx new file mode 100644 index 000000000..1b15b7881 --- /dev/null +++ b/modules/service/ServiceHoursGraph.tsx @@ -0,0 +1,45 @@ +/* eslint-disable import/no-unused-modules */ +import React, { useMemo } from 'react'; + +import type { FetchServiceHoursResponse } from '../../common/types/api'; +import type { AggType } from '../speed/constants/speeds'; +import { ScheduledAndDeliveredGraph } from './ScheduledAndDeliveredGraph'; + +interface ServiceHoursGraphProps { + serviceHours: FetchServiceHoursResponse; + agg: AggType; + startDate: string; + endDate: string; +} + +export const ServiceHoursGraph: React.FC = ( + props: ServiceHoursGraphProps +) => { + const { serviceHours, agg, startDate, endDate } = props; + + const scheduled = useMemo(() => { + return { + label: 'Scheduled service hours', + data: serviceHours.map((sh) => ({ date: sh.date, value: sh.scheduled })), + style: { fillPattern: 'striped' as const }, + }; + }, [serviceHours]); + + const delivered = useMemo(() => { + return { + label: 'Delivered service hours', + data: serviceHours.map((sh) => ({ date: sh.date, value: sh.delivered })), + }; + }, [serviceHours]); + + return ( + + ); +}; diff --git a/modules/service/ServiceOverviewWrapper.tsx b/modules/service/ServiceOverviewWrapper.tsx index 529ebd05d..198ee5abc 100644 --- a/modules/service/ServiceOverviewWrapper.tsx +++ b/modules/service/ServiceOverviewWrapper.tsx @@ -22,7 +22,7 @@ export const ServiceOverviewWrapper: React.FC = ({ startDate, endDate, }) => { - const { average, percentDelivered } = getServiceWidgetValues(data, predictedData.counts); + const { average, percentDelivered } = getServiceWidgetValues(data, predictedData); return ( <> @@ -49,7 +49,6 @@ export const ServiceOverviewWrapper: React.FC = ({ predictedData={predictedData} startDate={startDate} endDate={endDate} - showTitle /> diff --git a/modules/service/utils/utils.ts b/modules/service/utils/utils.ts index f99328801..cffb09484 100644 --- a/modules/service/utils/utils.ts +++ b/modules/service/utils/utils.ts @@ -3,15 +3,15 @@ import type { DeliveredTripMetrics, ScheduledService } from '../../../common/typ import type { Line } from '../../../common/types/lines'; export const getServiceWidgetValues = ( - datapoints: DeliveredTripMetrics[], - predictedData: number[] + deliveredTripMetrics: DeliveredTripMetrics[], + predictedData: ScheduledService ) => { - const totals = datapoints.reduce( + const totals = deliveredTripMetrics.reduce( (totals, datapoint, index) => { - if (datapoint.count && predictedData[index]) { + if (datapoint.count && predictedData.counts[index].count) { return { actual: totals.actual + datapoint.count, - scheduled: totals.scheduled + predictedData[index], + scheduled: totals.scheduled + predictedData.counts[index].count, }; } return { actual: totals.actual, scheduled: totals.scheduled }; @@ -19,13 +19,16 @@ export const getServiceWidgetValues = ( { actual: 0, scheduled: 0 } ); const percentDelivered = totals.actual / totals.scheduled; - const datapointsCount = datapoints.filter((datapoint) => datapoint.miles_covered).length; - const current = datapoints[datapoints.length - 1].count; - const delta = current - datapoints[0].count; - const average = datapoints.reduce((sum, speed) => sum + speed.count, 0) / datapointsCount; - const peak = datapoints.reduce( + const datapointsCount = deliveredTripMetrics.filter( + (datapoint) => datapoint.miles_covered + ).length; + const current = deliveredTripMetrics[deliveredTripMetrics.length - 1].count; + const delta = current - deliveredTripMetrics[0].count; + const average = + deliveredTripMetrics.reduce((sum, speed) => sum + speed.count, 0) / datapointsCount; + const peak = deliveredTripMetrics.reduce( (max, speed) => (speed.count > max.count ? speed : max), - datapoints[0] + deliveredTripMetrics[0] ); return { current, delta, average, peak, percentDelivered }; }; @@ -37,7 +40,7 @@ export const getPercentageData = ( ) => { const scheduled = data.map((datapoint, index) => { return datapoint.miles_covered && predictedData.counts[index] - ? (100 * datapoint.count) / (predictedData.counts[index] / 2) + ? (100 * datapoint.count) / (predictedData.counts[index].count / 2) : Number.NaN; }); const peak = data.map((datapoint) => diff --git a/server/.chalice/policy.json b/server/.chalice/policy.json index 8b9dc8271..0a8ef6b95 100644 --- a/server/.chalice/policy.json +++ b/server/.chalice/policy.json @@ -35,6 +35,7 @@ "Effect": "Allow", "Resource": [ "arn:aws:dynamodb:*:*:table/DeliveredTripMetrics", + "arn:aws:dynamodb:*:*:table/DeliveredTripMetricsExtended", "arn:aws:dynamodb:*:*:table/DeliveredTripMetricsWeekly", "arn:aws:dynamodb:*:*:table/DeliveredTripMetricsMonthly", "arn:aws:dynamodb:*:*:table/ScheduledServiceDaily", diff --git a/server/app.py b/server/app.py index c3a0f0af2..98b96a64f 100644 --- a/server/app.py +++ b/server/app.py @@ -12,8 +12,9 @@ mbta_v3, speed, speed_restrictions, + scheduled_service, + service_hours, predictions, - service_levels, ridership, ) @@ -185,7 +186,7 @@ def get_scheduled_service(): end_date = parse_user_date(query["end_date"]) route_id = query.get("route_id") agg = query["agg"] - response = service_levels.get_scheduled_service( + response = scheduled_service.get_scheduled_service_counts( start_date=start_date, end_date=end_date, route_id=route_id, @@ -226,6 +227,22 @@ def get_speed_restrictions(): return json.dumps(response) +@app.route("/api/service_hours", cors=cors_config) +def get_service_hours(): + query = app.current_request.query_params + line_id = query.get("line_id") + start_date = parse_user_date(query["start_date"]) + end_date = parse_user_date(query["end_date"]) + agg = query["agg"] + response = service_hours.get_service_hours( + single_route_id=line_id, + start_date=start_date, + end_date=end_date, + agg=agg, + ) + return json.dumps(response) + + @app.route("/api/time_predictions", cors=cors_config) def get_time_predictions(): query = app.current_request.query_params diff --git a/server/chalicelib/aggregation.py b/server/chalicelib/aggregation.py index b63d0e751..b2f86510e 100644 --- a/server/chalicelib/aggregation.py +++ b/server/chalicelib/aggregation.py @@ -30,7 +30,7 @@ def train_peak_status(df): def faster_describe(grouped): # This does the same thing as pandas.DataFrame.describe(), but is up to 25x faster! # also, we can specify population std instead of sample. - stats = grouped.aggregate(["count", "mean", "min", "median", "max"]) + stats = grouped.aggregate(["count", "mean", "min", "median", "max", "sum"]) std = grouped.std(ddof=0) q1 = grouped.quantile(0.25) q3 = grouped.quantile(0.75) @@ -106,7 +106,10 @@ def travel_times_all(sdate, edate, from_stops, to_stops): by_date = calc_travel_times_by_date(df) by_time = calc_travel_times_by_time(df) - return {"by_date": by_date.to_dict("records"), "by_time": by_time.to_dict("records")} + return { + "by_date": by_date.to_dict("records"), + "by_time": by_time.to_dict("records"), + } def travel_times_over_time(sdate, edate, from_stops, to_stops): diff --git a/server/chalicelib/dynamo.py b/server/chalicelib/dynamo.py index 432e6eb62..abe6959a3 100644 --- a/server/chalicelib/dynamo.py +++ b/server/chalicelib/dynamo.py @@ -3,6 +3,7 @@ import boto3 from dynamodb_json import json_util as ddb_json from chalicelib import constants +from typing import List import concurrent.futures # Create a DynamoDB resource @@ -19,7 +20,13 @@ def query_daily_trips_on_line(table_name, line, start_date, end_date): route_keys = constants.LINE_TO_ROUTE_MAP[line] with concurrent.futures.ThreadPoolExecutor(max_workers=4) as executor: futures = [ - executor.submit(query_daily_trips_on_route, table_name, route_key, start_date, end_date) + executor.submit( + query_daily_trips_on_route, + table_name, + route_key, + start_date, + end_date, + ) for route_key in route_keys ] results = [] @@ -54,3 +61,22 @@ def query_agg_trip_metrics(start_date: str, end_date: str, table_name: str, line condition = line_condition & date_condition response = table.query(KeyConditionExpression=condition) return ddb_json.loads(response["Items"]) + + +def query_extended_trip_metrics( + start_date: date, + end_date: date, + route_ids: List[str], +): + table = dynamodb.Table("DeliveredTripMetricsExtended") + start_date_str = start_date.strftime("%Y-%m-%d") + end_date_str = end_date.strftime("%Y-%m-%d") + response_dicts = [] + for route_id in route_ids: + route_condition = Key("route").eq(route_id) + date_condition = Key("date").between(start_date_str, end_date_str) + condition = route_condition & date_condition + response = table.query(KeyConditionExpression=condition) + responses = ddb_json.loads(response["Items"]) + response_dicts.extend(responses) + return response_dicts diff --git a/server/chalicelib/sampling.py b/server/chalicelib/sampling.py new file mode 100644 index 000000000..6acde96e9 --- /dev/null +++ b/server/chalicelib/sampling.py @@ -0,0 +1,64 @@ +from datetime import date +from typing import Dict, List, Literal +import pandas as pd + +from .data_funcs import date_range + + +def resample_and_aggregate( + values: Dict[str, any], # Keys should be date strings + agg: Literal["daily", "weekly", "monthly"], + avg_type=Literal["mean", "median"], +): + # parse start_date and end_date to pandas datetime64 + start_date = pd.to_datetime(min(values.keys())) + end_date = pd.to_datetime(max(values.keys())) + + if agg == "daily": + return values + df = pd.DataFrame(list(values.items()), columns=["date", "value"]) + df["date"] = pd.to_datetime(df["date"]) + + # Set 'date' as index for resampling + df.set_index("date", inplace=True) + + if agg == "monthly": + df_agg = df.resample("M") + else: + df_agg = df.resample("W-SUN") + + df_agg = df_agg.mean() if avg_type == "mean" else df_agg.median() + + # Drop the first week or month if it is incomplete + start_date = df.index.min() + if start_date.weekday() != 6: # 6 is Sunday + df_agg = df_agg.iloc[1:] + + df_agg = df_agg.reset_index() + df_agg.dropna(inplace=True) + + if agg == "weekly": + # Pandas resample uses the end date of the range as the index. So we subtract 6 days to convert to first date of the range. + df_agg["date"] = df_agg["date"] - pd.Timedelta(days=6) + + # drop any rows where date is not between start_date and end_date + df_agg = df_agg[(df_agg["date"] >= start_date) & (df_agg["date"] <= end_date)].copy() + + df_agg["date"] = df_agg["date"].dt.strftime("%Y-%m-%d") + + return {row["date"]: row["value"] for _, row in df_agg.iterrows()} + + +def resample_list_of_values_with_range( + values: List[any], + start_date: date, + end_date: date, + agg: Literal["daily", "weekly", "monthly"], + avg_type=Literal["mean", "median"], +): + if agg == "daily": + return agg + dates = date_range(start_date, end_date) + values_dict = {date: values[index] for index, date in enumerate(dates)} + resampled = resample_and_aggregate(values_dict, agg, avg_type) + return list(resampled.values()) diff --git a/server/chalicelib/service_levels.py b/server/chalicelib/scheduled_service.py similarity index 61% rename from server/chalicelib/service_levels.py rename to server/chalicelib/scheduled_service.py index f3e58dc95..0b3ed7279 100644 --- a/server/chalicelib/service_levels.py +++ b/server/chalicelib/scheduled_service.py @@ -1,9 +1,9 @@ from datetime import date, datetime from typing import List, Dict, TypedDict, Union, Literal -import pandas as pd from .dynamo import query_scheduled_service -from .data_funcs import index_by, date_range +from .data_funcs import index_by +from .sampling import resample_and_aggregate ByHourServiceLevels = List[int] DayKind = Union[Literal["weekday"], Literal["saturday"], Literal["sunday"]] @@ -25,9 +25,8 @@ class ByDayKindServiceLevels(TypedDict): class GetScheduledServiceResponse(TypedDict): start_date_service_levels: ByDayKindServiceLevels end_date_service_levels: ByDayKindServiceLevels - start_date: str - end_date: str - counts: List[int] + counts: Dict[str, int] + service_hours: Dict[str, float] def get_next_day_kind_service_levels( @@ -66,24 +65,6 @@ def get_service_levels( } -def get_weekly_scheduled_service(scheduled_service_arr, start_date, end_date): - df = pd.DataFrame({"value": scheduled_service_arr}, index=pd.date_range(start_date, end_date)) - weekly_df = df.resample("W-SUN").median() - # Drop the first week if it is incomplete - if datetime.fromisoformat(start_date.isoformat()).weekday() != 6: - weekly_df = weekly_df[1:] - return weekly_df["value"].tolist() - - -def get_monthly_scheduled_service(scheduled_service_arr, start_date, end_date): - df = pd.DataFrame({"value": scheduled_service_arr}, index=pd.date_range(start_date, end_date)) - monthly_df = df.resample("M").median() - # Drop the first month if it is incomplete - if datetime.fromisoformat(start_date.isoformat()).day != 1: - monthly_df = monthly_df[1:] - return monthly_df["value"].tolist() - - def get_scheduled_service( start_date: date, end_date: date, @@ -96,26 +77,62 @@ def get_scheduled_service( route_id=route_id, ) scheduled_service_by_day = index_by(scheduled_service, "date") - scheduled_service_arr = [] - for current_day in date_range(start_date, end_date): - current_day_iso = current_day.isoformat() - if current_day_iso in scheduled_service_by_day: - scheduled_service_count = scheduled_service_by_day[current_day_iso]["count"] - else: - scheduled_service_count = None - scheduled_service_arr.append(scheduled_service_count) - counts = [] - if agg == "daily": - counts = scheduled_service_arr - if agg == "weekly": - counts = get_weekly_scheduled_service(scheduled_service_arr, start_date, end_date) - if agg == "monthly": - counts = get_monthly_scheduled_service(scheduled_service_arr, start_date, end_date) - + counts_by_day = {} + service_minutes_by_day = {} + for current_day, current_day_service in scheduled_service_by_day.items(): + counts_by_day[current_day] = current_day_service["count"] + service_minutes_by_day[current_day] = current_day_service["serviceMinutes"] + counts = resample_and_aggregate( + values=counts_by_day, + agg=agg, + avg_type="median", + ) + service_minutes = resample_and_aggregate( + values=service_minutes_by_day, + agg=agg, + avg_type="median", + ) return { "counts": counts, + "service_minutes": service_minutes, "start_date": start_date.isoformat(), "end_date": end_date.isoformat(), "start_date_service_levels": get_service_levels(scheduled_service, False), "end_date_service_levels": get_service_levels(scheduled_service, True), } + + +def get_scheduled_service_counts( + start_date: date, + end_date: date, + agg: AggTypes, + route_id: str = None, +): + result = get_scheduled_service( + start_date=start_date, + end_date=end_date, + agg=agg, + route_id=route_id, + ) + return { + "start_date": result["start_date"], + "end_date": result["end_date"], + "start_date_service_levels": result["start_date_service_levels"], + "end_date_service_levels": result["end_date_service_levels"], + "counts": [{"date": date, "count": count} for (date, count) in result["counts"].items()], + } + + +def get_scheduled_service_hours( + start_date: date, + end_date: date, + agg: AggTypes, + route_id: str = None, +): + result = get_scheduled_service( + start_date=start_date, + end_date=end_date, + agg=agg, + route_id=route_id, + ) + return {date: result["service_minutes"][date] // 60 for date in result["service_minutes"]} diff --git a/server/chalicelib/service_hours.py b/server/chalicelib/service_hours.py new file mode 100644 index 000000000..3f9c1cdcb --- /dev/null +++ b/server/chalicelib/service_hours.py @@ -0,0 +1,90 @@ +from datetime import date +from concurrent import futures +from typing import List, TypedDict, Literal, Dict +import pandas as pd + +from chalicelib.scheduled_service import get_scheduled_service_hours +from chalicelib.dynamo import query_extended_trip_metrics +from chalicelib.sampling import resample_and_aggregate + +AggType = Literal["daily", "weekly", "monthly"] + +ROUTE_IDS_BY_LINE = { + "line-blue": ("line-blue",), + "line-orange": ("line-orange",), + "line-red": ("line-red", "line-red-a", "line-red-b"), + "line-green": ("line-green-b", "line-green-c", "line-green-d", "line-green-e"), +} + +SINGLE_ROUTE_IDS_BY_LINE_KEY = { + "line-blue": "Blue", + "line-orange": "Orange", + "line-red": "Red", + "line-green": "Green", +} + + +class ServiceHoursEntry(TypedDict): + date: str + scheduled: int + delivered: int + + +def get_delivered_service_times(response_dicts: List[Dict[str, any]], agg: AggType): + df = pd.DataFrame.from_records(response_dicts) + service_hours = {} + # unique service_dates + service_dates = df["date"].unique() + for today in service_dates: + df_for_date = df[df["date"] == today][ + [ + "dir_0_exclusive_count", + "dir_1_exclusive_count", + "dir_0_inclusive_mean", + "dir_1_inclusive_mean", + ] + ] + df_for_date["total_time"] = ( + df_for_date["dir_0_exclusive_count"] * df_for_date["dir_0_inclusive_mean"] + + df_for_date["dir_1_exclusive_count"] * df_for_date["dir_1_inclusive_mean"] + ) // 3600 + service_hours[today] = df_for_date.sum()["total_time"] + return resample_and_aggregate(values=service_hours, agg=agg, avg_type="median") + + +def get_service_hours( + single_route_id: str, + start_date: date, + end_date: date, + agg: AggType, +) -> List[ServiceHoursEntry]: + responses = [] + route_ids = ROUTE_IDS_BY_LINE[single_route_id] + single_route_id = SINGLE_ROUTE_IDS_BY_LINE_KEY[single_route_id] + with futures.ThreadPoolExecutor(max_workers=2) as executor: + scheduled_service_hours = executor.submit( + get_scheduled_service_hours, + start_date=start_date, + end_date=end_date, + route_id=single_route_id, + agg=agg, + ).result() + extended_trip_metrics = executor.submit( + query_extended_trip_metrics, + start_date=start_date, + end_date=end_date, + route_ids=route_ids, + ).result() + delivered_service_hours = get_delivered_service_times(extended_trip_metrics, agg) + for date_str in scheduled_service_hours.keys(): + scheduled = scheduled_service_hours.get(date_str) + delivered = delivered_service_hours.get(date_str) + if scheduled and delivered: + responses.append( + { + "date": date_str, + "scheduled": scheduled, + "delivered": delivered, + } + ) + return responses diff --git a/server/chalicelib/speed.py b/server/chalicelib/speed.py index 31b1aa2c0..1dadb5ca5 100644 --- a/server/chalicelib/speed.py +++ b/server/chalicelib/speed.py @@ -1,9 +1,18 @@ +from typing import TypedDict from chalice import BadRequestError, ForbiddenError from chalicelib import dynamo from datetime import datetime, timedelta import pandas as pd import numpy as np + +class TripMetricsByLineParams(TypedDict): + start_date: str + end_date: str + agg: str + line: str + + # Delta values put limits on the numbers of days for which data that can be requested. For each table it is approximately 150 entries. AGG_TO_CONFIG_MAP = { "daily": {"table_name": "DeliveredTripMetrics", "delta": 150}, @@ -26,7 +35,14 @@ def aggregate_actual_trips(actual_trips, agg, start_date): # Group each branch into one entry. Keep NaN entries as NaN df_grouped = ( df.groupby("date") - .agg({"miles_covered": "sum", "total_time": "sum", "count": "sum", "line": "first"}) + .agg( + { + "miles_covered": "sum", + "total_time": "sum", + "count": "sum", + "line": "first", + } + ) .reset_index() ) # set index to use datetime object. @@ -34,7 +50,7 @@ def aggregate_actual_trips(actual_trips, agg, start_date): return df_grouped.to_dict(orient="records") -def trip_metrics_by_line(params): +def trip_metrics_by_line(params: TripMetricsByLineParams): """ Get trip metrics grouped by line. The weekly and monthly dbs are already aggregated by line. The daily db is not, and gets aggregated on the fly.