Lastnotnulldatapoint lastNotNull support added

CHANGELOG.md

@@ -1,7 +1,7 @@
 # Changelog
 
 ## 1.x.x
-Adds resource links for webiste, license and yaml defs to the plugin landing page.
+Adds resource links for website, license and yaml defs to the plugin landing page.
 
 Fixes x-scaling ratio when the SVG x-dimension is smaller than the available
 window. Now it scales with the window whereas before it scaled at twice the rate
@@ -13,6 +13,21 @@ Adds the ability to drive the SVG background. New panelConfig terms:
 Can be used with the normal color-names, rbg, hex values. When the relevant term is
 undefined the background color is not driven.
 
+Extends the datapoint choice from 'last' to 'last' or 'lastNotNull'. 'last' remains
+the default. When coupled with graphite functions 'keepLastValue' and 'transformNull', 'last'
+offers the best control over the display. Where those graphite functions are not available
+'lastNotNull' offers a consistent display when dealing with sparse data. 'lastNotNull' starts
+with the same datapoint as 'last' and then walks back in time till a non-null value is found.
+It's configurable in the panel yaml at the panel level and overridable at the cell level and cell-attribute level. To make this testable a 'test' table has been added to the yaml to allow
+the testData to be generated in sparse mode.
+New panel config terms of:
+- datapoint
+- cells.cell-name.datapoint
+- cells.cell-name.label.datapoint
+- cells.cell-name.labelColor.datapoint
+- cells.cell-name.fillColor.datapoint
+- test.testDataSparse
+
 ## 1.5.0
 Fixes the grafana variable threshold matching to break out on first rule
 match for a given variable/cell tuple. Before it was continuing through

provisioning/dashboardData/datapointLastNotNull.yaml

@@ -0,0 +1,48 @@
+---
+
+#------------------------------------------------------------------------------
+# Panel Config
+
+gradientMode: "hue"
+cellIdPreamble: "cell-"
+
+cells: 
+  inbox_depth:
+    dataRef: "test-data-large-sin"
+    label:
+      separator: "cr"
+      units: "none"
+    labelColor:
+      thresholds:
+        - {color: "green", level: 0}
+        - {color: "orange", level: 500}
+        - {color: "red", level: 1000}
+  db_transactions:
+    dataRef: "test-data-large-cos"
+    label:
+      separator: "cr"
+      units: "ops"
+      decimalPoints: 0
+    fillColor:
+      thresholds:
+        - {color: "semi-dark-green", level: 0}
+        - {color: "orange", level: 400}
+        - {color: "red", level: 800}
+  start_rate:
+    dataRef: "test-data-small-sin"
+    label:
+      separator: "colon"
+      units: "pps"
+    labelColor:
+      thresholds:
+        - {color: "green", level: 0}
+        - {color: "orange", level: 100}
+  active_workers:
+    dataRef: "test-data-small-cos"
+    label:
+      separator: "cr"
+      units: "none"
+    labelColor:
+      thresholds:
+        - {color: "#888888", level: 0}
+        - {color: "light-blue", level: 100}

src/components/Config.tsx

@@ -1,3 +1,4 @@
+export type DatapointMode = 'last' | 'lastNotNull';
 export type ColorGradientMode = 'none' | 'hue';
 export type LabelSeparator = 'cr' | 'colon' | 'space' | 'replace';
 export type LinkUrlParams = 'none' | 'time' | 'all';
@@ -21,24 +22,31 @@ export type Link = {
 export type Background = {
   darkThemeColor: string | undefined;
   lightThemeColor: string | undefined;
-}
+};
+
+export type TestConfig = {
+  testDataSparse: boolean | undefined;
+};
 
 export type PanelConfigCellLabel = {
   dataRef: string | undefined;
+  datapoint: DatapointMode | undefined;
   separator: LabelSeparator;
   units: string;
   decimalPoints: number | null | undefined;
-}
+};
 
 export type PanelConfigCellColor = {
   dataRef: string | undefined;
+  datapoint: DatapointMode | undefined;
   gradientMode: ColorGradientMode | undefined;
   thresholdsRef: string | undefined;
   thresholds: Threshold[] | undefined;
-}
+};
 
 export type PanelConfigCell = {
   dataRef: string | undefined;
+  datapoint: DatapointMode | undefined;
   linkRef: string | undefined;
   link: Link | undefined;
   label: PanelConfigCellLabel | undefined;
@@ -54,9 +62,11 @@ export type SiteConfig = {
 };
 
 export type PanelConfig = {
+  test: TestConfig;
   background: Background;
   variableThresholdScalars: Map<string, VariableThresholdScalars[]>;
   gradientMode: ColorGradientMode;
+  datapoint: DatapointMode | undefined;
   cellIdPreamble: string;
   cellIdExtender: string;
   cellLabelDecimalPoints: number | undefined;
@@ -66,9 +76,11 @@ export type PanelConfig = {
 export function panelConfigFactory(config: any) {
   config = config || {};
   return {
+    test: config.test || {},
     background: config.background || {},
     variableThresholdScalars: new Map<string, VariableThresholdScalars[]>(Object.entries(config.variableThresholdScalars || {})),
     gradientMode: config.gradientMode || 'none',
+    datapoint: config.datapoint || 'last',
     cellIdPreamble: config.cellIdPreamble || '',
     cellIdExtender: config.cellIdExtender || '@flowrpt',
     cellLabelDecimalPoints: (typeof config.cellLabelDecimalPoints === 'undefined') ? 0 : config.cellLabelDecimalPoints,
@@ -95,7 +107,7 @@ function siteConfigDereference(siteConfig: SiteConfig) {
 }
 
 function panelConfigDereference(siteConfig: SiteConfig, panelConfig: PanelConfig) {
-  function colorDeref(color: PanelConfigCellColor | undefined) {
+  function colorDeref(cell: PanelConfigCell, color: PanelConfigCellColor | undefined) {
     if (color) {
       color.gradientMode = color.gradientMode || panelConfig.gradientMode;
       if (color.thresholds) {
@@ -106,20 +118,30 @@ function panelConfigDereference(siteConfig: SiteConfig, panelConfig: PanelConfig
       if (!color.thresholds && color.thresholdsRef) {
         color.thresholds = siteConfig.thresholds.get(color.thresholdsRef);
       }
+      if (typeof color.datapoint === 'undefined') {
+        color.datapoint = cell.datapoint || panelConfig.datapoint;
+      }
     }
   }
-
   panelConfig.cells.forEach((cell) => {
-    colorDeref(cell.labelColor);
-    colorDeref(cell.fillColor);
+    colorDeref(cell, cell.labelColor);
+    colorDeref(cell, cell.fillColor);
 
     if (!cell.link && cell.linkRef) {
       cell.link = siteConfig.links.get(cell.linkRef);
     }
-    if (cell.label && (typeof cell.label.decimalPoints === 'undefined')) {
-      cell.label.decimalPoints = panelConfig.cellLabelDecimalPoints;
+    if (cell.label) {
+      if (typeof cell.label.decimalPoints === 'undefined') {
+        cell.label.decimalPoints = panelConfig.cellLabelDecimalPoints;
+      }
+      if (typeof cell.label.datapoint === 'undefined') {
+        cell.label.datapoint = cell.datapoint || panelConfig.datapoint;
+      }
     }
-  });
+    if (typeof cell.datapoint === 'undefined') {
+      cell.datapoint = panelConfig.datapoint;
+    }
+});
 }
 
 export function configInit(siteConfig: SiteConfig, panelConfig: PanelConfig) {

src/components/FlowPanel.tsx

@@ -159,7 +159,7 @@ export const FlowPanel: React.FC<Props> = ({ options, data, width, height, timeZ
   let tsData = instrument('transform', seriesTransform)(dataFrames, timeMin, timeMax);
 
   if (options.testDataEnabled) {
-    instrument('seriesExtend', seriesExtend)(tsData, timeMin, timeMax);
+    instrument('seriesExtend', seriesExtend)(tsData, timeMin, timeMax, panelConfig?.test.testDataSparse);
   }
 
   instrument('seriesInterpolate', seriesInterpolate)(tsData, timeSliderScalarRef.current);

src/components/SvgUpdater.tsx

@@ -1,5 +1,6 @@
 import { getValueFormatterIndex, formattedValueToString, GrafanaTheme2 } from '@grafana/data';
 import { 
+  DatapointMode,
    LabelSeparator, Link,
    PanelConfig, PanelConfigCell, PanelConfigCellColor, PanelConfigCellLabel,
    SiteConfig, VariableThresholdScalars } from 'components/Config';
@@ -179,11 +180,21 @@ export function svgInit(doc: Document, grafanaTheme: GrafanaTheme2, panelConfig:
   return svgAttribs;
 } 
 
-function getCellValue(tsName: string, tsData: TimeSeriesData) {
+function getCellValue(datapoint: DatapointMode | undefined, tsName: string, tsData: TimeSeriesData) {
   let value = null;
   const ts = tsData.ts.get(tsName);
   if (ts && (typeof ts.time.valuesIndex === 'number')) {
     value = ts.values[ts.time.valuesIndex];
+
+    // lastNotNull results in a walkback till a non null value is found
+    if (datapoint === 'lastNotNull') {
+      for (let i = ts.time.valuesIndex; i >= 0; i--) {
+        value = ts.values[i];
+        if (typeof value === 'number') {
+          break;
+        }
+      }
+    }
   }
   return value;
 }
@@ -204,33 +215,35 @@ export function svgUpdate(svgHolder: SvgHolder, tsData: TimeSeriesData) {
   const variableValues = svgHolder.attribs.variableValues;
   const cells = svgHolder.attribs.cells;
   cells.forEach((cellData) => {
-
     // This function sources the dataRef from the inner paramData and scales it using
     // the variables to a threshold seed. If it doesn't exist it returns the passed in
     // default.
-    function thresholdSeed(paramData: PanelConfigCellColor | undefined, defaultSeed: number | null) {
+    function thresholdSeed(datapoint: DatapointMode | undefined, paramData: PanelConfigCellColor | undefined, defaultSeed: number | null) {
       if (paramData?.dataRef) {
-        const cellValue = getCellValue(paramData.dataRef, tsData);
+        const cellValue = getCellValue(datapoint, paramData.dataRef, tsData);
         return variableThresholdScaleValue(variableValues, cellData, cellValue);
       }
       else {
         return paramData ? defaultSeed : null;
       }
     }
     const cellDataRef = cellData.cellProps.dataRef;
-    const cellValue = cellDataRef ? getCellValue(cellDataRef, tsData) : null;
+    const cellValue = cellDataRef ? getCellValue(cellData.cellProps.datapoint, cellDataRef, tsData) : null;
     const cellValueSeed = variableThresholdScaleValue(variableValues, cellData, cellValue);
 
     const cellLabelData = cellData.cellProps.label;
-    const cellLabelValue = cellLabelData?.dataRef ? getCellValue(cellLabelData.dataRef, tsData) : cellValue;
+    const cellLabelDatapoint = cellData.cellProps.label?.datapoint;
+    const cellLabelValue = cellLabelData?.dataRef ? getCellValue(cellLabelDatapoint, cellLabelData.dataRef, tsData) : cellValue;
     const cellLabel = cellLabelData && (typeof cellLabelValue === 'number') ? formatCellValue(cellLabelData, cellLabelValue) : null;
 
     const cellFillColorData = cellData.cellProps.fillColor;
-    const cellFillColorSeed = thresholdSeed(cellFillColorData, cellValueSeed);
+    const cellFillColorDatapoint = cellData.cellProps.fillColor?.datapoint;
+    const cellFillColorSeed = thresholdSeed(cellFillColorDatapoint, cellFillColorData, cellValueSeed);
     const cellFillColor = cellFillColorData && (typeof cellFillColorSeed === 'number') ? getColor(cellFillColorData, cellFillColorSeed) : null;
 
     const cellLabelColorData = cellData.cellProps.labelColor;
-    const cellLabelColorSeed = thresholdSeed(cellLabelColorData, cellValueSeed);
+    const cellLabelColorDatapoint = cellData.cellProps.labelColor?.datapoint;
+    const cellLabelColorSeed = thresholdSeed(cellLabelColorDatapoint, cellLabelColorData, cellValueSeed);
     const cellLabelColor = cellLabelColorData && (typeof cellLabelColorSeed === 'number') ? getColor(cellLabelColorData, cellLabelColorSeed) : null;
 
     cellData.fillElements.forEach((el: HTMLElement) => {

src/components/TimeSeries.tsx

@@ -6,7 +6,7 @@ export type TimeSeries = {
     valuesIndex?: number | null;
     values: number[];
   }
-  values: number[];
+  values: Array<number | null>;
 };
 
 export type TimeSeriesData = {
@@ -16,15 +16,15 @@ export type TimeSeriesData = {
   ts: Map<string, TimeSeries>;
 };
 
-export function seriesExtend(tsData: TimeSeriesData, timeMin: number, timeMax: number) {
+export function seriesExtend(tsData: TimeSeriesData, timeMin: number, timeMax: number, dataSparse: boolean) {
   const create = function(datapoints: number, scalar: number, fn: (inp: number) => number) {
     const intervalTime = Math.ceil((timeMax - timeMin) / datapoints);
     const intervalValue = 2 * Math.PI / datapoints;
     let timeValues = [];
     let dataValues = [];
     for (let i = 0; i <  datapoints; i++) {
       timeValues.push(timeMin + (i * intervalTime));
-      dataValues.push(scalar * (1 + fn(i * intervalValue)));
+      dataValues.push(dataSparse && ((i % 10) > 5)? null : scalar * (1 + fn(i * intervalValue)));
     }
     return {
       time: {values: timeValues},

yaml_defs/panelConfig.yaml

@@ -10,6 +10,16 @@ anchorLinks:
       url: "https://grafana.com/"
       params: "time"
 
+#------------------------------------------------------------------------------
+# Test Settings
+
+# 1.6.0 onwards: This section details test settings used in validating the functionality.
+# They provide a way to drive particular code pathways.
+test:
+  # This changes the test timeSeries to be sparse and is used as a way to demonstrate the
+  # difference between datapoint 'last' and 'lastNotNull'. Undefined defaults to false.
+  testDataSparse: false
+
 #------------------------------------------------------------------------------
 # Grafana Variable Scalars
 
@@ -36,6 +46,19 @@ background:
   darkThemeColor: "yellow"
   lightThemeColor: "green"
 
+# From version 1.6.0: This defines which datapoint will be chosen from the time-series.
+# If your datasource supports graphite functions 'keepLastValue' and 'transformNull', using those
+# functions in combination with 'last' gives the most precise control. If you don't have them available,
+# 'lastNotNull' allows you to approximate it. With 'last' the closest datapoint based off the timeSlider
+# position is chosen. With 'lastNotNull', that datapoint is then walked back in time until a non-null value
+# is found.
+#
+# This is the panel level default which when undefined defaults to 'last'. It can be overriden at the cell
+# level and the cell-attribute level.
+#
+# Valid values are undefined, "last", "lastNotNull"
+datapoint: "last"
+
 # This defines the default color gradientMode. It defines the default value on cell labelColor.gradientMode
 # and fillColor.gradientMode for when the field is undefined. Values are "hue" and "none" with
 # "none" being the default when undefined.
@@ -68,6 +91,9 @@ cells:
     # by attribute basis.
     dataRef: "test-data-large-sin"
 
+    # From version 1.6.0: Optional cell level override. Full write up in the panel-level term definition.
+    datapoint: "last"
+
     # The label field defines how the text in the cell will be adjusted based on the
     # time-series value. The time series value is postpended with a separator. If no
     # adjustment is wanted the field should be omitted. Note that multiline label preambles
@@ -76,6 +102,9 @@ cells:
       # Optional attribute level override for the associated time-series.
       dataRef: "test-data-large-cos"
 
+      # From version 1.6.0: Optional attribute level override. Requires corresponding attribute level dataRef override.
+      datapoint: "last"
+
       # Valid values are cr, space, colon, replace
       separator: "cr"
 
@@ -94,6 +123,9 @@ cells:
       # Optional attribute level override for the associated time-series.
       dataRef: "test-data-large-cos"
 
+      # From version 1.6.0: Optional attribute level override. Requires corresponding attribute level dataRef override.
+      datapoint: "last"
+
       # This defines the coloring gradientMode. When set to "hue", color values will be interpolated between
       # the threshold levels. When set to "none" the colors are absolute and change when a threshold is
       # crossed. The "hue" gradientMode will work with all color definitions apart from unrecognised color names.
@@ -129,6 +161,9 @@ cells:
       # See labelColor
       dataRef: "test-data-large-cos"
 
+      # See labelColor
+      datapoint: "last"
+
       # See labelColor
       gradientMode: "hue"