Brex frontend 02/03/2022

frontend/amundsen_application/__init__.py

@@ -20,6 +20,7 @@
 from amundsen_application.api.preview.dashboard.v0 import \
     dashboard_preview_blueprint
 from amundsen_application.api.preview.v0 import preview_blueprint
+from amundsen_application.api.freshness.v0 import freshness_blueprint
 from amundsen_application.api.quality.v0 import quality_blueprint
 from amundsen_application.api.search.v1 import search_blueprint
 from amundsen_application.api.v0 import blueprint
@@ -88,6 +89,7 @@ def create_app(config_module_class: str = None, template_folder: str = None) ->
     app.register_blueprint(search_blueprint)
     app.register_blueprint(api_bp)
     app.register_blueprint(dashboard_preview_blueprint)
+    app.register_blueprint(freshness_blueprint)
     init_routes(app)
 
     init_custom_routes = app.config.get('INIT_CUSTOM_ROUTES')

frontend/amundsen_application/api/freshness/v0.py

@@ -0,0 +1,73 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import logging
+
+from http import HTTPStatus
+
+from flask import Response, jsonify, make_response, request, current_app as app
+from flask.blueprints import Blueprint
+from marshmallow import ValidationError
+from werkzeug.utils import import_string
+
+from amundsen_application.models.preview_data import PreviewDataSchema
+from amundsen_application.api.metadata.v0 import _get_table_metadata
+
+LOGGER = logging.getLogger(__name__)
+DATA_FRESHNESS_CLIENT_CLASS = None
+DATA_FRESHNESS_CLIENT_INSTANCE = None
+
+freshness_blueprint = Blueprint('freshness', __name__, url_prefix='/api/freshness/v0')
+
+
+@freshness_blueprint.route('/', methods=['POST'])
+def get_table_freshness() -> Response:
+    global DATA_FRESHNESS_CLIENT_INSTANCE
+    global DATA_FRESHNESS_CLIENT_CLASS
+
+    try:
+        if DATA_FRESHNESS_CLIENT_INSTANCE is None:
+            if (app.config['DATA_FRESHNESS_CLIENT_ENABLED']
+                    and app.config['DATA_FRESHNESS_CLIENT'] is not None):
+                DATA_FRESHNESS_CLIENT_CLASS = import_string(app.config['DATA_FRESHNESS_CLIENT'])
+                DATA_FRESHNESS_CLIENT_INSTANCE = DATA_FRESHNESS_CLIENT_CLASS()
+            else:
+                payload = jsonify({'freshnessData': {'error_text': 'A client for the freshness must be configured'}})
+                return make_response(payload, HTTPStatus.NOT_IMPLEMENTED)
+
+        # get table metadata and pass to data_freshness_client
+        # data_freshness_client need to check if the table has any column
+        # that can be used to as freshness indicator
+        params = request.get_json()
+        if not all(param in params for param in ['database', 'cluster', 'schema', 'tableName']):
+            payload = jsonify({'freshnessData': {'error_text': 'Missing parameters in request payload'}})
+            return make_response(payload, HTTPStatus.FORBIDDEN)
+
+        table_key = f'{params["database"]}://{params["cluster"]}.{params["schema"]}/{params["tableName"]}'
+        # the index and source parameters are not referenced inside the function
+        table_metadata = _get_table_metadata(table_key=table_key, index=0, source='')
+
+        response = DATA_FRESHNESS_CLIENT_INSTANCE.get_freshness_data(params=table_metadata)
+        status_code = response.status_code
+
+        freshness_data = json.loads(response.data).get('freshness_data')
+        if status_code == HTTPStatus.OK:
+            # validate the returned data
+            try:
+                data = PreviewDataSchema().load(freshness_data)
+                payload = jsonify({'freshnessData': data})
+            except ValidationError as err:
+                logging.error('Freshness data dump returned errors: ' + str(err.messages))
+                raise Exception('The data freshness client did not return a valid object')
+        else:
+            message = 'Encountered error: Freshness client request failed with code ' + str(status_code)
+            logging.error(message)
+            # only necessary to pass the error text
+            payload = jsonify({'freshnessData': {'error_text': freshness_data.get('error_text', '')}, 'msg': message})
+        return make_response(payload, status_code)
+    except Exception as e:
+        message = f'Encountered exception: {str(e)}'
+        logging.exception(message)
+        payload = jsonify({'freshnessData': {}, 'msg': message})
+        return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)

frontend/amundsen_application/base/base_data_freshness_client.py

@@ -0,0 +1,22 @@
+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+import abc
+from typing import Dict
+
+from flask import Response
+
+
+class BaseDataFreshnessClient(abc.ABC):
+    @abc.abstractmethod
+    def __init__(self) -> None:
+        pass  # pragma: no cover
+
+    @abc.abstractmethod
+    def get_freshness_data(self, params: Dict, optionalHeaders: Dict = None) -> Response:
+        """
+        Returns a Response object, where the response data represents a json object
+        with the freshness data accessible on 'freshness_data' key. The freshness data should
+        match amundsen_application.models.preview_data.PreviewDataSchema
+        """
+        raise NotImplementedError  # pragma: no cover

frontend/amundsen_application/static/jest.config.js

@@ -20,7 +20,7 @@ module.exports = {
     },
     './js/ducks': {
       branches: 60, // 75
-      functions: 80,
+      functions: 75,
       lines: 80,
       statements: 80,
     },

frontend/amundsen_application/static/js/ducks/rootSaga.ts

@@ -60,6 +60,7 @@ import {
   getTableDataWatcher,
   getColumnDescriptionWatcher,
   getPreviewDataWatcher,
+  getFreshnessDataWatcher,
   getTableDescriptionWatcher,
   getTableQualityChecksWatcher,
   updateColumnDescriptionWatcher,
@@ -136,6 +137,7 @@ export default function* rootSaga() {
     getColumnDescriptionWatcher(),
 
     getPreviewDataWatcher(),
+    getFreshnessDataWatcher(),
     getTableDescriptionWatcher(),
     getTableQualityChecksWatcher(),
     updateColumnDescriptionWatcher(),

frontend/amundsen_application/static/js/ducks/tableMetadata/api/v0.ts

@@ -37,6 +37,7 @@ export type TableData = TableMetadata & {
 };
 export type DescriptionAPI = { description: string } & MessageAPI;
 export type PreviewDataAPI = { previewData: PreviewData } & MessageAPI;
+export type FreshnessDataAPI = { freshnessData: PreviewData } & MessageAPI;
 export type TableDataAPI = { tableData: TableData } & MessageAPI;
 export type RelatedDashboardDataAPI = {
   dashboards: DashboardResource[];
@@ -197,6 +198,27 @@ export function getPreviewData(queryParams: TablePreviewQueryParams) {
     });
 }
 
+export function getFreshnessData(queryParams: TablePreviewQueryParams) {
+  return axios({
+    url: '/api/freshness/v0/',
+    method: 'POST',
+    data: queryParams,
+  })
+    .then((response: AxiosResponse<FreshnessDataAPI>) => ({
+      data: response.data.freshnessData,
+      status: response.status,
+    }))
+    .catch((e: AxiosError<FreshnessDataAPI>) => {
+      const { response } = e;
+      let data = {};
+      if (response && response.data && response.data.freshnessData) {
+        data = response.data.freshnessData;
+      }
+      const status = response ? response.status : null;
+      return Promise.reject({ data, status });
+    });
+}
+
 export function getTableQualityChecksSummary(key: string) {
   const tableQueryParams = getTableQueryParams({
     key,

frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts

@@ -30,6 +30,9 @@ import {
   GetTableQualityChecksResponse,
   UpdateColumnDescription,
   UpdateColumnDescriptionRequest,
+  GetFreshnessData,
+  GetFreshnessDataRequest,
+  GetFreshnessDataResponse,
   UpdateTableDescription,
   UpdateTableDescriptionRequest,
   UpdateTableOwner,
@@ -45,6 +48,11 @@ export const initialPreviewState = {
   status: null,
 };
 
+export const initialFreshnessState = {
+  data: {},
+  status: null,
+};
+
 export const initialTableDataState: TableMetadata = {
   badges: [],
   cluster: '',
@@ -84,6 +92,7 @@ export const initialQualityChecksState = {
 export const initialState: TableMetadataReducerState = {
   isLoading: true,
   preview: initialPreviewState,
+  freshness: initialFreshnessState,
   statusCode: null,
   tableData: initialTableDataState,
   tableOwners: initialOwnersState,
@@ -276,6 +285,36 @@ export function getPreviewDataSuccess(
   };
 }
 
+export function getFreshnessData(
+  queryParams: TablePreviewQueryParams
+): GetFreshnessDataRequest {
+  return { payload: { queryParams }, type: GetFreshnessData.REQUEST };
+}
+export function getFreshnessDataFailure(
+  data: PreviewData,
+  status: number
+): GetFreshnessDataResponse {
+  return {
+    type: GetFreshnessData.FAILURE,
+    payload: {
+      data,
+      status,
+    },
+  };
+}
+export function getFreshnessDataSuccess(
+  data: PreviewData,
+  status: number
+): GetFreshnessDataResponse {
+  return {
+    type: GetFreshnessData.SUCCESS,
+    payload: {
+      data,
+      status,
+    },
+  };
+}
+
 export function getTableQualityChecks(
   key: string
 ): GetTableQualityChecksRequest {
@@ -338,6 +377,10 @@ export interface TableMetadataReducerState {
     data: PreviewData;
     status: number | null;
   };
+  freshness: {
+    data: PreviewData;
+    status: number | null;
+  };
   statusCode: number | null;
   tableData: TableMetadata;
   tableOwners: TableOwnerReducerState;
@@ -396,6 +439,12 @@ export default function reducer(
     case GetPreviewData.FAILURE:
     case GetPreviewData.SUCCESS:
       return { ...state, preview: (<GetPreviewDataResponse>action).payload };
+    case GetFreshnessData.FAILURE:
+    case GetFreshnessData.SUCCESS:
+      return {
+        ...state,
+        freshness: (<GetFreshnessDataResponse>action).payload,
+      };
     case UpdateTableOwner.REQUEST:
     case UpdateTableOwner.FAILURE:
     case UpdateTableOwner.SUCCESS:

frontend/amundsen_application/static/js/ducks/tableMetadata/sagas.ts

@@ -13,13 +13,16 @@ import {
   getColumnDescriptionSuccess,
   getPreviewDataFailure,
   getPreviewDataSuccess,
+  getFreshnessDataFailure,
+  getFreshnessDataSuccess,
   getTableQualityChecksSuccess,
   getTableQualityChecksFailure,
 } from './reducer';
 
 import {
   GetPreviewData,
   GetPreviewDataRequest,
+  GetFreshnessDataRequest,
   GetTableData,
   GetTableDataRequest,
   GetColumnDescription,
@@ -30,6 +33,7 @@ import {
   UpdateColumnDescriptionRequest,
   UpdateTableDescription,
   UpdateTableDescriptionRequest,
+  GetFreshnessData,
   GetTableQualityChecksRequest,
   GetTableQualityChecks,
 } from './types';
@@ -182,6 +186,25 @@ export function* getPreviewDataWatcher(): SagaIterator {
   yield takeLatest(GetPreviewData.REQUEST, getPreviewDataWorker);
 }
 
+export function* getFreshnessDataWorker(
+  action: GetFreshnessDataRequest
+): SagaIterator {
+  try {
+    const response = yield call(
+      API.getFreshnessData,
+      action.payload.queryParams
+    );
+    const { data, status } = response;
+    yield put(getFreshnessDataSuccess(data, status));
+  } catch (error) {
+    const { data, status } = error;
+    yield put(getFreshnessDataFailure(data, status));
+  }
+}
+export function* getFreshnessDataWatcher(): SagaIterator {
+  yield takeLatest(GetFreshnessData.REQUEST, getFreshnessDataWorker);
+}
+
 export function* getTableQualityChecksWorker(
   action: GetTableQualityChecksRequest
 ): SagaIterator {

frontend/amundsen_application/static/js/ducks/tableMetadata/types.ts

@@ -137,6 +137,25 @@ export interface GetPreviewDataResponse {
   };
 }
 
+export enum GetFreshnessData {
+  REQUEST = 'amundsen/freshness/GET_FRESHNESS_DATA_REQUEST',
+  SUCCESS = 'amundsen/freshness/GET_FRESHNESS_DATA_SUCCESS',
+  FAILURE = 'amundsen/freshness/GET_FRESHNESS_DATA_FAILURE',
+}
+export interface GetFreshnessDataRequest {
+  type: GetFreshnessData.REQUEST;
+  payload: {
+    queryParams: TablePreviewQueryParams;
+  };
+}
+export interface GetFreshnessDataResponse {
+  type: GetFreshnessData.SUCCESS | GetFreshnessData.FAILURE;
+  payload: {
+    data: PreviewData;
+    status: number | null;
+  };
+}
+
 export enum UpdateTableOwner {
   REQUEST = 'amundsen/tableMetadata/UPDATE_TABLE_OWNER_REQUEST',
   SUCCESS = 'amundsen/tableMetadata/UPDATE_TABLE_OWNER_SUCCESS',

frontend/amundsen_application/static/js/fixtures/globalState.ts

@@ -233,6 +233,10 @@ const globalState: GlobalState = {
       data: {},
       status: null,
     },
+    freshness: {
+      data: {},
+      status: null,
+    },
     statusCode: 200,
     tableData: {
       badges: [],