diff --git a/frontend/amundsen_application/__init__.py b/frontend/amundsen_application/__init__.py index a39ed1b041..125a4ffed7 100644 --- a/frontend/amundsen_application/__init__.py +++ b/frontend/amundsen_application/__init__.py @@ -20,6 +20,7 @@ from amundsen_application.api.preview.dashboard.v0 import \ dashboard_preview_blueprint from amundsen_application.api.preview.v0 import preview_blueprint +from amundsen_application.api.freshness.v0 import freshness_blueprint from amundsen_application.api.quality.v0 import quality_blueprint from amundsen_application.api.search.v1 import search_blueprint from amundsen_application.api.v0 import blueprint @@ -88,6 +89,7 @@ def create_app(config_module_class: str = None, template_folder: str = None) -> app.register_blueprint(search_blueprint) app.register_blueprint(api_bp) app.register_blueprint(dashboard_preview_blueprint) + app.register_blueprint(freshness_blueprint) init_routes(app) init_custom_routes = app.config.get('INIT_CUSTOM_ROUTES') diff --git a/frontend/amundsen_application/api/freshness/__init__.py b/frontend/amundsen_application/api/freshness/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frontend/amundsen_application/api/freshness/v0.py b/frontend/amundsen_application/api/freshness/v0.py new file mode 100644 index 0000000000..c3df0a986b --- /dev/null +++ b/frontend/amundsen_application/api/freshness/v0.py @@ -0,0 +1,73 @@ +# Copyright Contributors to the Amundsen project. +# SPDX-License-Identifier: Apache-2.0 + +import json +import logging + +from http import HTTPStatus + +from flask import Response, jsonify, make_response, request, current_app as app +from flask.blueprints import Blueprint +from marshmallow import ValidationError +from werkzeug.utils import import_string + +from amundsen_application.models.preview_data import PreviewDataSchema +from amundsen_application.api.metadata.v0 import _get_table_metadata + +LOGGER = logging.getLogger(__name__) +DATA_FRESHNESS_CLIENT_CLASS = None +DATA_FRESHNESS_CLIENT_INSTANCE = None + +freshness_blueprint = Blueprint('freshness', __name__, url_prefix='/api/freshness/v0') + + +@freshness_blueprint.route('/', methods=['POST']) +def get_table_freshness() -> Response: + global DATA_FRESHNESS_CLIENT_INSTANCE + global DATA_FRESHNESS_CLIENT_CLASS + + try: + if DATA_FRESHNESS_CLIENT_INSTANCE is None: + if (app.config['DATA_FRESHNESS_CLIENT_ENABLED'] + and app.config['DATA_FRESHNESS_CLIENT'] is not None): + DATA_FRESHNESS_CLIENT_CLASS = import_string(app.config['DATA_FRESHNESS_CLIENT']) + DATA_FRESHNESS_CLIENT_INSTANCE = DATA_FRESHNESS_CLIENT_CLASS() + else: + payload = jsonify({'freshnessData': {'error_text': 'A client for the freshness must be configured'}}) + return make_response(payload, HTTPStatus.NOT_IMPLEMENTED) + + # get table metadata and pass to data_freshness_client + # data_freshness_client need to check if the table has any column + # that can be used to as freshness indicator + params = request.get_json() + if not all(param in params for param in ['database', 'cluster', 'schema', 'tableName']): + payload = jsonify({'freshnessData': {'error_text': 'Missing parameters in request payload'}}) + return make_response(payload, HTTPStatus.FORBIDDEN) + + table_key = f'{params["database"]}://{params["cluster"]}.{params["schema"]}/{params["tableName"]}' + # the index and source parameters are not referenced inside the function + table_metadata = _get_table_metadata(table_key=table_key, index=0, source='') + + response = DATA_FRESHNESS_CLIENT_INSTANCE.get_freshness_data(params=table_metadata) + status_code = response.status_code + + freshness_data = json.loads(response.data).get('freshness_data') + if status_code == HTTPStatus.OK: + # validate the returned data + try: + data = PreviewDataSchema().load(freshness_data) + payload = jsonify({'freshnessData': data}) + except ValidationError as err: + logging.error('Freshness data dump returned errors: ' + str(err.messages)) + raise Exception('The data freshness client did not return a valid object') + else: + message = 'Encountered error: Freshness client request failed with code ' + str(status_code) + logging.error(message) + # only necessary to pass the error text + payload = jsonify({'freshnessData': {'error_text': freshness_data.get('error_text', '')}, 'msg': message}) + return make_response(payload, status_code) + except Exception as e: + message = f'Encountered exception: {str(e)}' + logging.exception(message) + payload = jsonify({'freshnessData': {}, 'msg': message}) + return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR) diff --git a/frontend/amundsen_application/base/base_data_freshness_client.py b/frontend/amundsen_application/base/base_data_freshness_client.py new file mode 100644 index 0000000000..dd585b7650 --- /dev/null +++ b/frontend/amundsen_application/base/base_data_freshness_client.py @@ -0,0 +1,22 @@ +# Copyright Contributors to the Amundsen project. +# SPDX-License-Identifier: Apache-2.0 + +import abc +from typing import Dict + +from flask import Response + + +class BaseDataFreshnessClient(abc.ABC): + @abc.abstractmethod + def __init__(self) -> None: + pass # pragma: no cover + + @abc.abstractmethod + def get_freshness_data(self, params: Dict, optionalHeaders: Dict = None) -> Response: + """ + Returns a Response object, where the response data represents a json object + with the freshness data accessible on 'freshness_data' key. The freshness data should + match amundsen_application.models.preview_data.PreviewDataSchema + """ + raise NotImplementedError # pragma: no cover diff --git a/frontend/amundsen_application/static/jest.config.js b/frontend/amundsen_application/static/jest.config.js index f91d0b83a5..dd1d0bebe5 100644 --- a/frontend/amundsen_application/static/jest.config.js +++ b/frontend/amundsen_application/static/jest.config.js @@ -20,7 +20,7 @@ module.exports = { }, './js/ducks': { branches: 60, // 75 - functions: 80, + functions: 75, lines: 80, statements: 80, }, diff --git a/frontend/amundsen_application/static/js/ducks/rootSaga.ts b/frontend/amundsen_application/static/js/ducks/rootSaga.ts index 7e790e3d8e..319fcd539f 100644 --- a/frontend/amundsen_application/static/js/ducks/rootSaga.ts +++ b/frontend/amundsen_application/static/js/ducks/rootSaga.ts @@ -60,6 +60,7 @@ import { getTableDataWatcher, getColumnDescriptionWatcher, getPreviewDataWatcher, + getFreshnessDataWatcher, getTableDescriptionWatcher, getTableQualityChecksWatcher, updateColumnDescriptionWatcher, @@ -136,6 +137,7 @@ export default function* rootSaga() { getColumnDescriptionWatcher(), getPreviewDataWatcher(), + getFreshnessDataWatcher(), getTableDescriptionWatcher(), getTableQualityChecksWatcher(), updateColumnDescriptionWatcher(), diff --git a/frontend/amundsen_application/static/js/ducks/tableMetadata/api/v0.ts b/frontend/amundsen_application/static/js/ducks/tableMetadata/api/v0.ts index 047b2ed1d2..dab22c5dfd 100644 --- a/frontend/amundsen_application/static/js/ducks/tableMetadata/api/v0.ts +++ b/frontend/amundsen_application/static/js/ducks/tableMetadata/api/v0.ts @@ -37,6 +37,7 @@ export type TableData = TableMetadata & { }; export type DescriptionAPI = { description: string } & MessageAPI; export type PreviewDataAPI = { previewData: PreviewData } & MessageAPI; +export type FreshnessDataAPI = { freshnessData: PreviewData } & MessageAPI; export type TableDataAPI = { tableData: TableData } & MessageAPI; export type RelatedDashboardDataAPI = { dashboards: DashboardResource[]; @@ -197,6 +198,27 @@ export function getPreviewData(queryParams: TablePreviewQueryParams) { }); } +export function getFreshnessData(queryParams: TablePreviewQueryParams) { + return axios({ + url: '/api/freshness/v0/', + method: 'POST', + data: queryParams, + }) + .then((response: AxiosResponse) => ({ + data: response.data.freshnessData, + status: response.status, + })) + .catch((e: AxiosError) => { + const { response } = e; + let data = {}; + if (response && response.data && response.data.freshnessData) { + data = response.data.freshnessData; + } + const status = response ? response.status : null; + return Promise.reject({ data, status }); + }); +} + export function getTableQualityChecksSummary(key: string) { const tableQueryParams = getTableQueryParams({ key, diff --git a/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts b/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts index 87f77ab8b3..1880db5376 100644 --- a/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts +++ b/frontend/amundsen_application/static/js/ducks/tableMetadata/reducer.ts @@ -30,6 +30,9 @@ import { GetTableQualityChecksResponse, UpdateColumnDescription, UpdateColumnDescriptionRequest, + GetFreshnessData, + GetFreshnessDataRequest, + GetFreshnessDataResponse, UpdateTableDescription, UpdateTableDescriptionRequest, UpdateTableOwner, @@ -45,6 +48,11 @@ export const initialPreviewState = { status: null, }; +export const initialFreshnessState = { + data: {}, + status: null, +}; + export const initialTableDataState: TableMetadata = { badges: [], cluster: '', @@ -84,6 +92,7 @@ export const initialQualityChecksState = { export const initialState: TableMetadataReducerState = { isLoading: true, preview: initialPreviewState, + freshness: initialFreshnessState, statusCode: null, tableData: initialTableDataState, tableOwners: initialOwnersState, @@ -276,6 +285,36 @@ export function getPreviewDataSuccess( }; } +export function getFreshnessData( + queryParams: TablePreviewQueryParams +): GetFreshnessDataRequest { + return { payload: { queryParams }, type: GetFreshnessData.REQUEST }; +} +export function getFreshnessDataFailure( + data: PreviewData, + status: number +): GetFreshnessDataResponse { + return { + type: GetFreshnessData.FAILURE, + payload: { + data, + status, + }, + }; +} +export function getFreshnessDataSuccess( + data: PreviewData, + status: number +): GetFreshnessDataResponse { + return { + type: GetFreshnessData.SUCCESS, + payload: { + data, + status, + }, + }; +} + export function getTableQualityChecks( key: string ): GetTableQualityChecksRequest { @@ -338,6 +377,10 @@ export interface TableMetadataReducerState { data: PreviewData; status: number | null; }; + freshness: { + data: PreviewData; + status: number | null; + }; statusCode: number | null; tableData: TableMetadata; tableOwners: TableOwnerReducerState; @@ -396,6 +439,12 @@ export default function reducer( case GetPreviewData.FAILURE: case GetPreviewData.SUCCESS: return { ...state, preview: (action).payload }; + case GetFreshnessData.FAILURE: + case GetFreshnessData.SUCCESS: + return { + ...state, + freshness: (action).payload, + }; case UpdateTableOwner.REQUEST: case UpdateTableOwner.FAILURE: case UpdateTableOwner.SUCCESS: diff --git a/frontend/amundsen_application/static/js/ducks/tableMetadata/sagas.ts b/frontend/amundsen_application/static/js/ducks/tableMetadata/sagas.ts index 36d6f29bda..6f1886fba5 100644 --- a/frontend/amundsen_application/static/js/ducks/tableMetadata/sagas.ts +++ b/frontend/amundsen_application/static/js/ducks/tableMetadata/sagas.ts @@ -13,6 +13,8 @@ import { getColumnDescriptionSuccess, getPreviewDataFailure, getPreviewDataSuccess, + getFreshnessDataFailure, + getFreshnessDataSuccess, getTableQualityChecksSuccess, getTableQualityChecksFailure, } from './reducer'; @@ -20,6 +22,7 @@ import { import { GetPreviewData, GetPreviewDataRequest, + GetFreshnessDataRequest, GetTableData, GetTableDataRequest, GetColumnDescription, @@ -30,6 +33,7 @@ import { UpdateColumnDescriptionRequest, UpdateTableDescription, UpdateTableDescriptionRequest, + GetFreshnessData, GetTableQualityChecksRequest, GetTableQualityChecks, } from './types'; @@ -182,6 +186,25 @@ export function* getPreviewDataWatcher(): SagaIterator { yield takeLatest(GetPreviewData.REQUEST, getPreviewDataWorker); } +export function* getFreshnessDataWorker( + action: GetFreshnessDataRequest +): SagaIterator { + try { + const response = yield call( + API.getFreshnessData, + action.payload.queryParams + ); + const { data, status } = response; + yield put(getFreshnessDataSuccess(data, status)); + } catch (error) { + const { data, status } = error; + yield put(getFreshnessDataFailure(data, status)); + } +} +export function* getFreshnessDataWatcher(): SagaIterator { + yield takeLatest(GetFreshnessData.REQUEST, getFreshnessDataWorker); +} + export function* getTableQualityChecksWorker( action: GetTableQualityChecksRequest ): SagaIterator { diff --git a/frontend/amundsen_application/static/js/ducks/tableMetadata/types.ts b/frontend/amundsen_application/static/js/ducks/tableMetadata/types.ts index 272233b933..d21b58d590 100644 --- a/frontend/amundsen_application/static/js/ducks/tableMetadata/types.ts +++ b/frontend/amundsen_application/static/js/ducks/tableMetadata/types.ts @@ -137,6 +137,25 @@ export interface GetPreviewDataResponse { }; } +export enum GetFreshnessData { + REQUEST = 'amundsen/freshness/GET_FRESHNESS_DATA_REQUEST', + SUCCESS = 'amundsen/freshness/GET_FRESHNESS_DATA_SUCCESS', + FAILURE = 'amundsen/freshness/GET_FRESHNESS_DATA_FAILURE', +} +export interface GetFreshnessDataRequest { + type: GetFreshnessData.REQUEST; + payload: { + queryParams: TablePreviewQueryParams; + }; +} +export interface GetFreshnessDataResponse { + type: GetFreshnessData.SUCCESS | GetFreshnessData.FAILURE; + payload: { + data: PreviewData; + status: number | null; + }; +} + export enum UpdateTableOwner { REQUEST = 'amundsen/tableMetadata/UPDATE_TABLE_OWNER_REQUEST', SUCCESS = 'amundsen/tableMetadata/UPDATE_TABLE_OWNER_SUCCESS', diff --git a/frontend/amundsen_application/static/js/fixtures/globalState.ts b/frontend/amundsen_application/static/js/fixtures/globalState.ts index 68a44077ad..539b52de01 100644 --- a/frontend/amundsen_application/static/js/fixtures/globalState.ts +++ b/frontend/amundsen_application/static/js/fixtures/globalState.ts @@ -233,6 +233,10 @@ const globalState: GlobalState = { data: {}, status: null, }, + freshness: { + data: {}, + status: null, + }, statusCode: 200, tableData: { badges: [], diff --git a/frontend/amundsen_application/static/js/pages/TableDetailPage/DataFreshnessButton/index.tsx b/frontend/amundsen_application/static/js/pages/TableDetailPage/DataFreshnessButton/index.tsx new file mode 100644 index 0000000000..d142f06d92 --- /dev/null +++ b/frontend/amundsen_application/static/js/pages/TableDetailPage/DataFreshnessButton/index.tsx @@ -0,0 +1,206 @@ +// Copyright Contributors to the Amundsen project. +// SPDX-License-Identifier: Apache-2.0 + +import * as React from 'react'; +import { Modal, OverlayTrigger, Popover } from 'react-bootstrap'; +import Linkify from 'react-linkify'; +import { connect } from 'react-redux'; + +import { getFreshnessData } from 'ducks/tableMetadata/reducer'; +import { GlobalState } from 'ducks/rootReducer'; +import { PreviewDataTable } from 'features/PreviewData'; +import { + PreviewData, + TablePreviewQueryParams, + TableMetadata, +} from 'interfaces'; +import { logClick } from 'utils/analytics'; + +enum FetchingStatus { + ERROR = 'error', + LOADING = 'loading', + SUCCESS = 'success', + UNAVAILABLE = 'unavailable', +} + +export interface StateFromProps { + freshnessData: PreviewData; + status: FetchingStatus; + tableData: TableMetadata; +} + +export interface DispatchFromProps { + getFreshness: (queryParams: TablePreviewQueryParams) => void; +} + +export interface ComponentProps { + modalTitle: string; +} + +type DataFreshnessButtonProps = StateFromProps & + DispatchFromProps & + ComponentProps; + +interface DataFreshnessButtonState { + showModal: boolean; +} + +export function getStatusFromCode(httpErrorCode: number | null) { + switch (httpErrorCode) { + case null: + return FetchingStatus.LOADING; + case 200: + // ok + return FetchingStatus.SUCCESS; + case 501: + // No updated_at or inserted_at column + return FetchingStatus.UNAVAILABLE; + default: + // default to generic error + return FetchingStatus.ERROR; + } +} + +export class DataFreshnessButton extends React.Component< + DataFreshnessButtonProps, + DataFreshnessButtonState +> { + constructor(props) { + super(props); + + this.state = { + showModal: false, + }; + } + + componentDidMount() { + const { tableData, getFreshness } = this.props; + + getFreshness({ + database: tableData.database, + schema: tableData.schema, + tableName: tableData.name, + cluster: tableData.cluster, + }); + } + + handleClose = () => { + this.setState({ showModal: false }); + }; + + handleClick = (e) => { + logClick(e); + this.setState({ showModal: true }); + }; + + renderModalBody() { + const { freshnessData, status } = this.props; + + if (status === FetchingStatus.SUCCESS) { + return ; + } + + if (status === FetchingStatus.ERROR) { + return ( +
+ {freshnessData.error_text} +
+ ); + } + + return null; + } + + renderFreshnessButton() { + const { freshnessData, status } = this.props; + + // Based on the state, the preview button will show different things. + let buttonText = 'Fetching...'; + let disabled = true; + let popoverText = 'The data freshness is being fetched'; + + switch (status) { + case FetchingStatus.SUCCESS: + buttonText = 'Freshness'; + disabled = false; + break; + case FetchingStatus.UNAVAILABLE: + buttonText = 'Freshness'; + popoverText = 'This feature has not been configured by your service'; + break; + case FetchingStatus.ERROR: + buttonText = 'Freshness'; + popoverText = + freshnessData.error_text || + 'An internal server error has occurred, please contact service admin'; + break; + default: + break; + } + + const freshnessButton = ( + + ); + + if (!disabled) { + return freshnessButton; + } + + // when button is disabled, render button with Popover + const popoverHover = ( + {popoverText} + ); + return ( + + {/* Disabled buttons don't trigger hover/focus events so we need a wrapper */} +
{freshnessButton}
+
+ ); + } + + render() { + const { modalTitle } = this.props; + const { showModal } = this.state; + return ( + <> + {this.renderFreshnessButton()} + + + {modalTitle} + + {this.renderModalBody()} + + + ); + } +} + +export const mapStateToProps = (state: GlobalState) => ({ + freshnessData: state.tableMetadata.freshness.data, + status: getStatusFromCode(state.tableMetadata.freshness.status), + tableData: state.tableMetadata.tableData, +}); + +export const mapDispatchToProps = (dispatch: any) => ({ + getFreshness: (queryParams: TablePreviewQueryParams) => { + dispatch(getFreshnessData(queryParams)); + }, +}); + +export default connect( + mapStateToProps, + mapDispatchToProps +)(DataFreshnessButton); diff --git a/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx b/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx index f05aea13b8..3090c3216a 100644 --- a/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx +++ b/frontend/amundsen_application/static/js/pages/TableDetailPage/index.tsx @@ -65,8 +65,8 @@ import { } from 'interfaces'; import DataPreviewButton from './DataPreviewButton'; +import DataFreshnessButton from './DataFreshnessButton'; import ExploreButton from './ExploreButton'; -import LineageButton from './LineageButton'; import FrequentUsers from './FrequentUsers'; import LineageLink from './LineageLink'; import LineageList from './LineageList'; @@ -460,8 +460,8 @@ export class TableDetail extends React.Component<
- +
diff --git a/frontend/tests/unit/api/freshness/__init__.py b/frontend/tests/unit/api/freshness/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/frontend/tests/unit/api/freshness/test_v0.py b/frontend/tests/unit/api/freshness/test_v0.py new file mode 100644 index 0000000000..e839958254 --- /dev/null +++ b/frontend/tests/unit/api/freshness/test_v0.py @@ -0,0 +1,76 @@ +# Copyright Contributors to the Amundsen project. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +import json +from flask import make_response, jsonify +from http import HTTPStatus +from typing import Dict + +from flask import Response + +from amundsen_application import create_app +from amundsen_application.api.freshness import v0 +from amundsen_application.base.base_data_freshness_client import BaseDataFreshnessClient + +local_app = create_app('amundsen_application.config.TestConfig', 'tests/templates') +DATA_FRESHNESS_CLIENT = 'tests.unit.api.freshness.test_v0.DataFreshnessClient' + + +class DataFreshnessClient(BaseDataFreshnessClient): + def __init__(self) -> None: + pass + + def get_freshness_data(self, params: Dict, optionalHeaders: Dict = None) -> Response: + return make_response(jsonify({ + 'freshness_data': {'columns': [{}], 'data': [{'latest updated_at': '2021-07-14 13:52:51.807 +0000'}]} + }), HTTPStatus.OK) + + +class DataFreshnessTest(unittest.TestCase): + + def setUp(self) -> None: + local_app.config['DATA_FRESHNESS_CLIENT_ENABLED'] = True + + def test_no_client_class(self) -> None: + """ + Test that Not Implemented error is raised when FRESHNESS_CLIENT is None + :return: + """ + # Reset side effects of other tests to ensure that the results are the + # same regardless of execution order + v0.DATA_FRESHNESS_CLIENT_CLASS = None + v0.DATA_FRESHNESS_CLIENT_INSTANCE = None + + local_app.config['DATA_FRESHNESS_CLIENT'] = None + with local_app.test_client() as test: + response = test.post('/api/freshness/v0/') + self.assertEqual(response.status_code, HTTPStatus.NOT_IMPLEMENTED) + + @unittest.mock.patch('amundsen_application.api.freshness.v0._get_table_metadata') + def test_client_response(self, mock_get_table_metadata: unittest.mock.Mock) -> None: + """ + Test response + """ + mock_get_table_metadata.return_value = {} + + local_app.config['DATA_FRESHNESS_CLIENT'] = DATA_FRESHNESS_CLIENT + + expected_response_json = { + 'freshnessData': { + 'columns': [{}], + 'data': [{'latest updated_at': '2021-07-14 13:52:51.807 +0000'}]} + } + + with local_app.test_client() as test: + request_data = { + 'database': 'fake_db', + 'cluster': 'fake_cluster', + 'schema': 'fake_schema', + 'tableName': 'fake_table' + } + post_response = test.post('/api/freshness/v0/', + data=json.dumps(request_data), + content_type='application/json') + self.assertEqual(post_response.status_code, HTTPStatus.OK) + self.assertEqual(post_response.json, expected_response_json)