[Feature][Server] Add a filter input under the column check box list …

…when running the data profile(#240)
datavane · Oct 23, 2023 · b510de1 · b510de1
1 parent 45e5ff9
commit b510de1
Show file tree

Hide file tree

Showing 7 changed files with 85 additions and 18 deletions.
diff --git a/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogController.java b/datavines-server/src/main/java/io/datavines/server/api/controller/CatalogController.java
@@ -95,6 +95,12 @@ public Object executeDataProfileJobWithColumns(@Valid @RequestBody RunProfileReq
         return catalogEntityInstanceService.executeDataProfileJob(runProfileRequest);
     }
 
+    @ApiOperation(value = "get filter", response = String.class)
+    @GetMapping(value = "/profile/filter/{uuid}")
+    public Object getProfileJobFilter(@PathVariable String uuid) {
+        return catalogEntityInstanceService.getProfileJobFilter(uuid);
+    }
+
     @ApiOperation(value = "get selected column list", response = OptionItem.class, responseContainer = "list")
     @GetMapping(value = "/profile/selected-columns/{uuid}")
     public Object getProfileJobSelectedColumns(@PathVariable String uuid) {

diff --git a/...erver/src/main/java/io/datavines/server/api/dto/bo/catalog/profile/RunProfileRequest.java b/...erver/src/main/java/io/datavines/server/api/dto/bo/catalog/profile/RunProfileRequest.java
@@ -30,4 +30,6 @@ public class RunProfileRequest {
     private boolean selectAll;
 
     private List<String> selectedColumnList;
+
+    private String filter;
 }
diff --git a/...er/src/main/java/io/datavines/server/repository/service/CatalogEntityInstanceService.java b/...er/src/main/java/io/datavines/server/repository/service/CatalogEntityInstanceService.java
@@ -82,6 +82,8 @@ public interface CatalogEntityInstanceService extends IService<CatalogEntityInst
 
     List<String> getProfileJobSelectedColumns(String uuid);
 
+    String getProfileJobFilter(String uuid);
+
     boolean deleteInstanceByDataSourceId(Long datasourceId);
 
     IPage<JobExecutionVO> profileJobExecutionPage(String uuid, Integer pageNumber, Integer pageSize);

diff --git a/...in/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java b/...in/java/io/datavines/server/repository/service/impl/CatalogEntityInstanceServiceImpl.java
@@ -825,6 +825,7 @@ public long executeDataProfileJob(RunProfileRequest runProfileRequest) {
     @Override
     public long executeDataProfileJob(RunProfileRequest runProfileRequest, int runningNow) {
         String uuid = runProfileRequest.getUuid();
+        String filter = runProfileRequest.getFilter() == null ? "" : runProfileRequest.getFilter();
         CatalogEntityInstance entityInstance = getCatalogEntityInstance(uuid);
         if (entityInstance == null) {
             return -1L;
@@ -856,6 +857,7 @@ public long executeDataProfileJob(RunProfileRequest runProfileRequest, int runni
                 metricParameter.put("table", tableName);
                 metricParameter.put("entity_uuid", entityInstance.getUuid());
                 metricParameter.put("actual_value_type", "count");
+                metricParameter.put("filter", filter);
                 baseJobParameter.setMetricParameter(metricParameter);
                 baseJobParameter.setExpectedType("fix_value");
                 jobParameters.add(baseJobParameter);
@@ -909,6 +911,7 @@ public long executeDataProfileJob(RunProfileRequest runProfileRequest, int runni
                 metricParameter1.put("column", values[2]);
                 metricParameter1.put("entity_uuid", catalogEntityInstance.getUuid());
                 metricParameter1.put("actual_value_type", "count");
+                metricParameter1.put("filter", filter);
                 baseJobParameter.setMetricParameter(metricParameter1);
                 baseJobParameter.setExpectedType("fix_value");
                 jobParameters.add(baseJobParameter);
@@ -998,4 +1001,32 @@ public IPage<JobExecutionVO> profileJobExecutionPage(String uuid, Integer pageNu
 
         return jobExecutionService.getJobExecutionPage(pageParam);
     }
+
+    @Override
+    public String getProfileJobFilter(String uuid) {
+        CatalogEntityMetricJobRel rel = catalogEntityMetricJobRelService.getOne(new QueryWrapper<CatalogEntityMetricJobRel>().eq("entity_uuid", uuid).eq("metric_job_type", DATA_PROFILE.getDescription()));
+        String filter = "";
+        if (rel == null) {
+            return filter;
+        }
+        long jobId = rel.getMetricJobId();
+        Job job = jobService.getById(jobId);
+        if (job == null) {
+            return filter;
+        }
+        String parameter = job.getParameter();
+        try {
+            List<BaseJobParameter> jobParameterList = JSONUtils.toList(parameter,BaseJobParameter.class);
+            for (BaseJobParameter baseJobParameter : jobParameterList) {
+                Map<String, Object> metricParameter = baseJobParameter.getMetricParameter();
+                filter = (String) metricParameter.get("filter");
+                if(org.apache.commons.lang3.StringUtils.isNotBlank(filter)){
+                    break;
+                }
+            }
+        } catch (Exception e){
+            log.error("data profile job parameter:[{}] get filter error, ", e);
+        }
+        return filter;
+    }
 }
diff --git a/datavines-ui/Editor/components/Database/index.tsx b/datavines-ui/Editor/components/Database/index.tsx
@@ -26,6 +26,7 @@ import Schedule from '@/view/Main/HomeDetail/Jobs/components/Schedule';
 import SearchForm from './SearchForm';
 import store from '@/store';
 import { useLogger } from '@/view/Main/HomeDetail/Jobs/useLogger';
+import TextArea from "antd/lib/input/TextArea";
 
 type DIndexProps = {
     onShowModal?: (...args: any[]) => any;
@@ -468,13 +469,19 @@ const Index = ({ onShowModal, afterClose }:DIndexProps) => {
         setChooesColList(res);
         const resCheck = await $http.get(`/catalog/profile/selected-columns/${selectDatabases[currentIndex]?.uuid}?uuid=${selectDatabases[currentIndex]?.uuid}`);
         setColCheckList(resCheck);
+        const resFilter = await $http.get(`/catalog/profile/filter/${selectDatabases[currentIndex]?.uuid}?uuid=${selectDatabases[currentIndex]?.uuid}`);
+        setProfileFilterText(resFilter)
+        filterForm.setFieldValue('filter', resFilter);
         setChooesColModalOpen(true);
     };
+    const [filterForm] = Form.useForm();
+    const [profileFilterText, setProfileFilterText] = useState('');
     const runProfile = async () => {
         await $http.post('/catalog/profile/execute-select-columns', {
             selectAll: chooesColList.length === colCheckList.length,
             selectedColumnList: colCheckList,
             uuid: selectDatabases[currentIndex]?.uuid,
+            filter: profileFilterText
         }).then(() => {
             setChooesColModalOpen(false);
             message.success(intl.formatMessage({ id: 'common_success' }));
@@ -986,7 +993,7 @@ const Index = ({ onShowModal, afterClose }:DIndexProps) => {
             <Modal
                 width="400px"
                 open={chooesColModalOpen}
-                title={intl.formatMessage({ id: 'job_choose_col' })}
+                title={intl.formatMessage({ id: 'job_config_info' })}
                 maskClosable={false}
                 footer={[
                     <Button key="good" onClick={() => setChooesColModalOpen(false)}>
@@ -999,24 +1006,41 @@ const Index = ({ onShowModal, afterClose }:DIndexProps) => {
                 onCancel={() => setChooesColModalOpen(false)}
                 destroyOnClose
             >
-                <Table
+                <Form
+                    form={filterForm}
+                    layout='vertical'
                     style={{ marginTop: 20 }}
-                    rowSelection={{
-                        type: 'checkbox',
-                        onChange: (val:any[]) => setColCheckList(val),
-                        defaultSelectedRowKeys: colCheckList,
-                    }}
-                    size="small"
-                    dataSource={chooesColList}
-                    columns={chooseColumns}
-                    pagination={
-                        false
-                    }
-                    rowKey="name"
-                    scroll={{
-                        y: 300,
-                    }}
-                />
+                >
+                    <Form.Item
+                        label={`${intl.formatMessage({ id: 'job_choose_col' })}`}
+                        name='colList'
+                    >
+                        <Table
+                            rowSelection={{
+                                type: 'checkbox',
+                                onChange: (val:any[]) => setColCheckList(val),
+                                defaultSelectedRowKeys: colCheckList,
+                            }}
+                            size="small"
+                            dataSource={chooesColList}
+                            columns={chooseColumns}
+                            pagination={
+                                false
+                            }
+                            rowKey="name"
+                            scroll={{
+                                y: 300,
+                            }}
+                        />
+                    </Form.Item>
+                    <Form.Item
+                        label={`${intl.formatMessage({ id: 'dv_metric_condition' })}`}
+                        name='filter'
+                    >
+                        <TextArea autoComplete="off" value={profileFilterText} onChange={e => setProfileFilterText(e.target.value)}/>
+                    </Form.Item>
+                </Form>
+
             </Modal>
             <RenderModal />
             <Drawer width="50%" title={intl.formatMessage({ id: 'job_profile_execution_history' })} placement="right" onClose={onDrawerClose} open={openDrawer}>

diff --git a/datavines-ui/src/locale/en_US.ts b/datavines-ui/src/locale/en_US.ts
@@ -222,6 +222,7 @@ export default {
     job_avg: 'Avg',
     job_stdDev: 'StdDev',
     job_variance: 'Variance',
+    job_config_info: 'Job Config Info',
 
     job_choose_col: 'Choose Col',
     job_search_col: 'Search Col',

diff --git a/datavines-ui/src/locale/zh_CN.ts b/datavines-ui/src/locale/zh_CN.ts
@@ -227,6 +227,7 @@ export default {
     jobs_status: '执行状态',
     jobs_execution_time: '执行时间',
     job_search_col: '搜索列',
+    job_config_info: '配置信息',
 
     warn_sLAs_name: '名称',
     warn_sLAs_type: '类型',