diff --git a/.github/workflows/code-review.yml b/.github/workflows/code-review.yml deleted file mode 100644 index 4630e40094..0000000000 --- a/.github/workflows/code-review.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: Code Review Robot - -permissions: - contents: read - pull-requests: write - -on: - pull_request: - types: [opened, reopened, synchronize] - -jobs: - code-review: - # if: ${{ contains(github.event.*.labels.*.name, 'gpt review') }} # Optional; to run only when a label is attached - runs-on: ubuntu-latest - steps: - - name: Code Review - uses: anc95/ChatGPT-CodeReview@main - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - # Optional - LANGUAGE: Chinese - OPENAI_API_ENDPOINT: https://api.openai.com/v1 - MODEL: gpt-3.5-turbo # https://platform.openai.com/docs/models - PROMPT: Please check if there are any confusions or irregularities in the following code diffs. If there are, please point them out. If not, please reply with "No problem". # https://platform.openai.com/docs/api-reference/completions/create#completions/create-prompt - top_p: 1 # https://platform.openai.com/docs/api-reference/chat/create#chat/create-top_p - temperature: 1 # https://platform.openai.com/docs/api-reference/chat/create#chat/create-temperature - max_tokens: 10000 - MAX_PATCH_LENGTH: 10000 # if the patch/diff length is large than MAX_PATCH_LENGTH, will be ignored and won't review. By default, with no MAX_PATCH_LENGTH set, there is also no limit for the patch/diff length. \ No newline at end of file diff --git a/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java b/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java index 5c6b32b33e..60b57c77ea 100644 --- a/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java +++ b/dinky-admin/src/main/java/org/dinky/mapper/TaskMapper.java @@ -39,8 +39,6 @@ @Mapper public interface TaskMapper extends SuperMapper { - Integer queryAllSizeByName(String name); - List queryOnLineTaskByDoneStatus( @Param("parentIds") List parentIds, @Param("stepIds") List stepIds, diff --git a/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java index 395fc74c71..8f644c2b18 100644 --- a/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/catalogue/impl/CatalogueServiceImpl.java @@ -472,7 +472,7 @@ private String getFileText(File sourceFile) { } } } catch (Exception e) { - log.error("read file error, {} ", e); + log.error("read file error, {} ", e.getMessage(), e); } return sb.toString(); } @@ -555,7 +555,6 @@ public Result deleteCatalogueById(Integer catalogueId) { if (CollUtil.isNotEmpty(metricListByTaskId)) { metricListByTaskId.forEach(metrics -> monitorService.removeById(metrics.getId())); - // todo: 需要删除 paimon 中的监控数据, 但是 paimon 中没有提供删除接口 } } diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java index 56f206396f..b0bb24b11a 100644 --- a/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java +++ b/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java @@ -33,6 +33,7 @@ import org.dinky.data.dto.TaskDTO; import org.dinky.data.dto.TaskRollbackVersionDTO; import org.dinky.data.dto.TaskSubmitDto; +import org.dinky.data.enums.CatalogTypeMappingEnum; import org.dinky.data.enums.GatewayType; import org.dinky.data.enums.JobLifeCycle; import org.dinky.data.enums.JobStatus; @@ -87,11 +88,9 @@ import org.dinky.service.SavepointsService; import org.dinky.service.TaskService; import org.dinky.service.TaskVersionService; -import org.dinky.service.UDFService; import org.dinky.service.UDFTemplateService; import org.dinky.service.UserService; import org.dinky.service.catalogue.CatalogueService; -import org.dinky.service.resource.ResourcesService; import org.dinky.service.task.BaseTask; import org.dinky.utils.FragmentVariableUtils; import org.dinky.utils.JsonUtils; @@ -119,6 +118,7 @@ import javax.annotation.Resource; +import org.jetbrains.annotations.NotNull; import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.Lazy; @@ -126,6 +126,7 @@ import org.springframework.transaction.annotation.Transactional; import org.springframework.web.multipart.MultipartFile; +import com.alibaba.druid.pool.DruidDataSource; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.fasterxml.jackson.core.JsonProcessingException; @@ -164,8 +165,7 @@ public class TaskServiceImpl extends SuperServiceImpl implemen private final DataSourceProperties dsProperties; private final UserService userService; private final ApplicationContext applicationContext; - private final UDFService udfService; - private final ResourcesService resourcesService; + private final DruidDataSource druidProperties; @Resource @Lazy @@ -685,19 +685,7 @@ public Task initDefaultFlinkSQLEnv(Integer tenantId) { Task defaultFlinkSQLEnvTask = getTaskByNameAndTenantId(name, tenantId); - String sql = String.format( - "create catalog my_catalog with(\n " - + "'type' = 'dinky_mysql',\n" - + " 'username' = " - + "'%s',\n " - + "'password' = '%s',\n" - + " 'url' = '%s'\n" - + ")%suse catalog my_catalog%s", - dsProperties.getUsername(), - dsProperties.getPassword(), - dsProperties.getUrl(), - FlinkSQLConstant.SEPARATOR, - FlinkSQLConstant.SEPARATOR); + String sql = getStatementByCatalogType(CatalogTypeMappingEnum.ofDbType(druidProperties.getDbType())); if (null != defaultFlinkSQLEnvTask) { defaultFlinkSQLEnvTask.setStatement(sql); @@ -720,6 +708,31 @@ public Task initDefaultFlinkSQLEnv(Integer tenantId) { return defaultFlinkSQLEnvTask; } + private @NotNull String getStatementByCatalogType(CatalogTypeMappingEnum catalogTypeMappingEnum) { + String sql = String.format( + "create catalog my_catalog_%s with(\n " + + "'type' = '%s',\n" + + " 'username' = " + + "'%s',\n " + + "'password' = '%s',\n" + + " 'url' = '%s'\n" + + ")%suse catalog my_catalog_%s %s", + catalogTypeMappingEnum.getCatalogTypeName(), + catalogTypeMappingEnum.getCatalogTypeName(), + dsProperties.getUsername(), + dsProperties.getPassword(), + dsProperties.getUrl(), + FlinkSQLConstant.SEPARATOR, + catalogTypeMappingEnum.getCatalogTypeName(), + FlinkSQLConstant.SEPARATOR); + log.debug( + "Init default flink sql env sql:{}, yours dbType is:{}, catalogName is:{}", + sql, + catalogTypeMappingEnum.getDbType(), + catalogTypeMappingEnum.getCatalogTypeName()); + return sql; + } + @Override public Task getTaskByNameAndTenantId(String name, Integer tenantId) { return baseMapper.getTaskByNameAndTenantId(name, tenantId); @@ -778,7 +791,8 @@ public String getTaskAPIAddress() { @Override public Integer queryAllSizeByName(String name) { - return baseMapper.queryAllSizeByName(name); + Long value = baseMapper.selectCount(new LambdaQueryWrapper().likeRight(Task::getName, name + "-")); + return Math.toIntExact(value); } @Override diff --git a/dinky-admin/src/main/resources/application-pgsql.yml b/dinky-admin/src/main/resources/application-postgresql.yml similarity index 100% rename from dinky-admin/src/main/resources/application-pgsql.yml rename to dinky-admin/src/main/resources/application-postgresql.yml diff --git a/dinky-admin/src/main/resources/application.yml b/dinky-admin/src/main/resources/application.yml index 0855ab1b26..a71804ab9e 100644 --- a/dinky-admin/src/main/resources/application.yml +++ b/dinky-admin/src/main/resources/application.yml @@ -11,12 +11,12 @@ spring: application: name: Dinky profiles: - # The h2 database is used by default. If you need to use other databases, please set the configuration active to: mysql, currently supports [mysql, pgsql, h2] + # The h2 database is used by default. If you need to use other databases, please set the configuration active to: mysql, currently supports [mysql, postgresql, h2] # If you use mysql database, please configure mysql database connection information in application-mysql.yml - # If you use pgsql database, please configure pgsql database connection information in application-pgsql.yml + # If you use pgsql database, please configure pgsql database connection information in application-postgresql.yml # If you use the h2 database, please configure the h2 database connection information in application-h2.yml, # note: the h2 database is only for experience use, and the related data that has been created cannot be migrated, please use it with caution - active: ${DB_ACTIVE:h2} #[h2,mysql,pgsql] + active: ${DB_ACTIVE:h2} #[h2,mysql,postgresql] include: - jmx - flyway diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/V20241127.1.3.0__release.sql b/dinky-admin/src/main/resources/db/migration/pgsql/V20241127.1.3.0__release.sql deleted file mode 100644 index d5b522e889..0000000000 --- a/dinky-admin/src/main/resources/db/migration/pgsql/V20241127.1.3.0__release.sql +++ /dev/null @@ -1,78 +0,0 @@ - -SELECT add_column_if_not_exists('public','dinky_alert_history', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_history', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_job_history', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_job_instance', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_sys_menu', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_sys_role_menu', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_row_permissions', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_user_role', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_user_tenant', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_udf_manage', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_udf_template', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_catalogue', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_cluster_configuration', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_cluster', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_dashboard', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_database', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_flink_document', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_fragment', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_git_project', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_metrics', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_sys_operate_log', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_resources', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_savepoints', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_sys_token', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_task', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_task_version', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_alert_group', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_alert_instance', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_alert_rules', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','dinky_alert_template', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -ALTER TABLE public.dinky_alert_history DROP COLUMN IF EXISTS is_deleted; -SELECT add_column_if_not_exists('public','dinky_sys_login_log', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - -SELECT add_column_if_not_exists('public','metadata_column', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); -SELECT add_column_if_not_exists('public','metadata_database', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); -SELECT add_column_if_not_exists('public','metadata_database_property', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); -SELECT add_column_if_not_exists('public','metadata_function', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); -SELECT add_column_if_not_exists('public','metadata_table', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); -SELECT add_column_if_not_exists('public','metadata_table_property', 'is_delete', 'BOOLEAN', 'FALSE', 'is delete '); - - -CREATE UNIQUE INDEX unx_metadata_column ON metadata_column (table_id, column_name, is_delete); -CREATE UNIQUE INDEX unx_metadata_database ON metadata_database (database_name, is_delete); -CREATE UNIQUE INDEX unx_metadata_database_property ON metadata_database_property (database_id, key, is_delete); -CREATE UNIQUE INDEX unx_metadata_function ON metadata_function (function_name, database_id, is_delete); -CREATE UNIQUE INDEX unx_metadata_table ON metadata_table (table_name, database_id, is_delete); -CREATE UNIQUE INDEX unx_metadata_table_property ON metadata_table_property (table_id, key, is_delete); diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/R20240708.1.1.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/R20240708.1.1.0__release.sql similarity index 100% rename from dinky-admin/src/main/resources/db/migration/pgsql/R20240708.1.1.0__release.sql rename to dinky-admin/src/main/resources/db/migration/postgresql/R20240708.1.1.0__release.sql diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/R20240729.1.1.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/R20240729.1.1.0__release.sql similarity index 100% rename from dinky-admin/src/main/resources/db/migration/pgsql/R20240729.1.1.0__release.sql rename to dinky-admin/src/main/resources/db/migration/postgresql/R20240729.1.1.0__release.sql diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/V20240506.1.0.2__baseline.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20240506.1.0.2__baseline.sql similarity index 100% rename from dinky-admin/src/main/resources/db/migration/pgsql/V20240506.1.0.2__baseline.sql rename to dinky-admin/src/main/resources/db/migration/postgresql/V20240506.1.0.2__baseline.sql diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/V20240708.1.1.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20240708.1.1.0__release.sql similarity index 100% rename from dinky-admin/src/main/resources/db/migration/pgsql/V20240708.1.1.0__release.sql rename to dinky-admin/src/main/resources/db/migration/postgresql/V20240708.1.1.0__release.sql diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/V20240729.1.1.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20240729.1.1.0__release.sql similarity index 100% rename from dinky-admin/src/main/resources/db/migration/pgsql/V20240729.1.1.0__release.sql rename to dinky-admin/src/main/resources/db/migration/postgresql/V20240729.1.1.0__release.sql diff --git a/dinky-admin/src/main/resources/db/migration/pgsql/V20240827.1.2.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20240827.1.2.0__release.sql similarity index 100% rename from dinky-admin/src/main/resources/db/migration/pgsql/V20240827.1.2.0__release.sql rename to dinky-admin/src/main/resources/db/migration/postgresql/V20240827.1.2.0__release.sql diff --git a/dinky-admin/src/main/resources/db/migration/postgresql/V20241127.1.3.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20241127.1.3.0__release.sql new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dinky-admin/src/main/resources/db/migration/postgresql/V20241203.1.2.0__release.sql b/dinky-admin/src/main/resources/db/migration/postgresql/V20241203.1.2.0__release.sql new file mode 100644 index 0000000000..0e45e82425 --- /dev/null +++ b/dinky-admin/src/main/resources/db/migration/postgresql/V20241203.1.2.0__release.sql @@ -0,0 +1,41 @@ + + + +-- Table structure for public.dinky_user + +CREATE TABLE IF NOT EXISTS public.dinky_user_backup +( + id serial PRIMARY KEY NOT NULL, + username varchar(50) NOT NULL, + user_type int DEFAULT 1, + password varchar(50), + nickname varchar(50), + worknum varchar(50), + avatar bytea, + mobile varchar(20), + enabled boolean NOT NULL DEFAULT true, + super_admin_flag boolean DEFAULT false, + is_delete boolean NOT NULL DEFAULT false, + create_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + update_time timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP +); +COMMENT ON TABLE public.dinky_user_backup IS 'user'; +COMMENT ON COLUMN public.dinky_user_backup.id IS 'id'; +COMMENT ON COLUMN public.dinky_user_backup.username IS 'username'; +COMMENT ON COLUMN public.dinky_user_backup.password IS 'password'; +COMMENT ON COLUMN public.dinky_user_backup.nickname IS 'nickname'; +COMMENT ON COLUMN public.dinky_user_backup.worknum IS 'worknum'; +COMMENT ON COLUMN public.dinky_user_backup.avatar IS 'avatar'; +COMMENT ON COLUMN public.dinky_user_backup.mobile IS 'mobile phone'; +COMMENT ON COLUMN public.dinky_user_backup.enabled IS 'enabled'; +COMMENT ON COLUMN public.dinky_user_backup.super_admin_flag IS 'is super admin(0:false,1true)'; + + +insert into public.dinky_user_backup(id, username, user_type, password, nickname, worknum, avatar, mobile, enabled, super_admin_flag, is_delete, create_time, update_time) +select id, username, user_type, password, nickname, worknum, avatar, mobile, enabled, + case when super_admin_flag = 0 then false else true end as super_admin_flag + , is_delete, create_time, update_time +from public.dinky_user; + +drop table if exists public.dinky_user; +alter table public.dinky_user_backup rename to dinky_user; diff --git a/dinky-admin/src/main/resources/mapper/TaskMapper.xml b/dinky-admin/src/main/resources/mapper/TaskMapper.xml index 4f1f731d7e..031633ab65 100644 --- a/dinky-admin/src/main/resources/mapper/TaskMapper.xml +++ b/dinky-admin/src/main/resources/mapper/TaskMapper.xml @@ -46,11 +46,7 @@ - +