Skip to content

Commit

Permalink
Merge pull request #952 from Zosimer/dss-datamodel
Browse files Browse the repository at this point in the history
Linkis1.2.0[datamodel][dataassets][datawarehouse]
  • Loading branch information
zqburde authored Dec 30, 2022
2 parents d8a8515 + 050f510 commit 8469876
Show file tree
Hide file tree
Showing 934 changed files with 74,061 additions and 3,435 deletions.
4 changes: 4 additions & 0 deletions assembly/bin/executeSQL.sh
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,10 @@ function executeSQL() {
isSuccess "source dss_dataapi_ddl.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/dss_guide_ddl.sql"
isSuccess "source dss_guide_ddl.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/datamodel.sql"
isSuccess "source datamodel.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/datawarehouse.sql"
isSuccess "source datawarehouse.sql"
mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
isSuccess "source dss_dml_real.sql"
echo "Rebuild the table"
Expand Down
54 changes: 43 additions & 11 deletions assembly/bin/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -121,11 +121,18 @@ function replaceCommonIp() {
DSS_DATA_API_SERVER_PORT=9208
fi

if [ -z "$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP" ]; then
DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=$LOCAL_IP
if [ -z "$DSS_DATA_ASSETS_SERVER_INSTALL_IP" ]; then
DSS_DATA_ASSETS_SERVER_INSTALL_IP=$LOCAL_IP
fi
if [ -z "$DSS_DATA_GOVERNANCE_SERVER_PORT" ]; then
DSS_DATA_GOVERNANCE_SERVER_PORT=9209
if [ -z "$DSS_DATA_ASSETS_SERVER_INSTALL_IP" ]; then
DSS_DATA_ASSETS_SERVER_INSTALL_IP=9300
fi
if [ -z "$DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP" ]; then
DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP=9400
fi

if [ -z "$DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP" ]; then
DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP=9500
fi

if [ -z "$DSS_GUIDE_SERVER_INSTALL_IP" ]; then
Expand Down Expand Up @@ -343,13 +350,38 @@ function installDssProject() {
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
installPackage

###install dss-data-governance-server
SERVER_NAME=dss-data-governance-server
SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATA_GOVERNANCE_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-governance/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-data-governance
LOG_PATH=$SERVER_HOME/logs/dss-data-governance/$SERVER_NAME
###install dss-data-assets-server
SERVER_NAME=dss-data-assets-server
SERVER_IP=$DSS_DATA_ASSETS_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATA_ASSETS_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-assets-server/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-data-assets-server
LIB_PATH=$SERVER_HOME/lib/atlas-application.properties
LOG_PATH=$SERVER_HOME/logs/dss-data-assets-server/$SERVER_NAME
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
installPackage

###install dss-data-warehouse-server
SERVER_NAME=dss-data-warehouse-server
SERVER_IP=$DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATA_WAREHOUSE_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-warehouse-server/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-data-warehouse-server
LOG_PATH=$SERVER_HOME/logs/dss-data-warehouse-server/$SERVER_NAME
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
installPackage

###install dss-datamodel-center-server
SERVER_NAME=dss-datamodel-center-server
SERVER_IP=$DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP
SERVER_PORT=$DSS_DATAMODEL_CENTER_SERVER_PORT
UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-datamodel-center-server/$SERVER_NAME
LIB_PATH=$SERVER_HOME/lib/dss-datamodel-center-server
LOG_PATH=$SERVER_HOME/logs/dss-datamodel-center-server/$SERVER_NAME
CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
Expand Down
15 changes: 10 additions & 5 deletions assembly/config/config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,19 @@ DSS_FLOW_EXECUTION_SERVER_PORT=9006
###dss-scriptis-server
DSS_SCRIPTIS_SERVER_INSTALL_IP=127.0.0.1
DSS_SCRIPTIS_SERVER_PORT=9008
###dss-data-assets-server
DSS_DATA_ASSETS_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_ASSETS_SERVER_PORT=9300
###dss-datamodel-center-server
DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP=127.0.0.1
DSS_DATAMODEL_CENTER_SERVER_PORT=9400
###dss-data-warehouse-server
DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_WAREHOUSE_SERVER_PORT=9500

###dss-data-api-server
DSS_DATA_API_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_API_SERVER_PORT=9208
###dss-data-governance-server
DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=127.0.0.1
DSS_DATA_GOVERNANCE_SERVER_PORT=9209
###dss-guide-server
DSS_GUIDE_SERVER_INSTALL_IP=127.0.0.1
DSS_GUIDE_SERVER_PORT=9210
Expand All @@ -83,5 +89,4 @@ EMAIL_PORT=25
[email protected]
EMAIL_PASSWORD=xxxxx
EMAIL_PROTOCOL=smtp
############## ############## dss_appconn_instance configuration end ############## ##############

############## ############## dss_appconn_instance configuration end ############## ##############
2 changes: 1 addition & 1 deletion assembly/dss-package/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
<parent>
<artifactId>dss</artifactId>
<groupId>com.webank.wedatasphere.dss</groupId>
<version>1.1.0</version>
<version>1.2.0</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>dss-package</artifactId>
Expand Down
30 changes: 25 additions & 5 deletions assembly/dss-package/src/main/assembly/distribution.xml
Original file line number Diff line number Diff line change
Expand Up @@ -287,20 +287,40 @@
<!--dss-data-api-server-->
<fileSet>
<directory>
${basedir}/../../dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/
${basedir}/../../dss-apps/dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/
</directory>
<outputDirectory>lib/dss-data-api/dss-data-api-server</outputDirectory>
<outputDirectory>lib/dss-apps/dss-data-api-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>


<fileSet>
<directory>
${basedir}/../../dss-apps/dss-dataasset-management/dss-data-assets-server/target/out/dss-data-assets-server/lib/
</directory>
<outputDirectory>lib/dss-apps/dss-data-assets-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>

<fileSet>
<directory>
${basedir}/../../dss-apps/dss-datamodel-center/dss-datamodel-center-server/target/out/dss-datamodel-center-server/lib/
</directory>
<outputDirectory>lib/dss-apps/dss-datamodel-center-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
</fileSet>

<!--dss-data-governance-server-->
<fileSet>
<directory>
${basedir}/../../dss-data-governance/dss-data-governance-server/target/out/dss-data-governance-server/lib/
${basedir}/../../dss-apps/dss-datawarehouse-design/dss-data-warehouse-server/target/out/dss-data-warehouse-server/lib/
</directory>
<outputDirectory>lib/dss-data-governance/dss-data-governance-server</outputDirectory>
<outputDirectory>lib/dss-apps/dss-data-warehouse-server</outputDirectory>
<includes>
<include>**/*</include>
</includes>
Expand Down
2 changes: 1 addition & 1 deletion assembly/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
<parent>
<artifactId>dss</artifactId>
<groupId>com.webank.wedatasphere.dss</groupId>
<version>1.1.0</version>
<version>1.2.0</version>
</parent>
<packaging>pom</packaging>
<modelVersion>4.0.0</modelVersion>
Expand Down
67 changes: 67 additions & 0 deletions conf/atlas-application.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# Generated by Apache Ambari. Sun Feb 27 10:44:18 2022

atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
atlas.audit.hbase.zookeeper.quorum=hdp
atlas.audit.zookeeper.session.timeout.ms=60000
atlas.authentication.keytab=/etc/security/keytabs/atlas.service.keytab
atlas.authentication.method.file=true
atlas.authentication.method.file.filename=/home/hadoop/software/apache-atlas-2.0.0/conf/users-credentials.properties
atlas.authentication.method.kerberos=false
atlas.authentication.method.ldap=false
atlas.authentication.method.ldap.ad.base.dn=
atlas.authentication.method.ldap.ad.bind.dn=
atlas.authentication.method.ldap.ad.bind.password=
atlas.authentication.method.ldap.ad.default.role=ROLE_USER
atlas.authentication.method.ldap.ad.domain=
atlas.authentication.method.ldap.ad.referral=ignore
atlas.authentication.method.ldap.ad.url=
atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0})
atlas.authentication.method.ldap.base.dn=
atlas.authentication.method.ldap.bind.dn=
atlas.authentication.method.ldap.bind.password=
atlas.authentication.method.ldap.default.role=ROLE_USER
atlas.authentication.method.ldap.groupRoleAttribute=cn
atlas.authentication.method.ldap.groupSearchBase=
atlas.authentication.method.ldap.groupSearchFilter=
atlas.authentication.method.ldap.referral=ignore
atlas.authentication.method.ldap.type=ldap
atlas.authentication.method.ldap.url=
atlas.authentication.method.ldap.user.searchfilter=
atlas.authentication.method.ldap.userDNpattern=uid=
atlas.authentication.principal=atlas
atlas.authorizer.impl=ranger
atlas.cluster.name=bigdata_all_in_one
atlas.enableTLS=false
atlas.graph.index.search.solr.mode=cloud
atlas.graph.index.search.solr.wait-searcher=true
atlas.graph.index.search.solr.zookeeper-url=hadoop:2181/infra-solr
atlas.graph.storage.hbase.table=atlas_janus
atlas.graph.storage.hostname=hdp
atlas.kafka.auto.commit.enable=false
atlas.kafka.bootstrap.servers=hadoop:6667
atlas.kafka.hook.group.id=atlas
atlas.kafka.zookeeper.connect=hadoop:2181
atlas.kafka.zookeeper.connection.timeout.ms=30000
atlas.kafka.zookeeper.session.timeout.ms=60000
atlas.kafka.zookeeper.sync.time.ms=20
atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
atlas.lineage.schema.query.Table=Table where __guid='%s'\, columns
atlas.notification.create.topics=true
atlas.notification.embedded=false
atlas.notification.replicas=1
atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
atlas.proxyusers=
atlas.rest.address=http://hadoop:21000
atlas.server.address.id1=hadoop:21000
atlas.server.bind.address=0.0.0.0
atlas.server.ha.enabled=false
atlas.server.http.port=21000
atlas.server.https.port=21443
atlas.server.ids=id1
atlas.simple.authz.policy.file=/home/hadoop/software/apache-atlas-2.0.0/conf/atlas-simple-authz-policy.json
atlas.solr.kerberos.enable=false
atlas.ssl.exclude.protocols=TLSv1.2
atlas.sso.knox.browser.useragent=
atlas.sso.knox.enabled=false
atlas.sso.knox.providerurl=
atlas.sso.knox.publicKey=
88 changes: 88 additions & 0 deletions conf/dss-data-assets-server.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#
# /*
# * Copyright 2019 WeBank
# *
# * Licensed under the Apache License, Version 2.0 (the "License");
# * you may not use this file except in compliance with the License.
# * You may obtain a copy of the License at
# *
# * http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software
# * distributed under the License is distributed on an "AS IS" BASIS,
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# * See the License for the specific language governing permissions and
# * limitations under the License.
# */
#
# Spring configurations
spring.server.port=9300
spring.spring.application.name=dss-data-assets-server
wds.linkis.test.mode=true
wds.linkis.log.clear=true
wds.linkis.server.version=v1


wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/hive?useSSL=false&characterEncoding=UTF-8

wds.linkis.server.mybatis.datasource.username=root

wds.linkis.server.mybatis.datasource.password=root





wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.governance.restful

##mybatis
wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/governance/dao/impl/*.xml

wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.governance.entity

wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.governance.dao


# atlas config
atlas.rest.address=http://localhost:21000
atlas.username=admin
atlas.password=admin123
atlas.client.readTimeoutMSecs=60000
atlas.client.connectTimeoutMSecs=60000

# create root classification
atlas.root.indicator=datamodel_indicator
atlas.root.measure=datamodel_measure
atlas.root.dimension=datamodel_dimension
atlas.root.layer=datamodel_layer
atlas.root.theme=datamodel_theme

# create root glossary
atlas.root.label=datamodel_label
atlas.root.collection=datamodel_collection


# hive metadata config
metastore.datasource.driver=com.mysql.jdbc.Driver
metastore.datasource.url=jdbc:mysql://localhost:3306/hive?characterEncoding=UTF-8
metastore.datasource.username=root
metastore.datasource.password=root


wds.workspace.client.serverurl=http://localhost:9001

wds.workspace.client.authtoken.key=hadoop

wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy

wds.workspace.client.discovery.enabled=false

wds.workspace.client.authtoken.value=hadoop

spring.spring.mvc.servlet.path=/api/rest_j/v1

wds.workspace.client.dws.version=v1

wds.wedatasphere.data.assert.client.strategy=static


53 changes: 0 additions & 53 deletions conf/dss-data-governance-server.properties

This file was deleted.

Loading

0 comments on commit 8469876

Please sign in to comment.