From 0075e934e4296861ecb805253c96e13389031701 Mon Sep 17 00:00:00 2001 From: jr-marquez Date: Mon, 22 Feb 2021 19:33:47 +0100 Subject: [PATCH] add oracle cdc agregamos cdc de oracle --- .gitignore | 4 ++ docker/docker-compose.yml | 11 +++- docker/scripts/go_sqlplus.sh | 16 ++++++ docker/scripts/oracle_setup_aws_rds.sql | 37 +++++++++++++ docker/scripts/oracle_setup_docker.sql | 69 +++++++++++++++++++++++++ terraform/aws/env-vars.sample | 2 + terraform/aws/outputs.tf | 15 ++++-- terraform/aws/userdata.tf | 4 +- terraform/aws/variables.tf | 7 ++- terraform/utils/instance.sh | 6 ++- 10 files changed, 162 insertions(+), 9 deletions(-) create mode 100755 docker/scripts/go_sqlplus.sh create mode 100755 docker/scripts/oracle_setup_aws_rds.sql create mode 100755 docker/scripts/oracle_setup_docker.sql diff --git a/.gitignore b/.gitignore index e7bcaed..1b7d3c4 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,7 @@ Icon Network Trash Folder Temporary Items .apdisk +terraform/aws/.terraform/plugins/darwin_amd64/terraform-provider-aws_v3.28.0_x5 +terraform/aws/.terraform/plugins/darwin_amd64/terraform-provider-template_v2.2.0_x4 +terraform/aws/terraform.tfstate +terraform/aws/terraform.tfstate.backup diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 9785ee6..80a81c3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -202,6 +202,7 @@ services: # Other systems + mysql: # *-----------------------------* # To connect to the DB: @@ -258,8 +259,6 @@ services: CONTROL_CENTER_REST_LISTENERS: "http://0.0.0.0:9021" PORT: 9021 -# CONTROL_CENTER_STREAMS_CONSUMER_REQUEST_TIMEOUT_MS: "960032" - mongo: image: mongo:4.2.5 hostname: mongo @@ -295,3 +294,11 @@ services: - "9300:9300" environment: discovery.type: single-node + oracle: + image: store/oracle/database-enterprise:12.2.0.1 + hostname: oracle + volumes: + - ./scripts:/scripts + container_name: oracle + ports: + - 1521:1521 diff --git a/docker/scripts/go_sqlplus.sh b/docker/scripts/go_sqlplus.sh new file mode 100755 index 0000000..4e6ee53 --- /dev/null +++ b/docker/scripts/go_sqlplus.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +export ORACLE_HOME=/u01/app/oracle/product/12.2.0/dbhome_1 +export ORACLE_SID=ORCLCDB +export PATH=${PATH}:${ORACLE_HOME}/bin +export TNS_ADMIN=${ORACLE_HOME}/admin/ORCLCDB + +SQL_FILE=${1} + +if [ -z "$SQL_FILE" ]; then + sqlplus '/ as sysdba' +else + sqlplus '/ as sysdba' @${SQL_FILE} +fi + + diff --git a/docker/scripts/oracle_setup_aws_rds.sql b/docker/scripts/oracle_setup_aws_rds.sql new file mode 100755 index 0000000..8d71498 --- /dev/null +++ b/docker/scripts/oracle_setup_aws_rds.sql @@ -0,0 +1,37 @@ +-- Amazon RDS +CREATE USER myuser IDENTIFIED BY PASSWORD DEFAULT TABLESPACE USERS; +GRANT CREATE SESSION, SELECT ANY TRANSACTION, SELECT ANY DICTIONARY TO myuser; +GRANT SELECT ON SYSTEM.LOGMNR_COL$ TO myuser; +GRANT SELECT ON SYSTEM.LOGMNR_OBJ$ TO myuser; +GRANT SELECT ON SYSTEM.LOGMNR_USER$ TO myuser; +GRANT SELECT ON SYSTEM.LOGMNR_UID$ TO myuser; +ALTER USER myuser QUOTA UNLIMITED ON USERS; + +GRANT CREATE SESSION TO myuser; +GRANT CREATE TABLE TO myuser; +GRANT CREATE SEQUENCE TO myuser; +GRANT CREATE TRIGGER TO myuser; +GRANT LOGMINING TO myuser; + +--/ +begin + rdsadmin.rdsadmin_util.grant_sys_object('DBMS_LOGMNR', 'MYUSER', 'EXECUTE'); +end; +/ + + +--/ +begin + rdsadmin.rdsadmin_util.alter_supplemental_logging('ADD'); + rdsadmin.rdsadmin_util.alter_supplemental_logging('ADD','ALL'); +end; +/ + + + + +--ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +GRANT FLASHBACK ANY TABLE TO myuser; + + diff --git a/docker/scripts/oracle_setup_docker.sql b/docker/scripts/oracle_setup_docker.sql new file mode 100755 index 0000000..f287ca1 --- /dev/null +++ b/docker/scripts/oracle_setup_docker.sql @@ -0,0 +1,69 @@ +prompt Starting Setup + +prompt Check if the ORACLE database is in archive log mode +select log_mode from v$database; + +prompt Turn on ARCHIVELOG mode +SHUTDOWN IMMEDIATE; +STARTUP MOUNT; +ALTER DATABASE ARCHIVELOG; +ALTER DATABASE OPEN; + +prompt Check if the ORACLE database is in archive log mode +select log_mode from v$database; + +prompt Enable supplemental logging for all columns +ALTER SESSION SET CONTAINER=cdb$root; +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +-- to be run in the CDB +-- credit : https://docs.confluent.io/kafka-connect-oracle-cdc/current +CREATE ROLE C##CDC_PRIVS; +GRANT CREATE SESSION, +EXECUTE_CATALOG_ROLE, +SELECT ANY TRANSACTION, +SELECT ANY DICTIONARY TO C##CDC_PRIVS; +GRANT SELECT ON SYSTEM.LOGMNR_COL$ TO C##CDC_PRIVS; +GRANT SELECT ON SYSTEM.LOGMNR_OBJ$ TO C##CDC_PRIVS; +GRANT SELECT ON SYSTEM.LOGMNR_USER$ TO C##CDC_PRIVS; +GRANT SELECT ON SYSTEM.LOGMNR_UID$ TO C##CDC_PRIVS; + +CREATE USER C##myuser IDENTIFIED BY password CONTAINER=ALL; +GRANT C##CDC_PRIVS TO C##myuser CONTAINER=ALL; +ALTER USER C##myuser QUOTA UNLIMITED ON sysaux; +ALTER USER C##myuser SET CONTAINER_DATA = (CDB$ROOT, ORCLPDB1) CONTAINER=CURRENT; + +ALTER SESSION SET CONTAINER=CDB$ROOT; +GRANT CREATE SESSION, ALTER SESSION, SET CONTAINER, LOGMINING, EXECUTE_CATALOG_ROLE TO C##myuser CONTAINER=ALL; +GRANT SELECT ON GV_$DATABASE TO C##myuser CONTAINER=ALL; +GRANT SELECT ON V_$LOGMNR_CONTENTS TO C##myuser CONTAINER=ALL; +GRANT SELECT ON GV_$ARCHIVED_LOG TO C##myuser CONTAINER=ALL; +GRANT CONNECT TO C##myuser CONTAINER=ALL; +GRANT CREATE TABLE TO C##myuser CONTAINER=ALL; +GRANT CREATE SEQUENCE TO C##myuser CONTAINER=ALL; +GRANT CREATE TRIGGER TO C##myuser CONTAINER=ALL; + +ALTER SESSION SET CONTAINER=cdb$root; +ALTER DATABASE ADD SUPPLEMENTAL LOG DATA (ALL) COLUMNS; + +GRANT FLASHBACK ANY TABLE TO C##myuser; +GRANT FLASHBACK ANY TABLE TO C##myuser container=all; + + +prompt Create some objects +CREATE TABLE C##MYUSER.emp +( + i INTEGER GENERATED BY DEFAULT AS IDENTITY, + name VARCHAR2(100), + PRIMARY KEY (i) +) tablespace sysaux; + +insert into C##MYUSER.emp (name) values ('Bob'); +insert into C##MYUSER.emp (name) values ('Jane'); +insert into C##MYUSER.emp (name) values ('Mary'); +insert into C##MYUSER.emp (name) values ('Alice'); + +prompt All Done + +exit; + diff --git a/terraform/aws/env-vars.sample b/terraform/aws/env-vars.sample index 744ad88..dfeff3a 100644 --- a/terraform/aws/env-vars.sample +++ b/terraform/aws/env-vars.sample @@ -3,3 +3,5 @@ export TF_VAR_aws_secret_key= export TF_VAR_aws_region= export TF_VAR_ssh_key_name= export TF_VAR_instance_count= +export TF_VAR_docker_login= +export TF_VAR_docker_password= diff --git a/terraform/aws/outputs.tf b/terraform/aws/outputs.tf index fc5dceb..e1d1f50 100644 --- a/terraform/aws/outputs.tf +++ b/terraform/aws/outputs.tf @@ -3,12 +3,21 @@ ########################################### output "PublicIPs" { - value = tonumber(var.instance_count) >= 1 ? " Public IP Adresses are ${join(",",formatlist("%s", aws_instance.ksqldb-demo.*.public_ip),)} " : "Confluent Cloud Platform on AWS is disabled" + value ="${formatlist( + "Public IP = %s", + aws_instance.ksqldb-demo.*.public_ip + )}" } output "SSH" { - value = tonumber(var.instance_count) >= 1 ? " SSH Access: ssh -i ~/keys/jrMarquez_terraform.pem ec2-user@${join(",",formatlist("%s", aws_instance.ksqldb-demo.*.public_ip),)} " : "Confluent Cloud Platform on AWS is disabled" + value = "${formatlist( + "SSH Access: ssh -i ~/keys/jrMarquez_terraform.pem ec2-user@%s", + aws_instance.ksqldb-demo.*.public_ip + )}" } output "C3" { - value = tonumber(var.instance_count) >= 1 ? " Control Center: http://${join(",",formatlist("%s", aws_instance.ksqldb-demo.*.public_ip),)}:9021" : "Confluent Cloud Platform on AWS is disabled" + value = "${formatlist( + "Control Center: http://%s:9021", + aws_instance.ksqldb-demo.*.public_ip + )}" } diff --git a/terraform/aws/userdata.tf b/terraform/aws/userdata.tf index c9b432d..ebfef4f 100644 --- a/terraform/aws/userdata.tf +++ b/terraform/aws/userdata.tf @@ -6,6 +6,8 @@ data "template_file" "confluent_instance" { template = file("../utils/instance.sh") vars = { - confluent_ksqldb_demo = var.confluentksqldbdemo + confluent_ksqldb_demo = var.confluentksqldbdemo, + docker_login=var.docker_login, + docker_password=var.docker_password } } diff --git a/terraform/aws/variables.tf b/terraform/aws/variables.tf index 6d8694f..8edea18 100644 --- a/terraform/aws/variables.tf +++ b/terraform/aws/variables.tf @@ -20,7 +20,10 @@ variable "instance_type_resource" { variable "confluentksqldbdemo" { default = "https://github.com/jr-marquez/ksqldbWorkshop/archive/main.zip" } - +variable "docker_login" { +} +variable "docker_password" { +} variable "instance_count" { - default = "1" + default = "2" } diff --git a/terraform/utils/instance.sh b/terraform/utils/instance.sh index 28320f0..1e2b1e8 100644 --- a/terraform/utils/instance.sh +++ b/terraform/utils/instance.sh @@ -43,4 +43,8 @@ SCRIPT2="sed -i 's/CONTROL_CENTER_KSQL_WORKSHOP_ADVERTISED_URL: http:\/\/localho bash -c "$SCRIPT1" bash -c "$SCRIPT2" #bash -c "$SCRIPT4" - +chmod 666 /var/run/docker.sock +docker login -u ${docker_login} -p ${docker_password} +docker-compose up -d +sleep 120 +docker-compose exec oracle /scripts/go_sqlplus.sh /scripts/oracle_setup_docker \ No newline at end of file