From e426b988ebf2a18dbffd58149195475a73f8f702 Mon Sep 17 00:00:00 2001 From: Amninder Kaur Date: Thu, 22 Feb 2024 09:34:24 +1100 Subject: [PATCH] chore: create image from template --- .cargo/config.toml | 2 - .env.example | 8 ++ .github/workflows/npm_publish.yml | 5 +- .github/workflows/rust_build.yml | 2 +- Cargo.lock | 161 +++++++++++++-------------- Cargo.toml | 1 - Dockerfile.scylla_pg_migration | 7 +- Makefile | 6 + bin/component-test-setup.sh | 5 +- scylla_operations/Cargo.toml | 2 +- scylla_pg_core/Cargo.toml | 9 +- scylla_pg_core/src/bin/db_migrate.rs | 35 +----- scylla_pg_core/src/connection.rs | 21 +++- scylla_pg_js/Cargo.toml | 2 +- scylla_pg_monitor/Cargo.toml | 6 +- 15 files changed, 140 insertions(+), 132 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 0a9effe..3ccea90 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -13,5 +13,3 @@ linker = "aarch64-linux-gnu-gcc" [target.aarch64-unknown-linux-musl] linker = "aarch64-linux-musl-gcc" rustflags = ["-C", "target-feature=-crt-static"] - - diff --git a/.env.example b/.env.example index 1b08996..13dc8c4 100644 --- a/.env.example +++ b/.env.example @@ -14,3 +14,11 @@ MONITOR_POLLING_INTERVAL_IN_SECS=5 MONITOR_TASK_RETENTION_PERIOD_IN_SECS=864000 RUST_LOG=ERROR + +# POSTGRES ENV FOR CREATE DB +PG_DATABASE_TEMPLATE= +PG_DATABASE_ADMIN= +SCYLLA_CREATE_DB= + +# FOR DROPPING THE DB +#SCYLLA_CLEANUP_MODE= \ No newline at end of file diff --git a/.github/workflows/npm_publish.yml b/.github/workflows/npm_publish.yml index 6aac509..52e6405 100644 --- a/.github/workflows/npm_publish.yml +++ b/.github/workflows/npm_publish.yml @@ -81,7 +81,7 @@ jobs: uses: dtolnay/rust-toolchain@stable if: ${{ !matrix.settings.docker }} with: - toolchain: stable + toolchain: stable 3 months ago target: ${{ matrix.settings.target }} - name: Cache cargo uses: actions/cache@v4 @@ -177,5 +177,4 @@ jobs: - run: ls -R ./scylla_pg_js/artifacts - name: Publish - run: PATH=$PATH:$(pwd) bin/npm-release.sh # only run on release created or tag - + run: PATH=$PATH:$(pwd) bin/npm-release.sh # only run on release created or tag \ No newline at end of file diff --git a/.github/workflows/rust_build.yml b/.github/workflows/rust_build.yml index 55baf35..8403060 100644 --- a/.github/workflows/rust_build.yml +++ b/.github/workflows/rust_build.yml @@ -10,7 +10,7 @@ jobs: - uses: actions/checkout@v3 - uses: dtolnay/rust-toolchain@1.75.0 with: - toolchain: stable + toolchain: stable 3 months ago - run: bin/ubuntu-setup.sh - run: rustup component add rustfmt clippy - name: cargo build diff --git a/Cargo.lock b/Cargo.lock index 39d6acc..82d0aad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -52,6 +52,54 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b09b5178381e0874812a9b157f7fe84982617e48f71f4e3235482775e5b540" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -85,17 +133,6 @@ dependencies = [ "syn", ] -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -198,6 +235,12 @@ dependencies = [ "windows-targets 0.52.0", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "config" version = "0.13.4" @@ -343,29 +386,26 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" [[package]] -name = "env_logger" -version = "0.9.3" +name = "env_filter" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" dependencies = [ - "atty", - "humantime", "log", "regex", - "termcolor", ] [[package]] name = "env_logger" -version = "0.10.2" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +checksum = "6c012a26a7f605efc424dd53697843a72be7dc86ad2d01f7814337794a12231d" dependencies = [ + "anstream", + "anstyle", + "env_filter", "humantime", - "is-terminal", "log", - "regex", - "termcolor", ] [[package]] @@ -383,16 +423,6 @@ dependencies = [ "serde", ] -[[package]] -name = "errno" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "fallible-iterator" version = "0.2.0" @@ -545,15 +575,6 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - [[package]] name = "hermit-abi" version = "0.3.4" @@ -618,17 +639,6 @@ dependencies = [ "hashbrown 0.14.3", ] -[[package]] -name = "is-terminal" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bad00257d07be169d870ab665980b06cdb366d792ad690bf2e76876dc503455" -dependencies = [ - "hermit-abi 0.3.4", - "rustix", - "windows-sys 0.52.0", -] - [[package]] name = "itoa" version = "1.0.10" @@ -683,12 +693,6 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" -[[package]] -name = "linux-raw-sys" -version = "0.4.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" - [[package]] name = "lock_api" version = "0.4.11" @@ -843,7 +847,7 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.4", + "hermit-abi", "libc", ] @@ -1203,19 +1207,6 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" -[[package]] -name = "rustix" -version = "0.38.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" -dependencies = [ - "bitflags 2.4.2", - "errno", - "libc", - "linux-raw-sys", - "windows-sys 0.52.0", -] - [[package]] name = "ryu" version = "1.0.16" @@ -1253,7 +1244,7 @@ version = "0.1.1" dependencies = [ "async-trait", "chrono", - "env_logger 0.9.3", + "env_logger", "log", "scylla_models", "serde_json", @@ -1266,10 +1257,13 @@ dependencies = [ name = "scylla_pg_core" version = "0.1.22" dependencies = [ + "async-trait", "chrono", "config", "deadpool-postgres", "dotenv", + "env_logger", + "log", "refinery", "serde", "tokio", @@ -1281,7 +1275,7 @@ name = "scylla_pg_js" version = "0.1.22" dependencies = [ "chrono", - "env_logger 0.9.3", + "env_logger", "log", "napi", "napi-build", @@ -1318,13 +1312,13 @@ dependencies = [ [[package]] name = "scylla_pg_monitor" -version = "0.1.22" +version = "0.1.23" dependencies = [ "async-trait", "chrono", "config", "dotenv", - "env_logger 0.10.2", + "env_logger", "log", "scylla_models", "scylla_operations", @@ -1579,15 +1573,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "thiserror" version = "1.0.56" @@ -1860,6 +1845,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "uuid" version = "1.7.0" diff --git a/Cargo.toml b/Cargo.toml index 3c8d196..bc3ce78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,4 +15,3 @@ log = { version = "0.4", features = [ "std", "kv_unstable_serde", ] } -env_logger = "0.10" \ No newline at end of file diff --git a/Dockerfile.scylla_pg_migration b/Dockerfile.scylla_pg_migration index 0515dd1..9b174ed 100644 --- a/Dockerfile.scylla_pg_migration +++ b/Dockerfile.scylla_pg_migration @@ -17,7 +17,7 @@ RUN adduser \ WORKDIR /tmp COPY ./ . # Build binary in release mode -RUN cargo build --release --bin db_migrate +RUN cargo build --release --package scylla_pg_core # # Run image based on buster-slim to reduce image size while still using glibc @@ -30,7 +30,10 @@ COPY --from=BUILD /etc/passwd /etc/passwd COPY --from=BUILD /etc/group /etc/group # Copy binary from build COPY --from=BUILD /tmp/target/release/db_migrate ./ +COPY --from=BUILD /tmp/target/release/db_create ./ +COPY --from=BUILD /tmp/target/release/db_delete ./ +COPY --from=BUILD /tmp/bin/create_db_and_migrate.sh ./ # Use an unprivileged user USER ${USER}:${USER} # Entry point -CMD ["/opt/build/db_migrate"] +CMD ["/opt/build/create_db_and_migrate.sh"] diff --git a/Makefile b/Makefile index 1c959bd..9420f56 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,12 @@ pg.monitor: db.migrate: $(call pp,db migrate...) cargo run --bin db_migrate + +## db.create: 🧪 Creates a new DB +db.create: + $(call pp,db migrate...) + cargo run --bin db_create +# cargo run --bin db_delete truncate: $(call pp, truncate...) RUSTFLAGS="" cargo run --bin truncate diff --git a/bin/component-test-setup.sh b/bin/component-test-setup.sh index ff878fa..aaf7035 100755 --- a/bin/component-test-setup.sh +++ b/bin/component-test-setup.sh @@ -4,10 +4,13 @@ echo PG_HOST=127.0.0.1 >> .env echo PG_USER=admin >> .env echo PG_PASSWORD=admin >> .env echo PG_DATABASE=scylla >> .env +echo PG_DATABASE_ADMIN=postgres >> .env echo PG_PORT=5432 >> .env echo PG_POOL_SIZE=10 >> .env -echo DB_ENV=mig-test >> .env # run db-migrations +make withenv RECIPE=db.create +# sleep +sleep 3 make withenv RECIPE=db.migrate \ No newline at end of file diff --git a/scylla_operations/Cargo.toml b/scylla_operations/Cargo.toml index c9f6258..14ef1cc 100644 --- a/scylla_operations/Cargo.toml +++ b/scylla_operations/Cargo.toml @@ -13,7 +13,7 @@ tokio = {version = "1.20", features = ["macros", "rt"] } chrono = { version = "0.4.19", features = ["serde"] } # Logging log = { version = "0.4", features = ["kv_unstable", "std", "kv_unstable_serde"] } -env_logger = "0.9" +env_logger = "0.11.2" # crate depencies scylla_models = { path = "../scylla_models" } diff --git a/scylla_pg_core/Cargo.toml b/scylla_pg_core/Cargo.toml index bb4ba00..f36aadd 100644 --- a/scylla_pg_core/Cargo.toml +++ b/scylla_pg_core/Cargo.toml @@ -8,7 +8,6 @@ edition = "2021" [dependencies] -tokio="1.20" chrono = "0.4" dotenv = "0.15" config = "0.13" @@ -18,3 +17,11 @@ deadpool-postgres = { version = "0.10", features = ["serde"] } # db migrations refinery = {version="0.8", features=["tokio-postgres"]} + +# Logging +log = "0.4" +env_logger = "0.11.2" + +# tokio and async +async-trait = "0.1" +tokio = { version = "1", features = ["full"] } diff --git a/scylla_pg_core/src/bin/db_migrate.rs b/scylla_pg_core/src/bin/db_migrate.rs index 7bb51cf..6d8d15a 100644 --- a/scylla_pg_core/src/bin/db_migrate.rs +++ b/scylla_pg_core/src/bin/db_migrate.rs @@ -6,7 +6,6 @@ mod connection; use crate::config::PGConfig; use connection::get_client; -use tokio_postgres::{error::SqlState, Client}; mod embedded { use refinery::embed_migrations; @@ -15,13 +14,9 @@ mod embedded { #[tokio::main] async fn main() { dotenv::dotenv().ok(); + log::info!("Migrations"); let conf = PGConfig::from_env().unwrap(); - let is_env = std::env::var("DB_ENV").unwrap_or_default(); - if is_env == "mig-test" { - try_create_db(&conf).await.expect("could not create database"); - } else { - println!("skipping database creation for non test environment"); - } + log::info!("running migrations for {}", conf.pg_database); run_migrations(&conf).await; } @@ -29,36 +24,16 @@ async fn run_migrations(conf: &PGConfig) { let pg_config = conf.to_pg_config(); match get_client(&pg_config).await { Ok(mut client) => { - println!("running migrations"); + log::info!("running migrations {} by {} ", conf.pg_database, conf.pg_user); let migration_result = embedded::migrations::runner().run_async(&mut client).await; match migration_result { Ok(t) => println!("{t:?}"), Err(e) => panic!("{e}"), } } - Err(e) => panic!("{}", e), - } -} - -async fn try_create_db(conf: &PGConfig) -> Result<(), tokio_postgres::Error> { - let pg_config = conf.to_without_db_config(); - let client = get_client(&pg_config).await?; - create_db(&client, conf.pg_database.as_str()).await -} - -/// Create database -async fn create_db(client: &Client, database: &str) -> Result<(), tokio_postgres::Error> { - let create_db_ddl = format!("CREATE DATABASE \"{database}\""); - let db_exec_result = client.execute(&create_db_ddl, &[]).await; - match db_exec_result { - Ok(_) => Ok(()), Err(e) => { - if e.code() == Some(&SqlState::DUPLICATE_DATABASE) { - eprintln!("database already exists. proceeding with migrations."); - Ok(()) - } else { - Err(e) - } + log::error!("error connecting to database: {} by user {}", conf.pg_database, conf.pg_user); + panic!("{}", e) } } } diff --git a/scylla_pg_core/src/connection.rs b/scylla_pg_core/src/connection.rs index 9d730f9..fd20b55 100644 --- a/scylla_pg_core/src/connection.rs +++ b/scylla_pg_core/src/connection.rs @@ -29,7 +29,26 @@ pub async fn get_client(config: &Config) -> Result Result { + let (client, connection) = Config::new() + .host(&config.pg_host) + .port(config.pg_port) + .user(&config.pg_user) + .password(&config.pg_password) + .dbname(&config.pg_database) + .connect(tokio_postgres::NoTls) + .await?; + + // let (client, connection) = custom_config.connect(tokio_postgres::NoTls).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + log::error!("connection error: {e}"); } }); Ok(client) diff --git a/scylla_pg_js/Cargo.toml b/scylla_pg_js/Cargo.toml index 1a08951..3eddd08 100644 --- a/scylla_pg_js/Cargo.toml +++ b/scylla_pg_js/Cargo.toml @@ -27,7 +27,7 @@ log = { version = "0.4", features = [ "std", "kv_unstable_serde", ] } -env_logger = "0.9" +env_logger = "0.11.2" # db migrations #refinery = {version="0.8", features=["tokio-postgres"]} # crate depencies diff --git a/scylla_pg_monitor/Cargo.toml b/scylla_pg_monitor/Cargo.toml index dac77e4..8231a1e 100644 --- a/scylla_pg_monitor/Cargo.toml +++ b/scylla_pg_monitor/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "scylla_pg_monitor" -version = "0.1.22" +version = "0.1.23" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -24,8 +24,8 @@ tokio-postgres = { version = "0.7", features = ["with-serde_json-1", "with-uuid- scylla_models = { path = "../scylla_models" } scylla_pg_lib = { path = "../scylla_pg_lib" } scylla_pg_core = { path = "../scylla_pg_core" } -env_logger = {workspace = true} -log = {workspace = true} +env_logger = "0.11.2" +log = "0.4" [dev-dependencies] scylla_operations = {path = "../scylla_operations"}