diff --git a/.config b/.config index 476e7bd..377642a 100644 Binary files a/.config and b/.config differ diff --git a/run b/run new file mode 100755 index 0000000..7d5a86f --- /dev/null +++ b/run @@ -0,0 +1,271 @@ +#!/bin/zsh +SCWRYPTS_ROOT="${0:a:h}" +source "$SCWRYPTS_ROOT/zsh/common.zsh" || exit 42 +##################################################################### + +__RUN() { + local USAGE=' +Usage : scwrypts [OPTIONS ...] SCRIPT -- [SCRIPT OPTIONS ...] + +OPTIONS + -e, --env set environment; overwrites SCWRYPTS_ENV + -n, --no-log skip logging (useful when calling scwrypts as an api) + -l, --list print out command list and exit + + -h, --help display this message and exit +' + + cd "$SCWRYPTS_ROOT" + + local ENV_NAME="$SCWRYPTS_ENV" + local SEARCH_PATTERNS=() + + local ARGS_ERROR=0 + + while [[ $# -gt 0 ]] + do + case $1 in + -h | --help ) + echo $USAGE + return 0 + ;; + -n | --no-log ) + [ ! $SUBSCWRYPT ] && SUBSCWRYPT=0 + shift 1 + ;; + -e | --env ) + [ $ENV_NAME ] && __WARNING 'overwriting session environment' + ENV_NAME="$2" + __STATUS "using CLI environment '$ENV_NAME'" + shift 2 + ;; + -l | --list ) + __OUTPUT_COMMAND_LIST + return 0 + ;; + -- ) + shift 1 + break # pass arguments after '--' to the scwrypt + ;; + -* ) + __ERROR "unrecognized argument '$1'" + ((ARGS_ERROR+=1)) + shift 1 + ;; + * ) + SEARCH_PATTERNS+=$1 + shift 1 + ;; + esac + done + + [[ $ARGS_ERROR -gt 0 ]] && { + echo $USAGE + return 1 + } + + ########################################## + + local SCRIPT=$(__SELECT_SCRIPT $SEARCH_PATTERNS) + [ ! $SCRIPT ] && exit 2 + + local ENV_REQUIRED=$(__CHECK_ENV_REQUIRED && echo 1 || echo 0) + + [[ $ENV_REQUIRED -eq 1 ]] && { + [ ! $ENV_NAME ] && ENV_NAME=$(__SELECT_ENV) + local ENV_FILE=$(__GET_ENV_FILE $ENV_NAME) + + [ -f "$ENV_FILE" ] && source "$ENV_FILE" \ + || __FAIL 5 "missing or invalid environment '$ENV_NAME'" + + export ENV_NAME + } + + [ ! $SUBSCWRYPT ] \ + && [[ $ENV_NAME =~ prod ]] \ + && { __VALIDATE_UPSTREAM_TIMELINE || __ABORT; } + + local RUN_STRING=$(__GET_RUN_STRING $SCRIPT $ENV_NAME) + [ ! $RUN_STRING ] && exit 3 + + ########################################## + + local LOGFILE=$(__GET_LOGFILE $SCRIPT) + + local HEADER=$( + [ $SUBSCWRYPT ] && return 0 + echo '=====================================================================' + echo "script : $SCRIPT" + echo "run at : $(date)" + echo "config : $ENV_NAME" + [ ! $LOGFILE ] && echo '\033[1;33m------------------------------------------\033[0m' + ) + + [ ! $LOGFILE ] && { + [ $HEADER ] && echo $HEADER + [[ $SUBSCWRYPT -eq 0 ]] && { + eval $RUN_STRING $@ + exit $? + } || { + eval $RUN_STRING $@ /dev/tty 2>&1 + exit $? + } + } + + { + [ $HEADER ] && echo $HEADER + echo '\033[1;33m--- BEGIN OUTPUT -------------------------\033[0m' + eval $RUN_STRING $@ + EXIT_CODE=$? + echo '\033[1;33m--- END OUTPUT ---------------------------\033[0m' + + [[ $EXIT_CODE -eq 0 ]] && EXIT_COLOR='32m' || EXIT_COLOR='31m' + + echo "terminated with\\033[1;$EXIT_COLOR code $EXIT_CODE\\033[0m" + } 2>&1 | tee --append "$LOGFILE" + + exit $(\ + sed -n 's/^terminated with.*code \([0-9]*\).*$/\1/p' $LOGFILE \ + | tail -n1 + ) +} + +##################################################################### + +__OUTPUT_COMMAND_LIST() { + local LAST_TYPE LAST_SUBSET + for SCRIPT in $(__GET_AVAILABLE_SCRIPTS) + do + TYPE=$(echo $SCRIPT | sed 's/\/.*//') + SUBSET=$(echo $SCRIPT | sed 's/.*\/\(.*\)\/[^\/]*$/\1/') + [[ ! $LAST_TYPE =~ $TYPE ]] && { + echo >&2 + echo "\\033[1;32m$TYPE scwrypts\\033[0m" >&2 + LAST_SUBSET='' + } + [ $LAST_SUBSET ] && [[ ! $LAST_SUBSET =~ $SUBSET ]] && { + echo >&2 + } + printf ' - ' >&2 + echo $SCRIPT + LAST_TYPE=$TYPE + LAST_SUBSET=$SUBSET + done +} + +##################################################################### + +__SELECT_SCRIPT() { + local SCRIPT + local SCRIPTS=$(__GET_AVAILABLE_SCRIPTS) + local SEARCH=($@) + + [[ ${#SEARCH[@]} -eq 0 ]] && { + SCRIPT=$(echo $SCRIPTS | __FZF 'select a script') + } + + [[ ${#SEARCH[@]} -eq 1 ]] && [ -f ./$SEARCH ] && { + SCRIPT=$SEARCH + } + + [ ! $SCRIPT ] && [[ ${#SEARCH[@]} -gt 0 ]] && { + SCRIPT=$SCRIPTS + for PATTERN in $SEARCH + do + SCRIPT=$(echo $SCRIPT | grep $PATTERN) + done + + [ ! $SCRIPT ] && __FAIL 2 "no script found by name '$@'" + + [[ $(echo $SCRIPT | wc -l) -gt 1 ]] && { + __STATUS "more than one script matched '$@'" + SCRIPT=$(echo $SCRIPT | __FZF 'select a script') + } + } + + echo $SCRIPT +} + +__GET_RUN_STRING() { + local SCRIPT="$1" + local ENV_NAME="$2" + local TYPE=$(echo $SCRIPT | sed 's/\/.*$//') + + local RUN_STRING + + local _VIRTUALENV="$SCWRYPTS_VIRTUALENV_PATH/$TYPE/bin/activate" + [ -f $_VIRTUALENV ] && source $_VIRTUALENV + + case $TYPE in + py ) __CHECK_DEPENDENCY python || return 1 + RUN_STRING="python -m $(echo $SCRIPT | sed 's/\//./g; s/\.py$//; s/\.\.//')" + + CURRENT_PYTHON_VERSION=$(python --version | sed 's/^[^0-9]*\(3\.[^.]*\).*$/\1/') + + echo $__PREFERRED_PYTHON_VERSIONS | grep -q $CURRENT_PYTHON_VERSION || { + __WARNING "only tested on the following python versions: $(printf ', %s.x' ${__PREFERRED_PYTHON_VERSIONS[@]} | sed 's/^, //')" + __WARNING 'compatibility may vary' + } + ;; + + zsh ) __CHECK_DEPENDENCY zsh || return 1 + RUN_STRING="noglob ./$SCRIPT" + ;; + + zx ) __CHECK_DEPENDENCY zx || return 1 + RUN_STRING="FORCE_COLOR=3 ./$SCRIPT.mjs" + ;; + + * ) __ERROR "unsupported script type '$SCRIPT_TYPE'" + return 2 + ;; + esac + + RUN_STRING="SCWRYPTS_ENV='$ENV_NAME' $RUN_STRING" + [ -f $_VIRTUALENV ] && RUN_STRING="source '$_VIRTUALENV'; $RUN_STRING" + + echo $RUN_STRING +} + +__CHECK_ENV_REQUIRED() { + [ $CI ] && return 1 + + echo $SCRIPT | grep -q 'zsh/scwrypts/logs' && return 1 + + return 0 +} + +__VALIDATE_UPSTREAM_TIMELINE() { + __STATUS "on '$ENV_NAME'; checking diff against origin/main" + + git fetch --quiet origin main + local SYNC_STATUS=$? + + git diff --exit-code origin/main -- . >&2 + local DIFF_STATUS=$? + + [[ $SYNC_STATUS -eq 0 ]] && [[ $DIFF_STATUS -eq 0 ]] && { + __SUCCESS 'up-to-date with origin/main' + } || { + __WARNING + [[ $SYNC_STATUS -ne 0 ]] && __WARNING 'unable to synchronize with origin/main' + [[ $DIFF_STATUS -ne 0 ]] && __WARNING 'your branch differs from origin/main (diff listed above)' + __WARNING + + __yN 'continue?' || return 1 + } +} + +__GET_LOGFILE() { + local SCRIPT="$1" + + [ $SUBSCWRYPT ] \ + || [[ $SCRIPT =~ scwrypts/logs ]] \ + || [[ $SCRIPT =~ interactive ]] \ + && return 0 + + echo "$SCWRYPTS_LOG_PATH/$(echo $SCRIPT | sed 's/^\.\///; s/\//\%/g').log" +} + +##################################################################### +__RUN $@ diff --git a/scwrypts b/scwrypts index 8be6ae1..126270a 100755 --- a/scwrypts +++ b/scwrypts @@ -1,208 +1,2 @@ #!/bin/zsh -SCWRYPTS_ROOT="${0:a:h}" -source "$SCWRYPTS_ROOT/zsh/common.zsh" || exit 42 -##################################################################### - -__RUN() { - cd "$SCWRYPTS_ROOT" - - local ENV_NAME="$SCWRYPTS_ENV" - local SEARCH_PATTERNS=() - - while [[ $# -gt 0 ]] - do - case $1 in - -e | --env ) - [ $ENV_NAME ] && __WARNING 'overwriting session environment' - ENV_NAME="$2" - __STATUS "using CLI environment '$ENV_NAME'" - shift 2 - ;; - -- ) - shift 1 - break # pass arguments after '--' to the scwrypt - ;; - * ) - SEARCH_PATTERNS+=$1 - shift 1 - ;; - esac - done - - ########################################## - - local SCRIPT=$(__SELECT_SCRIPT $SEARCH_PATTERNS) - [ ! $SCRIPT ] && exit 2 - - local ENV_REQUIRED=$(__CHECK_ENV_REQUIRED && echo 1 || echo 0) - - [[ $ENV_REQUIRED -eq 1 ]] && { - [ ! $ENV_NAME ] && ENV_NAME=$(__SELECT_ENV) - local ENV_FILE=$(__GET_ENV_FILE $ENV_NAME) - - [ -f "$ENV_FILE" ] && source "$ENV_FILE" \ - || __FAIL 5 "missing or invalid environment '$ENV_NAME'" - - export ENV_NAME - } - - [ ! $SUBSCWRYPT ] \ - && [[ $ENV_NAME =~ prod ]] \ - && { __VALIDATE_UPSTREAM_TIMELINE || __ABORT; } - - local RUN_STRING=$(__GET_RUN_STRING $SCRIPT $ENV_NAME) - [ ! $RUN_STRING ] && exit 3 - - ########################################## - - local LOGFILE=$(__GET_LOGFILE $SCRIPT) - - local HEADER=$( - [ $SUBSCWRYPT ] && return 0 - echo '=====================================================================' - echo "script : $SCRIPT" - echo "run at : $(date)" - echo "config : $ENV_NAME" - [ ! $LOGFILE ] && echo '\033[1;33m------------------------------------------\033[0m' - ) - - [ ! $LOGFILE ] && { - [ $HEADER ] && echo $HEADER - eval $RUN_STRING $@ /dev/tty 2>&1 - exit $? - } - - { - [ $HEADER ] && echo $HEADER - echo '\033[1;33m--- BEGIN OUTPUT -------------------------\033[0m' - eval $RUN_STRING $@ - EXIT_CODE=$? - echo '\033[1;33m--- END OUTPUT ---------------------------\033[0m' - - [[ $EXIT_CODE -eq 0 ]] && EXIT_COLOR='32m' || EXIT_COLOR='31m' - - echo "terminated with\\033[1;$EXIT_COLOR code $EXIT_CODE\\033[0m" - } 2>&1 | tee --append "$LOGFILE" - - exit $(\ - sed -n 's/^terminated with.*code \([0-9]*\).*$/\1/p' $LOGFILE \ - | tail -n1 - ) -} - -##################################################################### - -__SELECT_SCRIPT() { - local SCRIPT - local SCRIPTS=$(__GET_AVAILABLE_SCRIPTS) - local SEARCH=($@) - - [[ ${#SEARCH[@]} -eq 0 ]] && { - SCRIPT=$(echo $SCRIPTS | __FZF 'select a script') - } - - [[ ${#SEARCH[@]} -eq 1 ]] && [ -f ./$SEARCH ] && { - SCRIPT=$SEARCH - } - - [ ! $SCRIPT ] && [[ ${#SEARCH[@]} -gt 0 ]] && { - SCRIPT=$SCRIPTS - for PATTERN in $SEARCH - do - SCRIPT=$(echo $SCRIPT | grep $PATTERN) - done - - [ ! $SCRIPT ] && __FAIL 2 "no script found by name '$@'" - - [[ $(echo $SCRIPT | wc -l) -gt 1 ]] && { - __STATUS "more than one script matched '$@'" - SCRIPT=$(echo $SCRIPT | __FZF 'select a script') - } - } - - echo $SCRIPT -} - -__GET_RUN_STRING() { - local SCRIPT="$1" - local ENV_NAME="$2" - local TYPE=$(echo $SCRIPT | sed 's/\/.*$//') - - local RUN_STRING - - local _VIRTUALENV="$SCWRYPTS_VIRTUALENV_PATH/$TYPE/bin/activate" - [ -f $_VIRTUALENV ] && source $_VIRTUALENV - - case $TYPE in - py ) __CHECK_DEPENDENCY python || return 1 - RUN_STRING="python -m $(echo $SCRIPT | sed 's/\//./g; s/\.py$//; s/\.\.//')" - - CURRENT_PYTHON_VERSION=$(python --version | sed 's/^[^0-9]*\(3\.[^.]*\).*$/\1/') - - echo $__PREFERRED_PYTHON_VERSIONS | grep -q $CURRENT_PYTHON_VERSION || { - __WARNING "only tested on the following python versions: $(printf ', %s.x' ${__PREFERRED_PYTHON_VERSIONS[@]} | sed 's/^, //')" - __WARNING 'compatibility may vary' - } - ;; - - zsh ) __CHECK_DEPENDENCY zsh || return 1 - RUN_STRING="./$SCRIPT" - ;; - - zx ) __CHECK_DEPENDENCY zx || return 1 - RUN_STRING="FORCE_COLOR=3 ./$SCRIPT.mjs" - ;; - - * ) __ERROR "unsupported script type '$SCRIPT_TYPE'" - return 2 - ;; - esac - - RUN_STRING="SCWRYPTS_ENV='$ENV_NAME' $RUN_STRING" - [ -f $_VIRTUALENV ] && RUN_STRING="source '$_VIRTUALENV'; $RUN_STRING" - - echo $RUN_STRING -} - -__CHECK_ENV_REQUIRED() { - [ $CI ] && return 1 - - echo $SCRIPT | grep -q 'zsh/scwrypts/logs' && return 1 - - return 0 -} - -__VALIDATE_UPSTREAM_TIMELINE() { - __STATUS "on '$ENV_NAME'; checking diff against origin/main" - - git fetch --quiet origin main - local SYNC_STATUS=$? - - git diff --exit-code origin/main -- . >&2 - local DIFF_STATUS=$? - - [[ $SYNC_STATUS -eq 0 ]] && [[ $DIFF_STATUS -eq 0 ]] && { - __SUCCESS 'up-to-date with origin/main' - } || { - __WARNING - [[ $SYNC_STATUS -ne 0 ]] && __WARNING 'unable to synchronize with origin/main' - [[ $DIFF_STATUS -ne 0 ]] && __WARNING 'your branch differs from origin/main (diff listed above)' - __WARNING - - __yN 'continue?' || return 1 - } -} - -__GET_LOGFILE() { - local SCRIPT="$1" - - [ $SUBSCWRYPT ] \ - || [[ $SCRIPT =~ scwrypts/logs ]] \ - || [[ $SCRIPT =~ interactive ]] \ - && return 0 - - echo "$SCWRYPTS_LOG_PATH/$(echo $SCRIPT | sed 's/^\.\///; s/\//\%/g').log" -} - -##################################################################### -__RUN $@ +source "${0:a:h}/run" diff --git a/zsh/aws/rds/common.zsh b/zsh/aws/rds/common.zsh index f4e44c4..5760f0b 100644 --- a/zsh/aws/rds/common.zsh +++ b/zsh/aws/rds/common.zsh @@ -3,20 +3,127 @@ _REQUIRED_ENV+=() source ${0:a:h}/../common.zsh ##################################################################### -__SELECT_CONNECTOR() { - local DB_TYPE="$1" +GET_DATABASE_CREDENTIALS() { + local PRINT_PASSWORD=0 + local ARGS_ERRORS=0 - CLIENTS_postgresql=(pgcli psql) - - local C CLIENT=none - for C in $(eval 'echo $CLIENTS_'$DB_TYPE) + while [[ $# -gt 0 ]] do - __CHECK_DEPENDENCY $C >/dev/null 2>&1 && { - CLIENT=$C - __STATUS "detected '$CLIENT' for $DB_TYPE" - break - } + case $1 in + --print-password ) PRINT_PASSWORD=1 ;; + * ) + __WARNING "unrecognized argument $1" + ARGS_ERRORS+=1 + ;; + esac + shift 1 done + [[ $ARGS_ERRORS -ne 0 ]] && return 1 + + ########################################## + + local DATABASE=$(SELECT_DATABASE) + [ ! $DATABASE ] && __ABORT + + DB_HOST="$(echo $DATABASE | jq -r '.host')" + [ ! $DB_HOST ] && { __ERROR 'unable to find host'; return 2; } + + DB_PORT="$(echo $DATABASE | jq -r '.port')" + [ ! $DB_PORT ] && DB_PORT=5432 + [[ $DB_PORT =~ ^null$ ]] && DB_PORT=5432 + + ########################################## + + local AUTH_METHOD=$(\ + echo "iam\nsecretsmanager\nuser-input" \ + | __FZF 'select an authentication method' \ + ) + [ ! $AUTH_METHOD ] && __ABORT + + case $AUTH_METHOD in + iam ) GET_AUTH__IAM ;; + secretsmanager ) GET_AUTH__SECRETSMANAGER ;; + user-input ) GET_AUTH__USER_INPUT ;; + esac + + __STATUS + __STATUS "host : $DB_HOST" + __STATUS "type : $DB_TYPE" + __STATUS "port : $DB_PORT" + __STATUS "database : $DB_NAME" + __STATUS "username : $DB_USER" + [[ $PRINT_PASSWORD -eq 1 ]] && __STATUS "password : $DB_PASS" + __STATUS +} + +GET_AUTH__IAM() { + DB_PASS=$(\ + _AWS rds generate-db-auth-token \ + --hostname $DB_HOST \ + --port $DB_PORT \ + --username $DB_USER \ + ) +} - echo $CLIENT +GET_AUTH__SECRETSMANAGER() { + local CREDENTIALS=$(GET_SECRETSMANAGER_CREDENTIALS) + echo $CREDENTIALS | jq -e '.pass' >/dev/null 2>&1 \ + && DB_PASS="'$(echo $CREDENTIALS | jq -r '.pass' | sed "s/'/'\"'\"'/g")'" + + echo $CREDENTIALS | jq -e '.password' >/dev/null 2>&1 \ + && DB_PASS="'$(echo $CREDENTIALS | jq -r '.password' | sed "s/'/'\"'\"'/g")'" + + echo $CREDENTIALS | jq -e '.user' >/dev/null 2>&1 \ + && DB_USER=$(echo $CREDENTIALS | jq -r '.user') + + echo $CREDENTIALS | jq -e '.username' >/dev/null 2>&1 \ + && DB_USER=$(echo $CREDENTIALS | jq -r '.username') + + echo $CREDENTIALS | jq -e '.name' >/dev/null 2>&1 \ + && DB_NAME=$(echo $CREDENTIALS | jq -r '.name') + + echo $CREDENTIALS | jq -e '.dbname' >/dev/null 2>&1 \ + && DB_NAME=$(echo $CREDENTIALS | jq -r '.dbname') } + +GET_SECRETSMANAGER_CREDENTIALS() { + local ID=$(\ + _AWS secretsmanager list-secrets \ + | jq -r '.[] | .[] | .Name' \ + | __FZF 'select a secret' \ + ) + [ ! $ID ] && return 1 + + _AWS secretsmanager get-secret-value --secret-id "$ID" \ + | jq -r '.SecretString' | jq +} + +SELECT_DATABASE() { + local DATABASES=$(GET_AVAILABLE_DATABASES) + [ ! $DATABASES ] && __FAIL 1 'no databases available' + + local ID=$(\ + echo $DATABASES | jq -r '.instance + " @ " + .cluster' \ + | __FZF 'select a database (instance@cluster)' \ + ) + [ ! $ID ] && __ABORT + + local INSTANCE=$(echo $ID | sed 's/ @ .*$//') + local CLUSTER=$(echo $ID | sed 's/^.* @ //') + + echo $DATABASES | jq "select (.instance == \"$INSTANCE\" and .cluster == \"$CLUSTER\")" +} + +GET_AVAILABLE_DATABASES() { + _AWS rds describe-db-instances \ + | jq -r '.[] | .[] | { + instance: .DBInstanceIdentifier, + cluster: .DBClusterIdentifier, + type: .Engine, + host: .Endpoint.Address, + port: .Endpoint.Port, + user: .MasterUsername, + database: .DBName + }' +} + diff --git a/zsh/aws/rds/create-backup b/zsh/aws/rds/create-backup new file mode 100755 index 0000000..bffa2d1 --- /dev/null +++ b/zsh/aws/rds/create-backup @@ -0,0 +1,22 @@ +#!/bin/zsh +_DEPENDENCIES+=() +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +RDS_INTERACTIVE_LOGIN() { + local DB_HOST DB_PORT DB_NAME DB_USER DB_PASS + GET_DATABASE_CREDENTIALS $@ || return 1 + + __RUN_SCWRYPT 'zsh/db/postgres/pg_dump' -- \ + --host $DB_HOST \ + --port $DB_PORT \ + --name $DB_NAME \ + --user $DB_USER \ + --pass $DB_PASS \ + ; +} + + +##################################################################### +RDS_INTERACTIVE_LOGIN $@ diff --git a/zsh/aws/rds/interactive-login b/zsh/aws/rds/interactive-login index 80d262f..a25d999 100755 --- a/zsh/aws/rds/interactive-login +++ b/zsh/aws/rds/interactive-login @@ -4,118 +4,19 @@ _REQUIRED_ENV+=() source ${0:a:h}/common.zsh ##################################################################### -__CONNECT_TO_RDS() { - local DATABASE=$(__SELECT_DATABASE) - [ ! $DATABASE ] && __ABORT - - local DB_HOST DB_USER DB_PORT DB_NAME DB_AUTH DB_TYPE - - DB_HOST=$(echo $DATABASE | jq -r '.host') - DB_USER=$(echo $DATABASE | jq -r '.user') - DB_PORT=$(echo $DATABASE | jq -r '.port') - DB_TYPE=$(echo $DATABASE | jq -r '.type') - - [[ $DB_PORT =~ null ]] && DB_PORT=5432 - DB_NAME=postgres - - local AUTH_METHODS=(iam secretsmanager user-input) - local AUTH_METHOD=$(\ - echo $AUTH_METHODS | sed 's/\s\+/\n/g' \ - | __FZF 'select an authentication method' \ - ) - - [ ! $AUTH_METHOD ] && __ABORT - - case $AUTH_METHOD in - iam ) - DB_AUTH=$(\ - _AWS rds generate-db-auth-token \ - --hostname $DB_HOST \ - --port $DB_PORT \ - --username $DB_USER \ - ) - ;; - secretsmanager ) - CREDENTIALS=$(__GET_SECRETSMANAGER_CREDENTIALS) - echo $CREDENTIALS | jq -e '.pass' >/dev/null 2>&1 \ - && DB_AUTH="'$(echo $CREDENTIALS | jq -r '.pass' | sed "s/'/'\"'\"'/g")'" - - echo $CREDENTIALS | jq -e '.password' >/dev/null 2>&1 \ - && DB_AUTH="'$(echo $CREDENTIALS | jq -r '.password' | sed "s/'/'\"'\"'/g")'" - - echo $CREDENTIALS | jq -e '.user' >/dev/null 2>&1 \ - && DB_USER=$(echo $CREDENTIALS | jq -r '.user') - - echo $CREDENTIALS | jq -e '.username' >/dev/null 2>&1 \ - && DB_USER=$(echo $CREDENTIALS | jq -r '.username') - - echo $CREDENTIALS | jq -e '.name' >/dev/null 2>&1 \ - && DB_NAME=$(echo $CREDENTIALS | jq -r '.name') - - echo $CREDENTIALS | jq -e '.dbname' >/dev/null 2>&1 \ - && DB_NAME=$(echo $CREDENTIALS | jq -r '.dbname') - ;; - user-input ) - ;; - esac - - __STATUS - __STATUS "host : $DB_HOST" - __STATUS "type : $DB_TYPE" - __STATUS "port : $DB_PORT" - __STATUS "database : $DB_NAME" - __STATUS "username : $DB_USER" - __STATUS +RDS_INTERACTIVE_LOGIN() { + local DB_HOST DB_PORT DB_NAME DB_USER DB_PASS + GET_DATABASE_CREDENTIALS $@ || return 1 __RUN_SCWRYPT 'zsh/db/interactive/postgres' -- \ --host $DB_HOST \ --port $DB_PORT \ --name $DB_NAME \ --user $DB_USER \ - --pass $DB_AUTH \ + --pass $DB_PASS \ ; } -__SELECT_DATABASE() { - local DATABASES=$(__GET_AVAILABLE_DATABASES) - [ ! $DATABASES ] && __FAIL 1 'no databases available' - - local ID=$(\ - echo $DATABASES | jq -r '.instance + " @ " + .cluster' \ - | __FZF 'select a database (instance@cluster)' \ - ) - [ ! $ID ] && __ABORT - - local INSTANCE=$(echo $ID | sed 's/ @ .*$//') - local CLUSTER=$(echo $ID | sed 's/^.* @ //') - - echo $DATABASES | jq "select (.instance == \"$INSTANCE\" and .cluster == \"$CLUSTER\")" -} - -__GET_AVAILABLE_DATABASES() { - _AWS rds describe-db-instances \ - | jq -r '.[] | .[] | { - instance: .DBInstanceIdentifier, - cluster: .DBClusterIdentifier, - type: .Engine, - host: .Endpoint.Address, - port: .Endpoint.Port, - user: .MasterUsername, - database: .DBName - }' -} - -__GET_SECRETSMANAGER_CREDENTIALS() { - local ID=$(\ - _AWS secretsmanager list-secrets \ - | jq -r '.[] | .[] | .Name' \ - | __FZF 'select a secret' \ - ) - [ ! $ID ] && return 1 - - _AWS secretsmanager get-secret-value --secret-id "$ID" \ - | jq -r '.SecretString' | jq -} ##################################################################### -__CONNECT_TO_RDS +RDS_INTERACTIVE_LOGIN $@ diff --git a/zsh/aws/rds/load-backup b/zsh/aws/rds/load-backup new file mode 100755 index 0000000..e5487e6 --- /dev/null +++ b/zsh/aws/rds/load-backup @@ -0,0 +1,22 @@ +#!/bin/zsh +_DEPENDENCIES+=() +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +RDS_INTERACTIVE_LOGIN() { + local DB_HOST DB_PORT DB_NAME DB_USER DB_PASS + GET_DATABASE_CREDENTIALS $@ || return 1 + + __RUN_SCWRYPT 'zsh/db/postgres/pg_restore' -- \ + --host $DB_HOST \ + --port $DB_PORT \ + --name $DB_NAME \ + --user $DB_USER \ + --pass $DB_PASS \ + ; +} + + +##################################################################### +RDS_INTERACTIVE_LOGIN $@ diff --git a/zsh/db/common.zsh b/zsh/db/common.zsh index 1191a72..146325e 100644 --- a/zsh/db/common.zsh +++ b/zsh/db/common.zsh @@ -2,3 +2,23 @@ _DEPENDENCIES+=() _REQUIRED_ENV+=() source ${0:a:h}/../common.zsh ##################################################################### + + +GET_POSTGRES_LOGIN_ARGS() { + while [[ $# -gt 0 ]] + do + case $1 in + --host | -h ) _HOST="$2"; shift 2 ;; + --name | -d ) _NAME="$2"; shift 2 ;; + --pass | -w ) _PASS="$2"; shift 2 ;; + --port | -p ) _PORT="$2"; shift 2 ;; + --user | -U ) _USER="$2"; shift 2 ;; + * ) shift 1 ;; + esac + done + + [ ! $_HOST ] && _HOST=127.0.0.1 + [ ! $_NAME ] && _NAME=postgres + [ ! $_PORT ] && _PORT=5432 + [ ! $_USER ] && _USER=postgres +} diff --git a/zsh/db/interactive/postgres b/zsh/db/interactive/postgres index 2333492..9b310dd 100755 --- a/zsh/db/interactive/postgres +++ b/zsh/db/interactive/postgres @@ -8,23 +8,7 @@ source ${0:a:h}/common.zsh _LOGIN_POSTGRES() { local _HOST _NAME _PASS _PORT _USER - - while [[ $# -gt 0 ]] - do - case $1 in - --host | -h ) _HOST="$2"; shift 2 ;; - --name | -d ) _NAME="$2"; shift 2 ;; - --pass | -w ) _PASS="$2"; shift 2 ;; - --port | -p ) _PORT="$2"; shift 2 ;; - --user | -U ) _USER="$2"; shift 2 ;; - * ) shift 1 ;; - esac - done - - [ ! $_HOST ] && _HOST=127.0.0.1 - [ ! $_NAME ] && _NAME=postgres - [ ! $_PORT ] && _PORT=5432 - [ ! $_USER ] && _USER=postgres + GET_POSTGRES_LOGIN_ARGS $@ local DATA_DIR="$SCWRYPTS_DATA_PATH/db/$_HOST" [ ! -d $DATA_DIR ] && mkdir -p $DATA_DIR diff --git a/zsh/db/postgres/common.zsh b/zsh/db/postgres/common.zsh new file mode 100644 index 0000000..1191a72 --- /dev/null +++ b/zsh/db/postgres/common.zsh @@ -0,0 +1,4 @@ +_DEPENDENCIES+=() +_REQUIRED_ENV+=() +source ${0:a:h}/../common.zsh +##################################################################### diff --git a/zsh/db/postgres/pg_dump b/zsh/db/postgres/pg_dump new file mode 100755 index 0000000..9a62e55 --- /dev/null +++ b/zsh/db/postgres/pg_dump @@ -0,0 +1,44 @@ +#!/bin/zsh +_DEPENDENCIES+=( + pg_dump +) +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +BACKUP_POSTGRES() { + local _HOST _NAME _PASS _PORT _USER + GET_POSTGRES_LOGIN_ARGS $@ + + local DATA_DIR="$SCWRYPTS_DATA_PATH/db/$_HOST/$_NAME/pg_dump" + [ ! -d $DATA_DIR ] && mkdir -p $DATA_DIR + cd $DATA_DIR + + local OUTPUT_FILE="$DATA_DIR/$_NAME.dump" + [ -f $OUTPUT_FILE ] && { + local BACKUP_COUNT=$(ls "$DATA_DIR/$_NAME."*".dump" | wc -l) + ls "$DATA_DIR/$_NAME."*".dump" + + __INFO "discovered previous dump for '$_HOST/$_NAME'" + __INFO "backing up previous dump to '$_NAME.$BACKUP_COUNT.dump'" + + mv "$OUTPUT_FILE" "$DATA_DIR/$_NAME.$BACKUP_COUNT.dump" + } + + __STATUS "making backup of : $_USER@$_HOST:$_PORT/$_NAME" + __STATUS "output file : $OUTPUT_FILE" + + PGPASSWORD="$_PASS" pg_dump \ + --verbose \ + --format custom \ + --host "$_HOST" \ + --port "$_PORT" \ + --username "$_USER" \ + --dbname "$_NAME" \ + --file "$OUTPUT_FILE" \ + && { __SUCCESS "finished backup of '$_HOST/$_NAME'"; __SUCCESS "saved to '$OUTPUT_FILE'"; } \ + || { __ERROR "error creating backup for '$_HOST/$_NAME' (see above)"; return 1; } +} + +##################################################################### +BACKUP_POSTGRES $@ diff --git a/zsh/db/postgres/pg_restore b/zsh/db/postgres/pg_restore new file mode 100755 index 0000000..885eb3d --- /dev/null +++ b/zsh/db/postgres/pg_restore @@ -0,0 +1,55 @@ +#!/bin/zsh +_DEPENDENCIES+=( + pg_dump +) +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +BACKUP_POSTGRES() { + local _HOST _NAME _PASS _PORT _USER + GET_POSTGRES_LOGIN_ARGS $@ + + local DATA_DIR="$SCWRYPTS_DATA_PATH/db/$_HOST/$_NAME/pg_restore" + [ ! -d $DATA_DIR ] && mkdir -p $DATA_DIR + cd $DATA_DIR + + local INPUT_FILE="$DATA_DIR/$_NAME.dump" + + [ ! -f $INPUT_FILE ] && { + local DUMP="$(dirname $DATA_DIR)/pg_dump/$_NAME.dump" + __STATUS $DUMP + ls $DUMP + + [ -f "$DUMP" ] && { + __SUCCESS "discovered previous scwrypts dump" + __SUCCESS "$DUMP" + __Yn 'restore from this backup?' && INPUT_FILE="$DUMP" + } + + [ ! -f "$INPUT_FILE" ] && { + __STATUS 'place backup in the following location:' + __STATUS "$INPUT_FILE" + } + + while [ ! -f $INPUT_FILE ]; do sleep 1; done + } + + __STATUS "backup file : $DATA_DIR" + __STATUS "database : $_USER@$_HOST:$_PORT/$_NAME" + + PGPASSWORD="$_PASS" pg_restore \ + --verbose \ + --single-transaction \ + --format custom \ + --host "$_HOST" \ + --port "$_PORT" \ + --username "$_USER" \ + --dbname "$_NAME" \ + "$INPUT_FILE" \ + && { __SUCCESS "finished restoring backup for '$_HOST/$_NAME'"; } \ + || { __ERROR "error restoring backup for '$_HOST/$_NAME' (see above)"; return 1; } +} + +##################################################################### +BACKUP_POSTGRES $@ diff --git a/zsh/redis/common.zsh b/zsh/redis/common.zsh new file mode 100644 index 0000000..3b4f44b --- /dev/null +++ b/zsh/redis/common.zsh @@ -0,0 +1,22 @@ +_DEPENDENCIES+=( + redis-cli +) +_REQUIRED_ENV+=() +source ${0:a:h}/../common.zsh + +[ ! $SCWRYPTS_CACHE_HOST ] && SCWRYPTS_CACHE_HOST=localhost +[ ! $SCWRYPTS_CACHE_PORT ] && SCWRYPTS_CACHE_PORT=6379 +##################################################################### + +_REDIS() { + local ARGS=() + + ARGS+=(-h $SCWRYPTS_CACHE_HOST) + ARGS+=(-p $SCWRYPTS_CACHE_PORT) + + [ $SCWRYPTS_CACHE_AUTH ] && ARGS+=(-a $SCWRYPTS_CACHE_AUTH) + + redis-cli ${ARGS[@]} $@ +} + +CACHE_ENABLED=$(_REDIS ping 2>&1 | grep -qi pong && echo 1 || echo 0) diff --git a/zsh/redis/curl b/zsh/redis/curl new file mode 100755 index 0000000..beef55d --- /dev/null +++ b/zsh/redis/curl @@ -0,0 +1,48 @@ +#!/bin/zsh +_DEPENDENCIES+=() +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +CURL_WITH_CACHE() { + [ ! $TTL ] && TTL=10 + [[ $CACHE_ENABLED -eq 0 ]] && { + curl $@ + return $? + } + + local ARGS=() + local URL + + while [[ $# -gt 0 ]] + do + case $1 in + -- ) shift 1 ;; + --*= ) ARGS+=($1); shift 1 ;; + --* ) ARGS+=($1 $2); shift 2 ;; + -* ) ARGS+=($1); shift 1 ;; + * ) URL=$1; break ;; + esac + done + + local KEY=$(GET_URL_KEY $URL) + local OUTPUT=$(_REDIS get $KEY 2>&1) + [ $OUTPUT ] && { + [[ ${#ARGS[@]} -gt 0 ]] && __WARN "cache hit found; ignoring arguments ($ARGS)" + echo $OUTPUT + return + } + + local OUTPUT=$(curl -s $@) + [ ! $OUTPUT ] && return 1 + + _REDIS set $KEY "$OUTPUT" >/dev/null + _REDIS expire $KEY $TTL >/dev/null + + echo $OUTPUT +} + +GET_URL_KEY() { echo "scwrypts:curl:$1" | sed 's/\s\+/+/g'; } + +##################################################################### +CURL_WITH_CACHE $@ diff --git a/zsh/youtube/README.md b/zsh/youtube/README.md new file mode 100644 index 0000000..1bf7b5d --- /dev/null +++ b/zsh/youtube/README.md @@ -0,0 +1,5 @@ +# ZSH Scwrypts +[![Generic Badge](https://img.shields.io/badge/ytdl--org-youtube--dl-informational.svg)](https://github.com/ytdl-org/youtube-dl) +
+ +Quick wrappers for downloading and trimming YouTube videos. diff --git a/zsh/youtube/common.zsh b/zsh/youtube/common.zsh new file mode 100644 index 0000000..3a81a29 --- /dev/null +++ b/zsh/youtube/common.zsh @@ -0,0 +1,45 @@ +_DEPENDENCIES+=( + youtube-dl + ffmpeg +) +_REQUIRED_ENV+=() +source ${0:a:h}/../common.zsh +##################################################################### + +YT__GLOBAL_ARGS=( + --no-call-home + --restrict-filenames + ) + +YT__OUTPUT_DIR="$SCWRYPTS_DATA_PATH/youtube" + +YT__GET_INFO() { + youtube-dl --dump-json ${YT__GLOBAL_ARGS[@]} $@ +} + +YT__GET_FILENAME() { + YT__GET_INFO $@ \ + | jq -r '._filename' \ + | sed 's/\.[^.]*$/\.mp4/' \ + ; +} + +YT__DOWNLOAD() { + local OUTPUT_DIR="$SCWRYPTS_DATA_PATH/youtube" + [ ! -d $YT__OUTPUT_DIR ] && mkdir -p $YT__OUTPUT_DIR + cd "$YT__OUTPUT_DIR" + youtube-dl ${YT__GLOBAL_ARGS[@]} $@ \ + --format 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/mp4' \ + ; +} + +GET_VIDEO_LENGTH() { + local FILENAME="$1" + + ffprobe \ + -v quiet \ + -show_entries format=duration \ + -of default=noprint_wrappers=1:nokey=1 \ + -i $FILENAME \ + ; +} diff --git a/zsh/youtube/download b/zsh/youtube/download new file mode 100755 index 0000000..33105e1 --- /dev/null +++ b/zsh/youtube/download @@ -0,0 +1,25 @@ +#!/bin/zsh +_DEPENDENCIES+=() +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +DOWNLOAD_VIDEO() { + local URLS=($@) + + [[ ${#URLS[@]} -eq 0 ]] && URLS=($(echo '' | __FZF_HEAD 'enter URL')) + [[ ${#URLS[@]} -eq 0 ]] && __ABORT + + local FILENAME=$(YT__GET_FILENAME $URLS) + [ ! $FILENAME ] && __ERROR "unable to download '$URLS'" + + __SUCCESS "Found '$FILENAME'" + __Yn "Proceed with download?" || return 1 + + YT__DOWNLOAD $URLS \ + && __SUCCESS "downloaded to '$YT__OUTPUT_DIR/$FILENAME'" \ + || { __ERROR "failed to download '$FILENAME'"; return 2; } +} + +##################################################################### +DOWNLOAD_VIDEO $@ diff --git a/zsh/youtube/get-audio-clip b/zsh/youtube/get-audio-clip new file mode 100755 index 0000000..dfaa82c --- /dev/null +++ b/zsh/youtube/get-audio-clip @@ -0,0 +1,51 @@ +#!/bin/zsh +_DEPENDENCIES+=() +_REQUIRED_ENV+=() +source ${0:a:h}/common.zsh +##################################################################### + +GET_AUDIO_CLIP() { + local URLS=($@) + + [[ ${#URLS[@]} -eq 0 ]] && URLS=($(echo '' | __FZF_HEAD 'enter URL')) + [[ ${#URLS[@]} -eq 0 ]] && __ABORT + + local FILENAME=$(YT__GET_FILENAME $URLS) + [ ! $FILENAME ] && __ERROR "unable to download '$URLS'" + + INPUT_FILE="$YT__OUTPUT_DIR/$FILENAME" + + [ ! -f "$INPUT_FILE" ] && { + __RUN_SCWRYPT youtube/download -- $URLS || return 1 + } + + __SUCCESS "video download '$FILENAME' detected!" + + LENGTH=$(GET_VIDEO_LENGTH "$INPUT_FILE") + [ ! $LENGTH ] && { __ERROR "unable to determine video length for '$INPUT_FILE'"; return 2; } + START_TIME=$(echo 0 | __FZF_HEAD "enter start time (0 ≤ t < $LENGTH)") + [ ! $START_TIME ] && __ABORT + END_TIME=$(echo $LENGTH | __FZF_HEAD "enter end time ($START_TIME > t ≥ $LENGTH)") + [ ! $END_TIME ] && __ABORT + + __STATUS + __STATUS "video : $FILENAME" + __STATUS "start time : $START_TIME" + __STATUS "end time : $END_TIME" + __STATUS + OUTPUT_FILE=$(echo '' \ + | __FZF_HEAD 'what should I call this clip? (.mp3)' \ + | sed 's/\.mp3$//' \ + ) + [ ! $OUTPUT_FILE ] && __ABORT + OUTPUT_FILE="$YT__OUTPUT_DIR/$OUTPUT_FILE.mp3" + + ffmpeg -i "$INPUT_FILE" -q:a 0 -map a \ + -ss $START_TIME -t $(($END_TIME - $START_TIME))\ + "$OUTPUT_FILE" \ + && __SUCCESS "created clip '$OUTPUT_FILE'" \ + || { __ERROR "error creating clip '$(basename $OUTPUT_FILE)' (see above)"; return 3; } +} + +##################################################################### +GET_AUDIO_CLIP $@