diff --git a/.gitignore b/.gitignore index 36ebb0c5..e1165410 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,13 @@ venv # Testing .pytest_cache/ + +# Ignore PSC puppet generated and copied files +.env +init_env.sh +docker/docker-compose.yml +docker/docker-compose.dev.yml +docker-compose.deployment.prod.yml +docker/docker-development.sh +docker/docker-deployment.sh +docker/ingest-api/nginx diff --git a/docker/docker-compose.deployment.yml b/docker/docker-compose.deployment.yml deleted file mode 100644 index 1a225e94..00000000 --- a/docker/docker-compose.deployment.yml +++ /dev/null @@ -1,48 +0,0 @@ -version: "3.7" - -services: - - ingest-api: - # Use the published image and tag from DockerHub - image: sennet/ingest-api:${INGEST_API_VERSION:?err} - volumes: - # Mount the VERSION file and BUILD file - - "../VERSION:/usr/src/app/VERSION" - - "../BUILD:/usr/src/app/BUILD" - # Mount data directory for Globus transfer PROD - - "/codcc/data:/codcc/data" - - "/codcc/scratch:/codcc/scratch" - - ingest-api-rq-worker: - build: - context: ./ingest-api - dockerfile: ./rq/Dockerfile - image: sennet/ingest-api-rq-server:${INGEST_API_VERSION:?err} - volumes: - # Mount the VERSION file and BUILD file - - "../VERSION:/usr/src/app/VERSION" - - "../BUILD:/usr/src/app/BUILD" - # Mount data directory for Globus transfer PROD - - "/codcc/data:/codcc/data" - - "/codcc/scratch:/codcc/scratch" - deploy: - replicas: 4 - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "10" - networks: - - sennet_docker_network - - ingest-api-rq-server: - image: redis:alpine - hostname: ingest-api-rq-server - container_name: ingest-api-rq-server - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "10" - networks: - - sennet_docker_network diff --git a/docker/docker-compose.development.yml b/docker/docker-compose.development.yml deleted file mode 100644 index 1e099326..00000000 --- a/docker/docker-compose.development.yml +++ /dev/null @@ -1,55 +0,0 @@ -version: "3.7" - -services: - - ingest-api: - build: - context: ./ingest-api - # Uncomment if tesitng against a specific branch of commons other than the PyPI package - # Will also need to use the 'git+https://github.com/hubmapconsortium/commons.git@${COMMONS_BRANCH}#egg=hubmap-commons' - # in src/requirements.txt accordingly - args: - # The commons github branch to be used during image build (default to main if not set or null) - - COMMONS_BRANCH=${COMMONS_BRANCH:-main} - # Build the image with name and tag - # Exit with an error message containing err if unset or empty in the environment - image: sennet/ingest-api:${INGEST_API_VERSION:?err} - volumes: - # Mount the VERSION file and BUILD file - - "../VERSION:/usr/src/app/VERSION" - - "../BUILD:/usr/src/app/BUILD" - # Mount the source code to container - - "../src:/usr/src/app/src" - # Mount data directory for Globus transfer - - "/codcc/dev-test/data-dev:/codcc/dev-test/data-dev" - - ingest-api-rq-worker: - build: - context: ./ingest-api - dockerfile: ./rq/Dockerfile - image: sennet/ingest-api-rq-server:${INGEST_API_VERSION:?err} - volumes: - - "../src:/usr/src/app/src" - # Mount data directory for Globus transfer - - "/codcc/dev-test/data-dev:/codcc/dev-test/data-dev" - deploy: - replicas: 4 - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "10" - networks: - - sennet_docker_network - - ingest-api-rq-server: - image: redis:alpine - hostname: ingest-api-rq-server - container_name: ingest-api-rq-server - logging: - driver: "json-file" - options: - max-size: "10m" - max-file: "10" - networks: - - sennet_docker_network diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 3de06228..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,38 +0,0 @@ -version: "3.7" - -# Will use the hostname when talking between services on the same network -services: - - ingest-api: - hostname: ingest-api - container_name: ingest-api - # Map host machine port 5555 to container port 8080 - # Only root can listen on ports below 1024, we use higher-numbered ports - # since nginx is running under non-root user hubmap - ports: - - "5555:8080" - healthcheck: - test: ["CMD", "curl", "--fail", "http://localhost:8080"] - interval: 1m30s - timeout: 10s - retries: 3 - start_period: 40s - environment: - - HOST_GID=${HOST_GID:-1002} - - HOST_UID=${HOST_UID:-1002} - init: true - restart: always - volumes: - # Mount the app config to container in order to keep it outside of the image - - "../src/instance:/usr/src/app/src/instance" - # Mount the logging to container - - "../log:/usr/src/app/log" - # Mount conf.d on host machine to the nginx conf.d on container - - "./ingest-api/nginx/conf.d:/etc/nginx/conf.d" - networks: - - sennet_docker_network - -networks: - # This is the network created by gateway to enable communicaton between multiple docker-compose projects - sennet_docker_network: - external: true diff --git a/docker/docker-deployment.sh b/docker/docker-deployment.sh deleted file mode 100755 index d6cf8927..00000000 --- a/docker/docker-deployment.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/bash - -# Print a new line and the banner -echo -echo "==================== INGEST-API ====================" - -function get_dir_of_this_script() { - # This function sets DIR to the directory in which this script itself is found. - # Thank you https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself - SCRIPT_SOURCE="${BASH_SOURCE[0]}" - while [ -h "$SCRIPT_SOURCE" ]; do # resolve $SCRIPT_SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" >/dev/null 2>&1 && pwd )" - SCRIPT_SOURCE="$(readlink "$SCRIPT_SOURCE")" - [[ $SCRIPT_SOURCE != /* ]] && SCRIPT_SOURCE="$DIR/$SCRIPT_SOURCE" # if $SCRIPT_SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" >/dev/null 2>&1 && pwd )" - echo 'DIR of script:' $DIR -} - -# Generate the build version based on git branch name and short commit hash and write into BUILD file -function generate_build_version() { - GIT_BRANCH_NAME=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') - GIT_SHORT_COMMIT_HASH=$(git rev-parse --short HEAD) - # Clear the old BUILD version and write the new one - truncate -s 0 ../BUILD - # Note: echo to file appends newline - echo $GIT_BRANCH_NAME:$GIT_SHORT_COMMIT_HASH >> ../BUILD - # Remmove the trailing newline character - truncate -s -1 ../BUILD - - echo "BUILD(git branch name:short commit hash): $GIT_BRANCH_NAME:$GIT_SHORT_COMMIT_HASH" -} - -# Set the version environment variable for the docker build -# Version number is from the VERSION file -# Also remove newlines and leading/trailing slashes if present in that VERSION file -function export_version() { - export INGEST_API_VERSION=$(tr -d "\n\r" < ../VERSION | xargs) - echo "INGEST_API_VERSION: $INGEST_API_VERSION" -} - - -if [[ "$1" != "start" && "$1" != "stop" && "$1" != "down" ]]; then - echo "Unknown command '$1', specify one of the following: start|stop|down" -else - # Always show the script dir - get_dir_of_this_script - - # Always export and show the version - export_version - - # Always show the build in case branch changed or new commits - generate_build_version - - # Print empty line - echo - - if [ "$1" = "start" ]; then - docker-compose -f docker-compose.yml -f docker-compose.deployment.yml -p ingest-api up -d - elif [ "$1" = "stop" ]; then - docker-compose -f docker-compose.yml -f docker-compose.deployment.yml -p ingest-api stop - elif [ "$1" = "down" ]; then - docker-compose -f docker-compose.yml -f docker-compose.deployment.yml -p ingest-api down - fi -fi - diff --git a/docker/docker-development.sh b/docker/docker-development.sh deleted file mode 100755 index 5d68bc18..00000000 --- a/docker/docker-development.sh +++ /dev/null @@ -1,118 +0,0 @@ -#!/bin/bash - -# Print a new line and the banner -echo -echo "==================== INGEST-API ====================" - -# The `absent_or_newer` checks if the copied src at docker/some-api/src directory exists -# and if the source src directory is newer. -# If both conditions are true `absent_or_newer` writes an error message -# and causes script to exit with an error code. -function absent_or_newer() { - if [ \( -e $1 \) -a \( $2 -nt $1 \) ]; then - echo "$1 is out of date" - exit -1 - fi -} - -function get_dir_of_this_script() { - # This function sets DIR to the directory in which this script itself is found. - # Thank you https://stackoverflow.com/questions/59895/how-to-get-the-source-directory-of-a-bash-script-from-within-the-script-itself - SCRIPT_SOURCE="${BASH_SOURCE[0]}" - while [ -h "$SCRIPT_SOURCE" ]; do # resolve $SCRIPT_SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" >/dev/null 2>&1 && pwd )" - SCRIPT_SOURCE="$(readlink "$SCRIPT_SOURCE")" - [[ $SCRIPT_SOURCE != /* ]] && SCRIPT_SOURCE="$DIR/$SCRIPT_SOURCE" # if $SCRIPT_SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located - done - DIR="$( cd -P "$( dirname "$SCRIPT_SOURCE" )" >/dev/null 2>&1 && pwd )" - echo 'DIR of script:' $DIR -} - -# Generate the build version based on git branch name and short commit hash and write into BUILD file -function generate_build_version() { - GIT_BRANCH_NAME=$(git branch | sed -n -e 's/^\* \(.*\)/\1/p') - GIT_SHORT_COMMIT_HASH=$(git rev-parse --short HEAD) - # Clear the old BUILD version and write the new one - truncate -s 0 ../BUILD - # Note: echo to file appends newline - echo $GIT_BRANCH_NAME:$GIT_SHORT_COMMIT_HASH >> ../BUILD - # Remmove the trailing newline character - truncate -s -1 ../BUILD - - echo "BUILD(git branch name:short commit hash): $GIT_BRANCH_NAME:$GIT_SHORT_COMMIT_HASH" -} - -# Set the version environment variable for the docker build -# Version number is from the VERSION file -# Also remove newlines and leading/trailing slashes if present in that VERSION file -function export_version() { - export INGEST_API_VERSION=$(tr -d "\n\r" < ../VERSION | xargs) - echo "INGEST_API_VERSION: $INGEST_API_VERSION" -} - - -if [[ "$1" != "check" && "$1" != "config" && "$1" != "build" && "$1" != "start" && "$1" != "stop" && "$1" != "down" ]]; then - echo "Unknown command '$1', specify one of the following: check|config|build|start|stop|down" -else - # Always show the script dir - get_dir_of_this_script - - # Always export and show the version - export_version - - # Always show the build in case branch changed or new commits - generate_build_version - - # Print empty line - echo - - if [ "$1" = "check" ]; then - # Bash array - config_paths=( - '../src/instance/app.cfg' - ) - - for pth in "${config_paths[@]}"; do - if [ ! -e $pth ]; then - echo "Missing file (relative path to DIR of script) :$pth" - exit -1 - fi - done - - absent_or_newer ingest-api/src ../src - - echo 'Checks complete, all good :)' - elif [ "$1" = "config" ]; then - docker-compose -f docker-compose.yml -f docker-compose.development.yml -p ingest-api config - elif [ "$1" = "build" ]; then - # Delete the copied source code dir if exists - if [ -d "ingest-api/src" ]; then - rm -rf ingest-api/src - fi - - # Copy over the src folder - cp -r ../src ingest-api/ - - # Delete old VERSION and BUILD files if found - if [ -f "ingest-api/VERSION" ]; then - rm -rf ingest-api/VERSION - fi - - if [ -f "ingest-api/BUILD" ]; then - rm -rf ingest-api/BUILD - fi - - # Copy over the VERSION and BUILD files - cp ../VERSION ingest-api - cp ../BUILD ingest-api - - docker-compose -f docker-compose.yml -f docker-compose.development.yml -p ingest-api build - elif [ "$1" = "start" ]; then - docker-compose -f docker-compose.yml -f docker-compose.development.yml -p ingest-api up -d - elif [ "$1" = "stop" ]; then - docker-compose -f docker-compose.yml -f docker-compose.development.yml -p ingest-api stop - elif [ "$1" = "down" ]; then - docker-compose -f docker-compose.yml -f docker-compose.development.yml -p ingest-api down - fi -fi - diff --git a/docker/example.env b/docker/example.env deleted file mode 100644 index a9e2f241..00000000 --- a/docker/example.env +++ /dev/null @@ -1 +0,0 @@ -LOG_STREAM=DEV \ No newline at end of file diff --git a/docker/ingest-api/nginx/conf.d/ingest-api.conf b/docker/ingest-api/nginx/conf.d/ingest-api.conf deleted file mode 100644 index 3f7d1bcc..00000000 --- a/docker/ingest-api/nginx/conf.d/ingest-api.conf +++ /dev/null @@ -1,46 +0,0 @@ -server { - # Only root can listen on ports below 1024, we use higher-numbered ports - # since nginx is running under non-root user hubmap - listen 8080; - - server_name localhost; - root /usr/share/nginx/html; - - # We need this logging for inspecting auth requests from other internal services - # Logging to the mounted volume for outside container access - access_log /usr/src/app/log/nginx_access_entity-api.log; - error_log /usr/src/app/log/nginx_error_entity-api.log warn; - - # Set file upload size limit to 10M, default is 1M. - client_max_body_size 10M; - - # Pass reqeusts to the uWSGI server using the "uwsgi" protocol on port 5000 - location / { - # Always enable CORS - # Response to preflight requests - if ($request_method = 'OPTIONS') { - add_header 'Access-Control-Allow-Origin' '*' always; - add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; - - # These are permitted headers to be used with the actual request - add_header 'Access-Control-Allow-Headers' 'Authorization, Cache-Control, Content-Type' always; - - # Cache the response to this preflight request in browser for the max age 86400 seconds (= 24 hours) - add_header 'Access-Control-Max-Age' 86400 always; - - # No Content - return 204; - } - - # Response to the original requests (HTTP methods are case-sensitive) with CORS enabled - if ($request_method ~ (POST|GET|PUT|DELETE)) { - add_header 'Access-Control-Allow-Origin' '*' always; - add_header 'Access-Control-Allow-Methods' 'GET, POST, PUT, DELETE, OPTIONS' always; - add_header 'Access-Control-Allow-Headers' 'Authorization, Cache-Control, Content-Type' always; - } - - include uwsgi_params; - uwsgi_pass uwsgi://localhost:5000; - } - -} diff --git a/docker/ingest-api/nginx/html/favicon.ico b/docker/ingest-api/nginx/html/favicon.ico new file mode 100644 index 00000000..d2a599ae Binary files /dev/null and b/docker/ingest-api/nginx/html/favicon.ico differ diff --git a/docker/ingest-api/nginx/nginx.conf b/docker/ingest-api/nginx/nginx.conf index 2ccd1460..58114dc5 100644 --- a/docker/ingest-api/nginx/nginx.conf +++ b/docker/ingest-api/nginx/nginx.conf @@ -1,9 +1,9 @@ -# The only change needed is to comment out the user nginx; line +# The only change needed is to comment out the user nginx; line # to avoid a warning since this directive is only meaningfull when Nginx is running as root # user nginx; # Set this as the number of CPU cores available on the machine -worker_processes 16; +worker_processes 32; error_log /var/log/nginx/error.log warn; pid /var/run/nginx.pid; @@ -15,7 +15,12 @@ events { http { - charset utf-8; + # Used by workspaces websocket connection upgrade - 5/16/2022 + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + include /etc/nginx/mime.types; default_type application/octet-stream; @@ -33,4 +38,30 @@ http { #gzip on; include /etc/nginx/conf.d/*.conf; + + # Port 80 on host maps to 8080 on container + server { + listen 8080; + root /usr/share/nginx/html; + + # We need this logging for inspecting auth requests from other internal services + # Logging to the mounted volume for outside container access + access_log /usr/src/app/log/nginx_access_ingest-api.log; + error_log /usr/src/app/log/nginx_error_ingest-api.log warn; + + # Set file upload size liit to 10M, default is 1M. + client_max_body_size 10M; + + location = /favicon.ico { + alias /usr/share/nginx/html/favicon.ico; + } + location / { + include uwsgi_params; + # Once authenticated/authorized, pass requests to the uWSGI server using the "uwsgi" protocol on port 5000 + include uwsgi_params; + uwsgi_read_timeout 300; + # We have to use localhost because the ingest API is running on the same container + uwsgi_pass uwsgi://localhost:5000; + } + } } diff --git a/src/uwsgi.ini b/src/uwsgi.ini index 83732add..99ceb7a4 100644 --- a/src/uwsgi.ini +++ b/src/uwsgi.ini @@ -1,8 +1,4 @@ [uwsgi] -# Need this plugin since uwsgi and uwsgi python plugin are installed with yum -# If uwsgi installed with pip, no need this -plugin = python36 - # So uwsgi knows where to mount the app chdir = /usr/src/app/src