Merge pull request #352 from isb-cgc/prod-sp

API v4

.circleci/config.yml

@@ -1,12 +1,84 @@
-version: 2
+version: 2.1
+commands:
+  deployment_setup:
+    steps:
+      - run:
+          name: "Set the Tier"
+          command: |
+            TIER=DEV
+            if [[ ${CIRCLE_BRANCH} =~ (prod|uat|test).* ]]; then
+              TIER=$(awk -F[\-_] '{print toupper($1)}' \<<< ${CIRCLE_BRANCH})
+            fi
+            echo "export TIER=${TIER}" >> $BASH_ENV
+            echo "Tier was identified as ${TIER} for branch ${CIRCLE_BRANCH}"
+      - run:
+          name: "Assign Project-level vars"
+          command: |
+            if [ ! -f deployment.key.json ]; then
+              echo "Deployment JSON keyfile not found - loading from CircleCI."
+              KEY=${DEPLOYMENT_KEY_ISB_CGC}
+            else
+              echo "Deployment JSON keyfile found."
+              KEY="NA"
+            fi
+            CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_ISB_CGC}
+            PROJECT_ID=${DEPLOYMENT_PROJECT_ID_ISB_CGC}
+            BUCKET=${DEPLOYMENT_BUCKET_ISB_CGC}
+            if [[ ${TIER} == "PROD" ]] || [[ ${TIER} == "DEV" ]]; then
+              BUCKET="${BUCKET}/${TIER,,}"
+              echo "Using default project ${PROJECT_ID} and related deployment SA and bucket."
+            elif [[ ${TIER} == "UAT" ]]; then
+              KEY=${DEPLOYMENT_KEY_ISB_CGC_UAT}
+              CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_ISB_CGC_UAT}
+              PROJECT_ID=${DEPLOYMENT_PROJECT_ID_ISB_CGC_UAT}
+              BUCKET=${DEPLOYMENT_BUCKET_ISB_CGC_UAT}
+            elif [[ ${TIER} == "TEST" ]]; then
+              KEY=${DEPLOYMENT_KEY_ISB_CGC_TEST}
+              CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_ISB_CGC_TEST}
+              PROJECT_ID=${DEPLOYMENT_PROJECT_ID_ISB_CGC_TEST}
+              BUCKET=${DEPLOYMENT_BUCKET_ISB_CGC_TEST}
+            else
+              echo "[ERROR] - Unrecognized tier: ${TIER} - exitng."
+              exit 1
+            fi
+
+            echo "export DEPLOYMENT_KEY=\"${KEY}\"" >> $BASH_ENV
+            echo "export DEPLOYMENT_CLIENT_EMAIL=${CLIENT_EMAIL}" >> $BASH_ENV
+            echo "export DEPLOYMENT_PROJECT_ID=${PROJECT_ID}" >> $BASH_ENV
+            echo "export DEPLOYMENT_BUCKET=${BUCKET}" >> $BASH_ENV
+
+  deployment_config:
+    steps:
+      - run:
+          name: "Set tier-specific configuration file"
+          command: |
+            sudo -E /bin/bash ./shell/pull_config.sh
+            xargs -a deployment_config.txt -I{} echo "export {}" >> $BASH_ENV
+
+  auth:
+    steps:
+      - run:
+          name: "Service Account Auth and Project Settings"
+          command: |
+            sudo -E /bin/bash ./shell/gcloud_authenticate.sh
+
+  install_cloud_sdk:
+    steps:
+      - run:
+          name: "Install CloudSDK"
+          command: |
+            echo "export CLOUDSDK_CORE_DISABLE_PROMPTS=1" >> $BASH_ENV
+            echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+            curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+            sudo apt-get update && sudo apt-get -y --allow-downgrades install google-cloud-sdk=251.0.0-0
+
 jobs:
   build_job:
     environment:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
-    name: Building Deployment Container
     working_directory: ~/ISB-CGC-API
     docker:
-      - image: circleci/python:2.7.14
+      - image: circleci/python:3.7.0-stretch
       - image: circleci/mysql:5.7
         environment:
           MYSQL_ROOT_HOST: "%"
@@ -33,30 +105,22 @@ jobs:
           key: isb-cgc-api-lib-{{ checksum "requirements.txt" }}
           paths:
             - ./lib
-      - save_cache:
-          key: isb-cgc-api-google-deps-{{ checksum "./shell/install-deps.sh" }}
-          paths:
-            - ./google-cloud-sdk
-            - ./google_appengine
+      - deployment_setup
+      - auth
+      - deployment_config
       - run:
-          name: Auth and Staging
+          name: Staging
           command: |
-            sudo -E /bin/sh ./shell/gcloud_authenticate.sh
             sudo -E /bin/bash ./shell/gcloud-pull-staging-files.sh
       - save_cache:
           key: isb-cgc-api-{{ epoch }}
           paths:
-            - ./api
-            - ./api_3
-            - ./bq_data_access
+            - ./apiv4
             - ./shell
             - ./ISB-CGC-Common
-            - ./saml
-            - ./lib
-            - ./appengine_config.py
-            - ./cgc_api.py
+            - ./Dockerfile
             - ./app.yaml
-            - ./manage.py
+            - ./openapi-appengine.yaml
             - ./settings.py
             - ./txt
             - ./json
@@ -65,9 +129,8 @@ jobs:
   deploy_job:
     environment:
       TZ: "/usr/share/zoneinfo/America/Los_Angeles"
-    name: Deploy to AppEngine Standard
     docker:
-      - image: circleci/python:2.7.14
+      - image: circleci/python:3.7.0-stretch
     working_directory: ~/ISB-CGC-API
     steps:
       - restore_cache:
@@ -76,11 +139,16 @@ jobs:
       - restore_cache:
           keys:
             - isb-cgc-api-google-deps-
+      - install_cloud_sdk
+      - deployment_setup
+      - auth
+      - deployment_config
       - deploy:
+          nane: Deployment
           command: |
-            sudo -E /bin/bash ./shell/gcloud_authenticate.sh
             sudo -E /bin/bash ./shell/unpack_for_deployment.sh
-            sudo -E ./google-cloud-sdk/bin/gcloud app deploy --verbosity=debug ./app.yaml --quiet
+            sudo -E gcloud endpoints services deploy ./openapi-appengine.yaml
+            sudo -E gcloud app deploy --verbosity=debug ./app.yaml --quiet
 workflows:
   version: 2
   build_and_deploy:
@@ -91,4 +159,8 @@ workflows:
             - build_job
           filters:
             branches:
-              only: prod
+              only:
+                - master
+                - test
+                - uat
+                - prod

.gitignore

@@ -73,19 +73,12 @@ target/
 .sass-cache
 #*
 *.json
-!isb_cgc_apiv3_openapiv2.json
-!isb_cgc_ccle_apiv3_openapiv2.json
-!isb_cgc_target_apiv3_openapiv2.json
-!isb_cgc_tcga_apiv3_openapiv2.json
-!isb_cgc_apiv3_openapiv3.json
-!isb_cgc_ccle_apiv3_openapiv3.json
-!isb_cgc_target_apiv3_openapiv3.json
-!isb_cgc_tcga_apiv3_openapiv3.json
 *.patch
 .env
 *.env
-scripts/metadata_featdef_tables.sql
+openapi-appengine.yaml
 app.yaml
+scripts/metadata_featdef_tables.sql
 
 # dummy datasets
 *.tsv

Dockerfile

@@ -0,0 +1,81 @@
+###
+#
+# Copyright 2019, Institute for Systems Biology
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+###
+
+# Dockerfile extending the generic Python image with application files for a
+# single application.
+FROM gcr.io/google_appengine/python
+
+# Create a virtualenv for dependencies. This isolates these packages from
+# system-level packages.
+# Use -p python3 or -p python3.7 to select python version. Default is version 2.
+RUN virtualenv /env -p python3
+
+# Setting these environment variables are the same as running
+# source /env/bin/activate.
+ENV VIRTUAL_ENV /env
+ENV PATH /env/bin:$PATH
+
+RUN apt-get update
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get install -y wget
+RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.9-1_all.deb" -P /tmp
+
+# install lsb-release (a dependency of mysql-apt-config), since dpkg doesn't
+# do dependency resolution
+RUN apt-get install -y lsb-release
+# add a debconf entry to select mysql-5.7 as the server version when we install
+# the mysql config package
+RUN echo "mysql-apt-config mysql-apt-config/select-server select mysql-5.7" | debconf-set-selections
+# having 'selected' mysql-5.7 for 'server', install the mysql config package
+RUN echo 'download mysql public build key'
+RUN apt-key del 1550412832
+RUN wget -O - -q 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x8C718D3B5072E1F5' | grep -v '>' | grep -v '<' | grep -v '{' > mysql_pubkey.asc
+RUN apt-key add mysql_pubkey.asc || exit 1
+RUN dpkg --install /tmp/mysql-apt-config_0.8.9-1_all.deb
+
+# fetch the updated package metadata (in particular, mysql-server-5.7)
+RUN apt-get update
+
+# aaaand now let's install mysql-server
+RUN apt-get install -y mysql-server
+
+# Get pip3 installed
+RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3
+
+RUN apt-get -y install build-essential
+RUN apt-get -y install --reinstall python-m2crypto python3-crypto
+RUN apt-get -y install libxml2-dev libxmlsec1-dev swig
+RUN pip3 install pexpect
+
+RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl
+RUN easy_install -U distribute
+
+ADD . /app
+
+# We need to recompile some of the items because of differences in compiler versions
+RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade
+RUN pip3 install gunicorn==19.9.0
+
+# Install Cloud SDK
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && apt-get install google-cloud-sdk -y
+# Install the Python Libraries
+RUN apt-get -y install google-cloud-sdk-app-engine-python
+
+ENV PYTHONPATH=/app:/app/apiv4:/app/lib:/app/ISB-CGC-Common:${PYTHONPATH}
+
+CMD gunicorn -b :$PORT apiv4:app -w 3 -t 130

Vagrantfile

@@ -0,0 +1,20 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure(2) do |config|
+    config.vm.box_url = "https://app.vagrantup.com/ubuntu/boxes/xenial64"
+    config.vm.box = "ubuntu/xenial64"
+
+    # API ports
+    config.vm.network "forwarded_port", guest: 8090, host: 8090
+    config.vm.network "forwarded_port", guest: 9000, host: 9000
+
+    config.vm.synced_folder ".", "/home/vagrant/API"
+    config.vm.synced_folder "../", "/home/vagrant/parentDir"
+
+    # Map Common and lib for API
+    config.vm.synced_folder "../ISB-CGC-Common", "/home/vagrant/API/ISB-CGC-Common"
+
+    config.vm.provision "shell", path: 'shell/install-deps.sh'
+    config.vm.provision "shell", path: 'shell/vagrant-start-server.sh'
+    config.vm.provision "shell", path: 'shell/vagrant-set-env.sh'
+end