From 2c8fb3abd4e0f30346728a9370ca7e3a97a9c70f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 2 Jan 2019 13:17:59 -0800 Subject: [PATCH 001/323] -> Move API to its own build --- Vagrantfile | 35 +++++++++++++++++++++++++++ shell/gcloud-pull-staging-files.sh | 7 ------ shell/install-deps.sh | 39 ++++++++++++++++-------------- shell/python-su.sh | 1 + shell/vagrant-set-env.sh | 2 ++ shell/vagrant-start-server.sh | 1 + 6 files changed, 60 insertions(+), 25 deletions(-) create mode 100644 Vagrantfile create mode 100644 shell/python-su.sh create mode 100644 shell/vagrant-set-env.sh create mode 100644 shell/vagrant-start-server.sh diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..4989f824 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,35 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure(2) do |config| + required_plugins = %w( vagrant-vbguest vagrant-disksize ) + _retry = false + required_plugins.each do |plugin| + unless Vagrant.has_plugin? plugin + system "vagrant plugin install #{plugin}" + _retry=true + end + end + + if (_retry) + exec "vagrant " + ARGV.join(' ') + end + + config.vm.box_url = "https://app.vagrantup.com/debian/boxes/stretch64" + config.vm.box = "debian/stretch64" + config.disksize.size = "40GB" + + # API ports + config.vm.network "forwarded_port", guest: 8090, host: 8090 + config.vm.network "forwarded_port", guest: 9000, host: 9000 + + config.vm.synced_folder ".", "/home/vagrant/API" + config.vm.synced_folder "../", "/home/vagrant/parentDir" + + # Map Common and lib for API + config.vm.synced_folder "../ISB-CGC-Common", "/home/vagrant/API/ISB-CGC-Common" + config.vm.synced_folder "../secure_files", "/home/vagrant/API/secure_files" + + config.vm.provision "shell", path: 'shell/install-deps.sh' + config.vm.provision "shell", path: 'shell/vagrant-start-server.sh' + config.vm.provision "shell", path: 'shell/vagrant-set-env.sh' +end \ No newline at end of file diff --git a/shell/gcloud-pull-staging-files.sh b/shell/gcloud-pull-staging-files.sh index b7c67101..bc44cd4c 100644 --- a/shell/gcloud-pull-staging-files.sh +++ b/shell/gcloud-pull-staging-files.sh @@ -10,13 +10,6 @@ mkdir ./txt ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${MANAGED_SERVICE_ACCOUNTS_JSON_FILE}" ./ ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${DEV_DATASET_JSON_FILE}" ./ -if [ -n "${DEV_NIH_AUTH_ON}" ]; then - ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/saml/advanced_settings.json" ./saml/advanced_settings.json - ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/saml/settings.json" ./saml/settings.json - ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/saml/certs/cert.pem" ./saml/certs/cert.pem - ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/saml/certs/key.pem" ./saml/certs/key.pem - ./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/NIH_FTP.txt" ./NIH_FTP.txt -fi # Pack staged files for caching cp --verbose *.json ./json diff --git a/shell/install-deps.sh b/shell/install-deps.sh index 76728625..c015be47 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -4,40 +4,44 @@ if [ -n "$CI" ]; then # Clone dependencies git clone -b master https://github.com/isb-cgc/ISB-CGC-Common.git else - export $(cat /home/vagrant/www/.env | grep -v ^# | xargs) 2> /dev/null + export $(cat /home/vagrant/API/.env | grep -v ^# | xargs) 2> /dev/null export HOME=/home/vagrant - export HOMEROOT=/home/vagrant/www + export HOMEROOT=/home/vagrant/API fi +export DEBIAN_FRONTEND=noninteractive + # Install and update apt-get info echo "Preparing System..." apt-get -y install software-properties-common +wget https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb +apt-get install -y lsb-release +dpkg -i mysql-apt-config_0.8.9-1_all.deb -if [ -n "$CI" ]; then - # CI Takes care of Python update - apt-get update -qq -else - # Add apt-get repository to update python from 2.7.6 (default) to latest 2.7.x - add-apt-repository -y ppa:jonathonf/python-2.7 - apt-get update -qq - apt-get install -qq -y python2.7 -fi +apt-get update -qq # Install apt-get dependencies echo "Installing Dependencies..." -apt-get install -qq -y unzip libffi-dev libssl-dev mysql-client libmysqlclient-dev python2.7-dev git -echo "Dependencies Installed " +apt-get install -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl dos2unix python3.5 +apt-get install -y --force-yes mysql-client +echo "Dependencies Installed" + +# If this is local development, clean out lib for a re-structuring +if [ -z "${CI}" ]; then + # Clean out lib to prevent confusion over multiple builds in local development + # and prep for local install + echo "Emptying out ${HOMEROOT}/lib/ ..." +fi # Install PIP + Dependencies -echo "Installing PIP..." -curl --silent https://bootstrap.pypa.io/get-pip.py | python -echo "...PIP installed." +echo "Installing pip3..." +curl --silent https://bootstrap.pypa.io/get-pip.py | python3 # Install our primary python libraries # If we're not on CircleCI, or we are but the lib directory isn't there (cache miss), install lib if [ -z "${CI}" ] || [ ! -d "lib" ]; then echo "Installing Python Libraries..." - pip install -q -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all + pip3 install -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all else echo "Using restored cache for Python Libraries" fi @@ -66,4 +70,3 @@ if [ -z "${CI}" ] || [ ! -d "google-cloud-sdk" ]; then else echo "Using restored cache for Google Cloud SDK." fi - diff --git a/shell/python-su.sh b/shell/python-su.sh new file mode 100644 index 00000000..8390271a --- /dev/null +++ b/shell/python-su.sh @@ -0,0 +1 @@ +sudo PYTHONPATH="$PYTHONPATH" /usr/bin/python3 "$@" \ No newline at end of file diff --git a/shell/vagrant-set-env.sh b/shell/vagrant-set-env.sh new file mode 100644 index 00000000..a88b6c18 --- /dev/null +++ b/shell/vagrant-set-env.sh @@ -0,0 +1,2 @@ +echo 'export PYTHONPATH=/home/vagrant/API:/home/vagrant/google_appengine:/home/vagrant/google_appengine/lib/protorpc-1.0:/home/vagrant/API/lib' | tee -a /home/vagrant/.bash_profile +chmod +x /home/vagrant/API/shell/python-su.sh \ No newline at end of file diff --git a/shell/vagrant-start-server.sh b/shell/vagrant-start-server.sh new file mode 100644 index 00000000..860fe309 --- /dev/null +++ b/shell/vagrant-start-server.sh @@ -0,0 +1 @@ +export $(cat /home/vagrant/parentDir/secure_files/.env | grep -v ^# | xargs) 2> /dev/null From 05acd1496fe281725d47272116799c5fec355483 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 25 Feb 2019 14:35:22 -0800 Subject: [PATCH 002/323] -> New API version --- api/Cohort.py | 1857 -------- api/__init__.py | 0 api/api_helpers.py | 508 -- api/data_access.py | 542 --- api/feature_access.py | 176 - api/isb_cgc_api/__init__.py | 0 api/isb_cgc_api/aliquots_annotations.py | 149 - .../cohorts_cloudstoragefilepaths.py | 115 - api/isb_cgc_api/cohorts_create.py | 174 - api/isb_cgc_api/cohorts_delete.py | 89 - api/isb_cgc_api/cohorts_get.py | 161 - api/isb_cgc_api/cohorts_googlegenomics.py | 127 - api/isb_cgc_api/cohorts_list.py | 157 - api/isb_cgc_api/cohorts_preview.py | 98 - api/isb_cgc_api/isb_cgc_api_helpers.py | 393 -- api/isb_cgc_api/message_classes.py | 366 -- api/isb_cgc_api/message_generator.py | 134 - api/isb_cgc_api/patients_annotations.py | 144 - api/isb_cgc_api/patients_get.py | 114 - api/isb_cgc_api/samples_annotations.py | 145 - .../samples_cloudstoragefilepaths.py | 90 - api/isb_cgc_api/samples_get.py | 214 - api/isb_cgc_api/samples_googlegenomics.py | 96 - api/isb_cgc_api/users_get.py | 143 - api/maf.py | 104 - api/metadata.py | 2754 ----------- api/pairwise.py | 144 - api/pairwise_api.py | 166 - api/schema/__init__.py | 0 api/schema/tcga_clinical.py | 283 -- api/seqpeek_api.py | 94 - api/seqpeek_view_api.py | 238 - api/single_feature_access.py | 131 - api_3/README.md | 6 - api_3/__init__.py | 0 api_3/api_helpers.py | 483 -- api_3/cloudstoragefilepaths_helper.py | 212 - api_3/cohort_create_preview_helper.py | 396 -- api_3/cohort_endpoint_helpers.py | 144 - api_3/data_access.py | 542 --- api_3/feature_access.py | 176 - api_3/isb_cgc_api/__init__.py | 0 api_3/isb_cgc_api/cohort_file_manifest.py | 198 - api_3/isb_cgc_api/cohort_get_list_helper.py | 408 -- .../cohorts_cloudstoragefilepaths.py | 68 - api_3/isb_cgc_api/cohorts_delete.py | 85 - api_3/isb_cgc_api/cohorts_get.py | 30 - api_3/isb_cgc_api/cohorts_list.py | 32 - api_3/isb_cgc_api/files_get_file_paths.py | 68 - api_3/isb_cgc_api/isb_cgc_api_helpers.py | 27 - api_3/isb_cgc_api_CCLE/__init__.py | 0 .../allowed_values_v3_CCLE.json | 243 - api_3/isb_cgc_api_CCLE/cohorts_create.py | 37 - api_3/isb_cgc_api_CCLE/cohorts_preview.py | 37 - api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py | 28 - api_3/isb_cgc_api_CCLE/message_classes.py | 111 - api_3/isb_cgc_api_CCLE/patients_get.py | 55 - .../samples_cloudstoragefilepaths.py | 33 - api_3/isb_cgc_api_CCLE/samples_get.py | 59 - .../samples_googlegenomics.py | 96 - api_3/isb_cgc_api_TARGET/__init__.py | 0 .../allowed_values_v3_TARGET.json | 243 - api_3/isb_cgc_api_TARGET/cohorts_create.py | 37 - api_3/isb_cgc_api_TARGET/cohorts_preview.py | 37 - .../isb_cgc_api_TARGET/isb_cgc_api_helpers.py | 28 - api_3/isb_cgc_api_TARGET/message_classes.py | 210 - api_3/isb_cgc_api_TARGET/patients_get.py | 55 - .../samples_cloudstoragefilepaths.py | 33 - api_3/isb_cgc_api_TARGET/samples_get.py | 60 - .../samples_googlegenomics.py | 96 - api_3/isb_cgc_api_TARGET/users_get.py | 32 - api_3/isb_cgc_api_TCGA/__init__.py | 0 .../isb_cgc_api_TCGA/aliquots_annotations.py | 72 - .../allowed_values_v3_TCGA.json | 835 ---- api_3/isb_cgc_api_TCGA/annotations_api.py | 100 - api_3/isb_cgc_api_TCGA/cohorts_create.py | 37 - api_3/isb_cgc_api_TCGA/cohorts_preview.py | 38 - api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py | 28 - api_3/isb_cgc_api_TCGA/message_classes.py | 532 --- .../isb_cgc_api_TCGA/patients_annotations.py | 72 - api_3/isb_cgc_api_TCGA/patients_get.py | 55 - api_3/isb_cgc_api_TCGA/samples_annotations.py | 74 - .../samples_cloudstoragefilepaths.py | 33 - api_3/isb_cgc_api_TCGA/samples_get.py | 60 - .../samples_googlegenomics.py | 96 - api_3/isb_cgc_api_TCGA/users_get.py | 32 - api_3/maf.py | 104 - api_3/message_generator.py | 243 - api_3/metadata.py | 1915 -------- api_3/pairwise.py | 144 - api_3/pairwise_api.py | 166 - api_3/patients_get_helper.py | 203 - api_3/samples_get_helper.py | 345 -- api_3/schema/__init__.py | 0 api_3/schema/tcga_clinical.py | 283 -- api_3/seqpeek_api.py | 94 - api_3/seqpeek_view_api.py | 238 - api_3/single_feature_access.py | 131 - api_3/test/__init__.py | 0 api_3/users_get_common.py | 102 - appengine_config.py | 3 - bq_data_access/__init__.py | 0 bq_data_access/clinical_data.py | 190 - bq_data_access/cohort_bigquery.py | 141 - bq_data_access/cohort_cloudsql.py | 152 - bq_data_access/cohort_utils.py | 61 - bq_data_access/copynumber_data.py | 164 - bq_data_access/data_access.py | 436 -- bq_data_access/errors.py | 28 - bq_data_access/feature_data_provider.py | 143 - bq_data_access/feature_search/__init__.py | 0 .../feature_search/clinical_searcher.py | 92 - bq_data_access/feature_search/common.py | 41 - .../feature_search/copynumber_search.py | 145 - .../feature_search/gexp_searcher.py | 155 - .../feature_search/gnab_searcher.py | 145 - .../feature_search/methylation_searcher.py | 167 - .../feature_search/microrna_searcher.py | 149 - bq_data_access/feature_search/mirna.py | 74 - bq_data_access/feature_search/mutation.py | 139 - bq_data_access/feature_search/protein.py | 141 - bq_data_access/feature_search/util.py | 49 - bq_data_access/feature_value_types.py | 101 - bq_data_access/gexp_data.py | 192 - bq_data_access/gnab_data.py | 176 - bq_data_access/maf_data.py | 233 - bq_data_access/metadata.py | 50 - bq_data_access/methylation_data.py | 295 -- bq_data_access/mirna_data.py | 228 - bq_data_access/mrna_data.py | 186 - bq_data_access/protein_data.py | 168 - bq_data_access/seqpeek/__init__.py | 0 bq_data_access/seqpeek/seqpeek_interpro.py | 72 - .../seqpeek/seqpeek_maf_formatter.py | 85 - bq_data_access/seqpeek/seqpeek_view.py | 268 -- bq_data_access/seqpeek_maf_data.py | 85 - bq_data_access/user_data.py | 456 -- bq_data_access/utils.py | 141 - cgc_api.py | 82 - isb_cgc_apiv3_openapiv2.json | 500 -- isb_cgc_apiv3_openapiv3.json | 571 --- isb_cgc_ccle_apiv3_openapiv2.json | 4083 ---------------- isb_cgc_ccle_apiv3_openapiv3.json | 4140 ----------------- isb_cgc_target_apiv3_openapiv2.json | 3612 -------------- isb_cgc_target_apiv3_openapiv3.json | 3672 --------------- isb_cgc_tcga_apiv3_openapiv2.json | 2819 ----------- isb_cgc_tcga_apiv3_openapiv3.json | 2907 ------------ main.py | 77 + requirements.txt | 3 +- 149 files changed, 79 insertions(+), 48081 deletions(-) delete mode 100644 api/Cohort.py delete mode 100755 api/__init__.py delete mode 100755 api/api_helpers.py delete mode 100755 api/data_access.py delete mode 100755 api/feature_access.py delete mode 100644 api/isb_cgc_api/__init__.py delete mode 100644 api/isb_cgc_api/aliquots_annotations.py delete mode 100644 api/isb_cgc_api/cohorts_cloudstoragefilepaths.py delete mode 100644 api/isb_cgc_api/cohorts_create.py delete mode 100644 api/isb_cgc_api/cohorts_delete.py delete mode 100644 api/isb_cgc_api/cohorts_get.py delete mode 100644 api/isb_cgc_api/cohorts_googlegenomics.py delete mode 100644 api/isb_cgc_api/cohorts_list.py delete mode 100644 api/isb_cgc_api/cohorts_preview.py delete mode 100644 api/isb_cgc_api/isb_cgc_api_helpers.py delete mode 100644 api/isb_cgc_api/message_classes.py delete mode 100644 api/isb_cgc_api/message_generator.py delete mode 100644 api/isb_cgc_api/patients_annotations.py delete mode 100644 api/isb_cgc_api/patients_get.py delete mode 100644 api/isb_cgc_api/samples_annotations.py delete mode 100644 api/isb_cgc_api/samples_cloudstoragefilepaths.py delete mode 100644 api/isb_cgc_api/samples_get.py delete mode 100644 api/isb_cgc_api/samples_googlegenomics.py delete mode 100644 api/isb_cgc_api/users_get.py delete mode 100755 api/maf.py delete mode 100755 api/metadata.py delete mode 100755 api/pairwise.py delete mode 100644 api/pairwise_api.py delete mode 100755 api/schema/__init__.py delete mode 100755 api/schema/tcga_clinical.py delete mode 100644 api/seqpeek_api.py delete mode 100644 api/seqpeek_view_api.py delete mode 100644 api/single_feature_access.py delete mode 100755 api_3/README.md delete mode 100755 api_3/__init__.py delete mode 100755 api_3/api_helpers.py delete mode 100755 api_3/cloudstoragefilepaths_helper.py delete mode 100755 api_3/cohort_create_preview_helper.py delete mode 100755 api_3/cohort_endpoint_helpers.py delete mode 100755 api_3/data_access.py delete mode 100755 api_3/feature_access.py delete mode 100755 api_3/isb_cgc_api/__init__.py delete mode 100644 api_3/isb_cgc_api/cohort_file_manifest.py delete mode 100755 api_3/isb_cgc_api/cohort_get_list_helper.py delete mode 100755 api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py delete mode 100755 api_3/isb_cgc_api/cohorts_delete.py delete mode 100755 api_3/isb_cgc_api/cohorts_get.py delete mode 100755 api_3/isb_cgc_api/cohorts_list.py delete mode 100755 api_3/isb_cgc_api/files_get_file_paths.py delete mode 100755 api_3/isb_cgc_api/isb_cgc_api_helpers.py delete mode 100755 api_3/isb_cgc_api_CCLE/__init__.py delete mode 100755 api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json delete mode 100755 api_3/isb_cgc_api_CCLE/cohorts_create.py delete mode 100755 api_3/isb_cgc_api_CCLE/cohorts_preview.py delete mode 100755 api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py delete mode 100755 api_3/isb_cgc_api_CCLE/message_classes.py delete mode 100755 api_3/isb_cgc_api_CCLE/patients_get.py delete mode 100755 api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py delete mode 100755 api_3/isb_cgc_api_CCLE/samples_get.py delete mode 100755 api_3/isb_cgc_api_CCLE/samples_googlegenomics.py delete mode 100755 api_3/isb_cgc_api_TARGET/__init__.py delete mode 100755 api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json delete mode 100755 api_3/isb_cgc_api_TARGET/cohorts_create.py delete mode 100755 api_3/isb_cgc_api_TARGET/cohorts_preview.py delete mode 100755 api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py delete mode 100755 api_3/isb_cgc_api_TARGET/message_classes.py delete mode 100755 api_3/isb_cgc_api_TARGET/patients_get.py delete mode 100755 api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py delete mode 100755 api_3/isb_cgc_api_TARGET/samples_get.py delete mode 100755 api_3/isb_cgc_api_TARGET/samples_googlegenomics.py delete mode 100755 api_3/isb_cgc_api_TARGET/users_get.py delete mode 100644 api_3/isb_cgc_api_TCGA/__init__.py delete mode 100644 api_3/isb_cgc_api_TCGA/aliquots_annotations.py delete mode 100755 api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json delete mode 100755 api_3/isb_cgc_api_TCGA/annotations_api.py delete mode 100644 api_3/isb_cgc_api_TCGA/cohorts_create.py delete mode 100644 api_3/isb_cgc_api_TCGA/cohorts_preview.py delete mode 100644 api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py delete mode 100644 api_3/isb_cgc_api_TCGA/message_classes.py delete mode 100644 api_3/isb_cgc_api_TCGA/patients_annotations.py delete mode 100644 api_3/isb_cgc_api_TCGA/patients_get.py delete mode 100644 api_3/isb_cgc_api_TCGA/samples_annotations.py delete mode 100644 api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py delete mode 100644 api_3/isb_cgc_api_TCGA/samples_get.py delete mode 100644 api_3/isb_cgc_api_TCGA/samples_googlegenomics.py delete mode 100644 api_3/isb_cgc_api_TCGA/users_get.py delete mode 100755 api_3/maf.py delete mode 100755 api_3/message_generator.py delete mode 100755 api_3/metadata.py delete mode 100755 api_3/pairwise.py delete mode 100644 api_3/pairwise_api.py delete mode 100755 api_3/patients_get_helper.py delete mode 100755 api_3/samples_get_helper.py delete mode 100755 api_3/schema/__init__.py delete mode 100755 api_3/schema/tcga_clinical.py delete mode 100644 api_3/seqpeek_api.py delete mode 100644 api_3/seqpeek_view_api.py delete mode 100644 api_3/single_feature_access.py delete mode 100755 api_3/test/__init__.py delete mode 100755 api_3/users_get_common.py delete mode 100755 bq_data_access/__init__.py delete mode 100755 bq_data_access/clinical_data.py delete mode 100755 bq_data_access/cohort_bigquery.py delete mode 100755 bq_data_access/cohort_cloudsql.py delete mode 100755 bq_data_access/cohort_utils.py delete mode 100755 bq_data_access/copynumber_data.py delete mode 100755 bq_data_access/data_access.py delete mode 100644 bq_data_access/errors.py delete mode 100644 bq_data_access/feature_data_provider.py delete mode 100755 bq_data_access/feature_search/__init__.py delete mode 100755 bq_data_access/feature_search/clinical_searcher.py delete mode 100755 bq_data_access/feature_search/common.py delete mode 100644 bq_data_access/feature_search/copynumber_search.py delete mode 100755 bq_data_access/feature_search/gexp_searcher.py delete mode 100644 bq_data_access/feature_search/gnab_searcher.py delete mode 100644 bq_data_access/feature_search/methylation_searcher.py delete mode 100644 bq_data_access/feature_search/microrna_searcher.py delete mode 100644 bq_data_access/feature_search/mirna.py delete mode 100644 bq_data_access/feature_search/mutation.py delete mode 100644 bq_data_access/feature_search/protein.py delete mode 100755 bq_data_access/feature_search/util.py delete mode 100755 bq_data_access/feature_value_types.py delete mode 100755 bq_data_access/gexp_data.py delete mode 100755 bq_data_access/gnab_data.py delete mode 100755 bq_data_access/maf_data.py delete mode 100755 bq_data_access/metadata.py delete mode 100755 bq_data_access/methylation_data.py delete mode 100755 bq_data_access/mirna_data.py delete mode 100755 bq_data_access/mrna_data.py delete mode 100755 bq_data_access/protein_data.py delete mode 100644 bq_data_access/seqpeek/__init__.py delete mode 100644 bq_data_access/seqpeek/seqpeek_interpro.py delete mode 100644 bq_data_access/seqpeek/seqpeek_maf_formatter.py delete mode 100644 bq_data_access/seqpeek/seqpeek_view.py delete mode 100755 bq_data_access/seqpeek_maf_data.py delete mode 100644 bq_data_access/user_data.py delete mode 100755 bq_data_access/utils.py delete mode 100644 cgc_api.py delete mode 100644 isb_cgc_apiv3_openapiv2.json delete mode 100644 isb_cgc_apiv3_openapiv3.json delete mode 100644 isb_cgc_ccle_apiv3_openapiv2.json delete mode 100644 isb_cgc_ccle_apiv3_openapiv3.json delete mode 100644 isb_cgc_target_apiv3_openapiv2.json delete mode 100644 isb_cgc_target_apiv3_openapiv3.json delete mode 100644 isb_cgc_tcga_apiv3_openapiv2.json delete mode 100644 isb_cgc_tcga_apiv3_openapiv3.json create mode 100644 main.py diff --git a/api/Cohort.py b/api/Cohort.py deleted file mode 100644 index e0a808e3..00000000 --- a/api/Cohort.py +++ /dev/null @@ -1,1857 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import re -import collections -from datetime import datetime -import endpoints -from protorpc import messages, message_types -from protorpc import remote -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -import django -import MySQLdb -import json -from metadata import MetadataItem -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms, Samples, Filters -from bq_data_access.cohort_bigquery import BigQueryCohortSupport -from api_helpers import * - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID -BASE_URL = settings.BASE_URL - -DEFAULT_COHORT_NAME = 'Untitled Cohort' - -IMPORTANT_FEATURES = [ - 'tumor_tissue_site', - 'gender', - 'vital_status', - 'country', - 'Study', - 'age_at_initial_pathologic_diagnosis', - 'TP53', - 'RB1', - 'NF1', - 'APC', - 'CTNNB1', - 'PIK3CA', - 'PTEN', - 'FBXW7', - 'NRAS', - 'ARID1A', - 'CDKN2A', - 'SMAD4', - 'BRAF', - 'NFE2L2', - 'IDH1', - 'PIK3R1', - 'HRAS', - 'EGFR', - 'BAP1', - 'KRAS', - 'DNAseq_data', - 'mirnPlatform', - 'cnvrPlatform', - 'methPlatform', - 'gexpPlatform', - 'rppaPlatform' -] - -BUILTIN_ENDPOINTS_PARAMETERS = [ - 'alt', - 'fields', - 'enum', - 'enumDescriptions', - 'key', - 'oauth_token', - 'prettyPrint', - 'quotaUser', - 'userIp' -] - -MAX_FILTER_VALUE_LENGTH = 496 # max_len of Filter.value is 512, but 507 is too long. 495 is ok. - - -class ReturnJSON(messages.Message): - msg = messages.StringField(1) - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -class Cohort(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - perm = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.IntegerField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - num_patients = messages.StringField(11) - num_samples = messages.StringField(12) - - -class CohortsList(messages.Message): - items = messages.MessageField(Cohort, 1, repeated=True) - count = messages.IntegerField(2) - - -class CohortPatientsSamplesList(messages.Message): - patients = messages.StringField(1, repeated=True) - patient_count = messages.IntegerField(2) - samples = messages.StringField(3, repeated=True) - sample_count = messages.IntegerField(4) - cohort_id = messages.IntegerField(5) - - -class PatientDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - - -class DataDetails(messages.Message): - sample_barcode = messages.StringField(1) - DataCenterName = messages.StringField(2) - DataCenterType = messages.StringField(3) - DataFileName = messages.StringField(4) - DataFileNameKey = messages.StringField(5) - DatafileUploaded = messages.StringField(6) - DataLevel = messages.StringField(7) - Datatype = messages.StringField(8) - GenomeReference = messages.StringField(9) - GG_dataset_id = messages.StringField(10) - GG_readgroupset_id = messages.StringField(11) - Pipeline = messages.StringField(12) - Platform = messages.StringField(13) - platform_full_name = messages.StringField(14) - Project = messages.StringField(15) - Repository = messages.StringField(16) - SDRFFileName = messages.StringField(17) - SecurityProtocol = messages.StringField(18) - CloudStoragePath = messages.StringField(19) - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(MetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - patient = messages.StringField(3) - data_details = messages.MessageField(DataDetails, 4, repeated=True) - data_details_count = messages.IntegerField(5) - error = messages.StringField(6) - - -class DataFileNameKeyList(messages.Message): - datafilenamekeys = messages.StringField(1, repeated=True) - count = messages.IntegerField(2) - - -class GoogleGenomicsItem(messages.Message): - sample_barcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomicsItem, 1, repeated=True) - count = messages.IntegerField(2) - - -class MetadataRangesItem(messages.Message): - age_at_initial_pathologic_diagnosis = messages.IntegerField(1, repeated=True) - age_at_initial_pathologic_diagnosis_lte = messages.IntegerField(2) - age_at_initial_pathologic_diagnosis_gte = messages.IntegerField(3) - - anatomic_neoplasm_subdivision = messages.StringField(4, repeated=True) - - avg_percent_lymphocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_lymphocyte_infiltration_lte = messages.FloatField(6) - avg_percent_lymphocyte_infiltration_gte = messages.FloatField(7) - - avg_percent_monocyte_infiltration = messages.FloatField(8, repeated=True) - avg_percent_monocyte_infiltration_lte = messages.FloatField(9) - avg_percent_monocyte_infiltration_gte = messages.FloatField(10) - - avg_percent_necrosis = messages.FloatField(11, repeated=True) - avg_percent_necrosis_lte = messages.FloatField(12) - avg_percent_necrosis_gte = messages.FloatField(13) - - avg_percent_neutrophil_infiltration = messages.FloatField(14, repeated=True) - avg_percent_neutrophil_infiltration_lte = messages.FloatField(15) - avg_percent_neutrophil_infiltration_gte = messages.FloatField(16) - - avg_percent_normal_cells = messages.FloatField(17, repeated=True) - avg_percent_normal_cells_lte = messages.FloatField(18) - avg_percent_normal_cells_gte = messages.FloatField(19) - - avg_percent_stromal_cells = messages.FloatField(20, repeated=True) - avg_percent_stromal_cells_lte = messages.FloatField(21) - avg_percent_stromal_cells_gte = messages.FloatField(22) - - avg_percent_tumor_cells = messages.FloatField(23, repeated=True) - avg_percent_tumor_cells_lte = messages.FloatField(24) - avg_percent_tumor_cells_gte = messages.FloatField(25) - - avg_percent_tumor_nuclei = messages.FloatField(26, repeated=True) - avg_percent_tumor_nuclei_lte = messages.FloatField(27) - avg_percent_tumor_nuclei_gte = messages.FloatField(28) - - batch_number = messages.IntegerField(29, repeated=True) - bcr = messages.StringField(30, repeated=True) - clinical_M = messages.StringField(31, repeated=True) - clinical_N = messages.StringField(32, repeated=True) - clinical_stage = messages.StringField(33, repeated=True) - clinical_T = messages.StringField(34, repeated=True) - colorectal_cancer = messages.StringField(35, repeated=True) - country = messages.StringField(36, repeated=True) - - days_to_birth = messages.IntegerField(37, repeated=True) - days_to_birth_lte = messages.IntegerField(38) - days_to_birth_gte = messages.IntegerField(39) - - days_to_collection = messages.IntegerField(40, repeated=True) - days_to_collection_lte = messages.IntegerField(41) - days_to_collection_gte = messages.IntegerField(42) - - days_to_death = messages.IntegerField(43, repeated=True) - days_to_death_lte = messages.IntegerField(44) - days_to_death_gte = messages.IntegerField(45) - - days_to_initial_pathologic_diagnosis = messages.IntegerField(46, repeated=True) - days_to_initial_pathologic_diagnosis_lte = messages.IntegerField(47) - days_to_initial_pathologic_diagnosis_gte = messages.IntegerField(48) - - days_to_last_followup = messages.IntegerField(49, repeated=True) - days_to_last_followup_lte = messages.IntegerField(50) - days_to_last_followup_gte = messages.IntegerField(51) - - days_to_submitted_specimen_dx = messages.IntegerField(52, repeated=True) - days_to_submitted_specimen_dx_lte = messages.IntegerField(53) - days_to_submitted_specimen_dx_gte = messages.IntegerField(54) - - ethnicity = messages.StringField(55, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(56, repeated=True) - gender = messages.StringField(57, repeated=True) - - has_Illumina_DNASeq = messages.StringField(58, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(59, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(60, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(61, repeated=True) - has_UNC_GA_RNASeq = messages.StringField(62, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(63, repeated=True) - has_GA_miRNASeq = messages.StringField(64, repeated=True) - has_RPPA = messages.StringField(65, repeated=True) - has_SNP6 = messages.StringField(66, repeated=True) - has_27k = messages.StringField(67, repeated=True) - has_450k = messages.StringField(68, repeated=True) - - height = messages.IntegerField(69, repeated=True) - height_lte = messages.IntegerField(70) - height_gte = messages.IntegerField(71) - - histological_type = messages.StringField(72, repeated=True) - history_of_colon_polyps = messages.StringField(73, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(74, repeated=True) - history_of_prior_malignancy = messages.StringField(75, repeated=True) - hpv_calls = messages.StringField(76, repeated=True) - hpv_status = messages.StringField(77, repeated=True) - icd_10 = messages.StringField(78, repeated=True) - icd_o_3_histology = messages.StringField(79, repeated=True) - icd_o_3_site = messages.StringField(80, repeated=True) - lymphatic_invasion = messages.StringField(81, repeated=True) - lymphnodes_examined = messages.StringField(82, repeated=True) - lymphovascular_invasion_present = messages.StringField(83, repeated=True) - - max_percent_lymphocyte_infiltration = messages.IntegerField(84, repeated=True) - max_percent_lymphocyte_infiltration_lte = messages.IntegerField(85) - max_percent_lymphocyte_infiltration_gte = messages.IntegerField(86) - - max_percent_monocyte_infiltration = messages.IntegerField(87, repeated=True) - max_percent_monocyte_infiltration_lte = messages.IntegerField(88) - max_percent_monocyte_infiltration_gte = messages.IntegerField(89) - - max_percent_necrosis = messages.IntegerField(90, repeated=True) - max_percent_necrosis_lte = messages.IntegerField(91) - max_percent_necrosis_gte = messages.IntegerField(92) - - max_percent_neutrophil_infiltration = messages.IntegerField(93, repeated=True) - max_percent_neutrophil_infiltration_lte = messages.IntegerField(94) - max_percent_neutrophil_infiltration_gte = messages.IntegerField(95) - - max_percent_normal_cells = messages.IntegerField(96, repeated=True) - max_percent_normal_cells_lte = messages.IntegerField(97) - max_percent_normal_cells_gte = messages.IntegerField(98) - - max_percent_stromal_cells = messages.IntegerField(99, repeated=True) - max_percent_stromal_cells_lte = messages.IntegerField(100) - max_percent_stromal_cells_gte = messages.IntegerField(101) - - max_percent_tumor_cells = messages.IntegerField(102, repeated=True) - max_percent_tumor_cells_lte = messages.IntegerField(103) - max_percent_tumor_cells_gte = messages.IntegerField(104) - - max_percent_tumor_nuclei = messages.IntegerField(105, repeated=True) - max_percent_tumor_nuclei_lte = messages.IntegerField(106) - max_percent_tumor_nuclei_gte = messages.IntegerField(107) - - menopause_status = messages.StringField(108, repeated=True) - - min_percent_lymphocyte_infiltration = messages.IntegerField(109, repeated=True) - min_percent_lymphocyte_infiltration_lte = messages.IntegerField(110) - min_percent_lymphocyte_infiltration_gte = messages.IntegerField(111) - - min_percent_monocyte_infiltration = messages.IntegerField(112, repeated=True) - min_percent_monocyte_infiltration_lte = messages.IntegerField(113) - min_percent_monocyte_infiltration_gte = messages.IntegerField(114) - - min_percent_necrosis = messages.IntegerField(115, repeated=True) - min_percent_necrosis_lte = messages.IntegerField(116) - min_percent_necrosis_gte = messages.IntegerField(117) - - min_percent_neutrophil_infiltration = messages.IntegerField(118, repeated=True) - min_percent_neutrophil_infiltration_lte = messages.IntegerField(119) - min_percent_neutrophil_infiltration_gte = messages.IntegerField(120) - - min_percent_normal_cells = messages.IntegerField(121, repeated=True) - min_percent_normal_cells_lte = messages.IntegerField(122) - min_percent_normal_cells_gte = messages.IntegerField(123) - - min_percent_stromal_cells = messages.IntegerField(124, repeated=True) - min_percent_stromal_cells_lte = messages.IntegerField(125) - min_percent_stromal_cells_gte = messages.IntegerField(126) - - min_percent_tumor_cells = messages.IntegerField(127, repeated=True) - min_percent_tumor_cells_lte = messages.IntegerField(128) - min_percent_tumor_cells_gte = messages.IntegerField(129) - - min_percent_tumor_nuclei = messages.IntegerField(130, repeated=True) - min_percent_tumor_nuclei_lte = messages.IntegerField(131) - min_percent_tumor_nuclei_gte = messages.IntegerField(132) - - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(133, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(134, repeated=True) - neoplasm_histologic_grade = messages.StringField(135, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(136, repeated=True) - - number_of_lymphnodes_examined = messages.IntegerField(137, repeated=True) - number_of_lymphnodes_examined_lte = messages.IntegerField(138) - number_of_lymphnodes_examined_gte = messages.IntegerField(139) - - number_of_lymphnodes_positive_by_he = messages.IntegerField(140, repeated=True) - number_of_lymphnodes_positive_by_he_lte = messages.IntegerField(141) - number_of_lymphnodes_positive_by_he_gte = messages.IntegerField(142) - - case_barcode = messages.StringField(143, repeated=True) - pathologic_M = messages.StringField(144, repeated=True) - pathologic_N = messages.StringField(145, repeated=True) - pathologic_stage = messages.StringField(146, repeated=True) - pathologic_T = messages.StringField(147, repeated=True) - person_neoplasm_cancer_status = messages.StringField(148, repeated=True) - pregnancies = messages.StringField(149, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(150, repeated=True) - primary_therapy_outcome_success = messages.StringField(151, repeated=True) - prior_dx = messages.StringField(152, repeated=True) - Project = messages.StringField(153, repeated=True) - - psa_value = messages.FloatField(154, repeated=True) - psa_value_lte = messages.FloatField(155) - psa_value_gte = messages.FloatField(156) - - race = messages.StringField(157, repeated=True) - residual_tumor = messages.StringField(158, repeated=True) - sample_barcode = messages.StringField(159, repeated=True) - SampleTypeCode = messages.StringField(160, repeated=True) - Study = messages.StringField(161, repeated=True) - tobacco_smoking_history = messages.StringField(162, repeated=True) - tumor_tissue_site = messages.StringField(163, repeated=True) - tumor_type = messages.StringField(164, repeated=True) - weiss_venous_invasion = messages.StringField(165, repeated=True) - vital_status = messages.StringField(166, repeated=True) - - weight = messages.IntegerField(167, repeated=True) - weight_lte = messages.IntegerField(168) - weight_gte = messages.IntegerField(169) - - year_of_initial_pathologic_diagnosis = messages.IntegerField(170, repeated=True) - year_of_initial_pathologic_diagnosis_lte = messages.IntegerField(171) - year_of_initial_pathologic_diagnosis_gte = messages.IntegerField(172) - - -def are_there_bad_keys(request): - ''' - Checks for unrecognized fields in an endpoint request - :param request: the request object from the endpoint - :return: boolean indicating True if bad (unrecognized) fields are present in the request - ''' - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - return unrecognized_param_dict != {} - - -def are_there_no_acceptable_keys(request): - """ - Checks for a lack of recognized fields in an endpoints request. Used in save_cohort and preview_cohort endpoints. - :param request: the request object from the endpoint - :return: boolean indicating True if there are no recognized fields in the request. - """ - param_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - } - return param_dict == {} - - -def construct_parameter_error_message(request, filter_required): - err_msg = '' - sorted_acceptable_keys = sorted([k.name for k in request.all_fields()], key=lambda s: s.lower()) - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - if unrecognized_param_dict: - bad_key_str = "'" + "', '".join(unrecognized_param_dict.keys()) + "'" - err_msg += "The following filters were not recognized: {}. ".format(bad_key_str) - if filter_required: - err_msg += "You must specify at least one of the following " \ - "case-sensitive filters: {}".format(sorted_acceptable_keys) - else: - err_msg += "Acceptable filters are: {}".format(sorted_acceptable_keys) - - return err_msg - - -def get_list_of_split_values_for_filter_model(large_value_list): - ''' - :rtype: list - :param large_value_list: protorpc.messages.FieldList - :return: list of smaller protorpc.messages.FieldLists - ''' - - return_list = [] - - # if length_of_list is larger than 512 characters, - # the Filter model will not be able to be saved - # with this as the value field - length_of_list = len('"' + '", "'.join(large_value_list) + '"') - while length_of_list > MAX_FILTER_VALUE_LENGTH: - new_smaller_list = [] - length_of_smaller_list = len('"' + '", "'.join(new_smaller_list) + '"') - while length_of_smaller_list < MAX_FILTER_VALUE_LENGTH: - try: - new_smaller_list.append(large_value_list.pop()) - except IndexError: - break - length_of_smaller_list = len('"' + '", "'.join(new_smaller_list) + '"') - large_value_list.append(new_smaller_list.pop()) - return_list.append(new_smaller_list) - length_of_list = len('"' + '", "'.join(large_value_list) + '"') - - if len(large_value_list): - return_list.append(large_value_list) - - return return_list - - -Cohort_Endpoints = endpoints.api(name='cohort_api', version='v1', - description="Get information about cohorts, patients, and samples. Create and delete cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - - -@Cohort_Endpoints.api_class(resource_name='cohort_endpoints') -class Cohort_Endpoints_API(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(token=messages.StringField(1), cohort_id=messages.IntegerField(2)) - - @endpoints.method(GET_RESOURCE, CohortsList, - path='cohorts_list', http_method='GET', name='cohorts.list') - def cohorts_list(self, request): - ''' - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. Optionally takes a cohort id as a parameter to - only list information about one cohort. - ''' - user_email = None - cursor = None - filter_cursor = None - parent_cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - query_dict = {'cohorts_cohort_perms.user_id': user_id, 'cohorts_cohort.active': unicode('1')} - - if cohort_id: - query_dict['cohorts_cohort.id'] = cohort_id - - query_str = 'select cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'cohorts_cohort_comments.content as comments, ' \ - 'cohorts_source.type as source_type, ' \ - 'cohorts_source.notes as source_notes ' \ - 'from cohorts_cohort_perms ' \ - 'join cohorts_cohort ' \ - 'on cohorts_cohort.id=cohorts_cohort_perms.cohort_id ' \ - 'join auth_user ' \ - 'on auth_user.id=cohorts_cohort_perms.user_id ' \ - 'left join cohorts_cohort_comments ' \ - 'on cohorts_cohort_comments.user_id=cohorts_cohort_perms.user_id ' \ - 'and cohorts_cohort_comments.cohort_id=cohorts_cohort.id ' \ - 'left join cohorts_source ' \ - 'on cohorts_source.cohort_id=cohorts_cohort_perms.cohort_id ' - - - - query_tuple = () - if query_dict: - query_str += ' where ' + '=%s and '.join(key for key in query_dict.keys()) + '=%s ' - query_tuple = tuple(value for value in query_dict.values()) - - query_str += 'group by ' \ - 'cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'comments, ' \ - 'source_type, ' \ - 'source_notes ' - - print query_str - print query_tuple - - filter_query_str = '' - parent_id_query_str = '' - row = None - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - data = [] - - for row in cursor.fetchall(): - filter_query_str = 'SELECT name, value ' \ - 'FROM cohorts_filters ' \ - 'WHERE cohorts_filters.resulting_cohort_id=%s' - - filter_cursor = db.cursor(MySQLdb.cursors.DictCursor) - filter_cursor.execute(filter_query_str, (str(row['id']),)) - filter_data = [] - for filter_row in filter_cursor.fetchall(): - filter_data.append(FilterDetails( - name=str(filter_row['name']), - value=str(filter_row['value']) - )) - - # getting the parent_id is a separate query since a single cohort - # may have multiple parent cohorts - parent_id_query_str = 'SELECT parent_id ' \ - 'FROM cohorts_source ' \ - 'WHERE cohort_id=%s' - - parent_cursor = db.cursor(MySQLdb.cursors.DictCursor) - parent_cursor.execute(parent_id_query_str, (str(row['id']),)) - parent_id_data = [] - for parent_row in parent_cursor.fetchall(): - if row.get('parent_id') is not None: - parent_id_data.append(int(parent_row['parent_id'])) - - data.append(Cohort( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - perm=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=None if row['source_type'] is None else str(row['source_type']), - source_notes=None if row['source_notes'] is None else str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data - )) - - if len(data) == 0: - optional_message = " matching cohort id " + str(cohort_id) if cohort_id is not None else "" - raise endpoints.NotFoundException("{} has no active cohorts{}." - .format(user_email, optional_message)) - return CohortsList(items=data, count=len(data)) - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "User {}'s cohorts not found. {}: {}".format(user_email, type(e), e)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tcohort query: {} {}\n\tfilter query: {} {}\n\tparent id query: {} {}' \ - .format(e, query_str, query_tuple, filter_query_str, str(row['id']), parent_id_query_str, - str(row['id'])) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(msg)) - finally: - if cursor: cursor.close() - if filter_cursor: filter_cursor.close() - if parent_cursor: parent_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - token=messages.StringField(2)) - - @endpoints.method(GET_RESOURCE, CohortPatientsSamplesList, - path='cohort_patients_samples_list', http_method='GET', - name='cohorts.cohort_patients_samples_list') - def cohort_patients_samples_list(self, request): - """ - Takes a cohort id as a required parameter and returns information about the participants - and samples in a particular cohort. Authentication is required. - User must have either READER or OWNER permissions on the cohort. - """ - - db = None - cursor = None - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - cohort_perms_query = "select count(*) from cohorts_cohort_perms where user_id=%s and cohort_id=%s" - cohort_perms_tuple = (user_id, cohort_id) - cohort_query = "select count(*) from cohorts_cohort where id=%s and active=%s" - cohort_tuple = (cohort_id, unicode('0')) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(cohort_perms_query, cohort_perms_tuple) - result = cursor.fetchone() - if int(result['count(*)']) == 0: - error_message = "{} does not have owner or reader permissions on cohort {}.".format(user_email, - cohort_id) - request_finished.send(self) - raise endpoints.ForbiddenException(error_message) - - cursor.execute(cohort_query, cohort_tuple) - result = cursor.fetchone() - if int(result['count(*)']) > 0: - error_message = "Cohort {} was deleted.".format(cohort_id) - request_finished.send(self) - raise endpoints.NotFoundException(error_message) - - except (IndexError, TypeError) as e: - logger.warn(e) - raise endpoints.NotFoundException("Cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tcohort permissions query: {} {}\n\tcohort query: {} {}' \ - .format(e, cohort_perms_query, cohort_perms_tuple, cohort_query, cohort_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving cohorts or cohort permissions. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - patient_query_str = 'select cohorts_samples.case_barcode ' \ - 'from cohorts_samples cs ' \ - 'inner join cohorts_cohort_perms ccp' \ - 'on ccp.cohort_id=cs.cohort_id ' \ - 'inner join cohorts_cohort cc' \ - 'on cs.cohort_id=cc.id ' \ - 'where cs.cohort_id=%s ' \ - 'and ccp.user_id=%s ' \ - 'and cc.active=%s ' \ - 'group by cs.case_barcode ' - - patient_query_tuple = (cohort_id, user_id, unicode('1')) - - sample_query_str = 'select cohorts_samples.sample_barcode ' \ - 'from cohorts_samples ' \ - 'inner join cohorts_cohort_perms ' \ - 'on cohorts_cohort_perms.cohort_id=cohorts_samples.cohort_id ' \ - 'inner join cohorts_cohort ' \ - 'on cohorts_samples.cohort_id=cohorts_cohort.id ' \ - 'where cohorts_samples.cohort_id=%s ' \ - 'and cohorts_cohort_perms.user_id=%s ' \ - 'and cohorts_cohort.active=%s ' \ - 'group by cohorts_samples.sample_barcode ' - - sample_query_tuple = (cohort_id, user_id, unicode('1')) - - try: - db = sql_connection() - - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(patient_query_str, patient_query_tuple) - patient_data = [] - for row in cursor.fetchall(): - patient_data.append(row['case_barcode']) - - cursor.execute(sample_query_str, sample_query_tuple) - sample_data = [] - for row in cursor.fetchall(): - sample_data.append(row['sample_barcode']) - - return CohortPatientsSamplesList(patients=patient_data, - patient_count=len(patient_data), - samples=sample_data, - sample_count=len(sample_data), - cohort_id=int(cohort_id)) - except (IndexError, TypeError) as e: - logger.warn(e) - raise endpoints.NotFoundException("Cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tpatient query: {} {}\n\tsample query: {} {}' \ - .format(e, patient_query_str, patient_query_tuple, sample_query_str, sample_query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving patients or samples. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(patient_barcode=messages.StringField(1, required=True)) - - @endpoints.method(GET_RESOURCE, PatientDetails, - path='patient_details', http_method='GET', name='cohorts.patient_details') - def patient_details(self, request): - """ - Returns information about a specific participant, - including a list of samples and aliquots derived from this patient. - Takes a participant barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - - clinical_cursor = None - sample_cursor = None - aliquot_cursor = None - db = None - - patient_barcode = request.get_assigned_value('patient_barcode') - - clinical_query_str = 'select * ' \ - 'from metadata_clinical ' \ - 'where case_barcode=%s' - - query_tuple = (str(patient_barcode),) - - sample_query_str = 'select sample_barcode ' \ - 'from metadata_biospecimen ' \ - 'where case_barcode=%s' - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where case_barcode=%s ' \ - 'group by AliquotBarcode' - try: - db = sql_connection() - clinical_cursor = db.cursor(MySQLdb.cursors.DictCursor) - clinical_cursor.execute(clinical_query_str, query_tuple) - row = clinical_cursor.fetchone() - - item = MetadataItem( - age_at_initial_pathologic_diagnosis=None if "age_at_initial_pathologic_diagnosis" not in row or row[ - "age_at_initial_pathologic_diagnosis"] is None else int( - row["age_at_initial_pathologic_diagnosis"]), - anatomic_neoplasm_subdivision=str(row["anatomic_neoplasm_subdivision"]), - batch_number=None if "batch_number" not in row or row["batch_number"] is None else int( - row["batch_number"]), - bcr=str(row["bcr"]), - clinical_M=str(row["clinical_M"]), - clinical_N=str(row["clinical_N"]), - clinical_stage=str(row["clinical_stage"]), - clinical_T=str(row["clinical_T"]), - colorectal_cancer=str(row["colorectal_cancer"]), - country=str(row["country"]), - days_to_birth=None if "days_to_birth" not in row or row['days_to_birth'] is None else int( - row["days_to_birth"]), - days_to_death=None if "days_to_death" not in row or row['days_to_death'] is None else int( - row["days_to_death"]), - days_to_initial_pathologic_diagnosis=None if "days_to_initial_pathologic_diagnosis" not in row or row[ - 'days_to_initial_pathologic_diagnosis'] is None else int( - row["days_to_initial_pathologic_diagnosis"]), - days_to_last_followup=None if "days_to_last_followup" not in row or row[ - 'days_to_last_followup'] is None else int( - row["days_to_last_followup"]), - days_to_submitted_specimen_dx=None if "days_to_submitted_specimen_dx" not in row or row[ - 'days_to_submitted_specimen_dx'] is None else int( - row["days_to_submitted_specimen_dx"]), - Study=str(row["Study"]), - ethnicity=str(row["ethnicity"]), - frozen_specimen_anatomic_site=str(row["frozen_specimen_anatomic_site"]), - gender=str(row["gender"]), - height=None if "height" not in row or row['height'] is None else int(row["height"]), - histological_type=str(row["histological_type"]), - history_of_colon_polyps=str(row["history_of_colon_polyps"]), - history_of_neoadjuvant_treatment=str(row["history_of_neoadjuvant_treatment"]), - history_of_prior_malignancy=str(row["history_of_prior_malignancy"]), - hpv_calls=str(row["hpv_calls"]), - hpv_status=str(row["hpv_status"]), - icd_10=str(row["icd_10"]), - icd_o_3_histology=str(row["icd_o_3_histology"]), - icd_o_3_site=str(row["icd_o_3_site"]), - lymphatic_invasion=str(row["lymphatic_invasion"]), - lymphnodes_examined=str(row["lymphnodes_examined"]), - lymphovascular_invasion_present=str(row["lymphovascular_invasion_present"]), - menopause_status=str(row["menopause_status"]), - mononucleotide_and_dinucleotide_marker_panel_analysis_status=str( - row["mononucleotide_and_dinucleotide_marker_panel_analysis_status"]), - mononucleotide_marker_panel_analysis_status=str(row["mononucleotide_marker_panel_analysis_status"]), - neoplasm_histologic_grade=str(row["neoplasm_histologic_grade"]), - new_tumor_event_after_initial_treatment=str(row["new_tumor_event_after_initial_treatment"]), - number_of_lymphnodes_examined=None if "number_of_lymphnodes_examined" not in row or row[ - 'number_of_lymphnodes_examined'] is None else int( - row["number_of_lymphnodes_examined"]), - number_of_lymphnodes_positive_by_he=None if "number_of_lymphnodes_positive_by_he" not in row or row[ - 'number_of_lymphnodes_positive_by_he'] is None else int( - row["number_of_lymphnodes_positive_by_he"]), - case_barcode=str(row["case_barcode"]), - pathologic_M=str(row["pathologic_M"]), - pathologic_N=str(row["pathologic_N"]), - pathologic_stage=str(row["pathologic_stage"]), - pathologic_T=str(row["pathologic_T"]), - person_neoplasm_cancer_status=str(row["person_neoplasm_cancer_status"]), - pregnancies=str(row["pregnancies"]), - primary_neoplasm_melanoma_dx=str(row["primary_neoplasm_melanoma_dx"]), - primary_therapy_outcome_success=str(row["primary_therapy_outcome_success"]), - prior_dx=str(row["prior_dx"]), - Project=str(row["Project"]), - psa_value=None if "psa_value" not in row or row["psa_value"] is None else float(row["psa_value"]), - race=str(row["race"]), - residual_tumor=str(row["residual_tumor"]), - tobacco_smoking_history=str(row["tobacco_smoking_history"]), - tumor_tissue_site=str(row["tumor_tissue_site"]), - tumor_type=str(row["tumor_type"]), - weiss_venous_invasion=str(row["weiss_venous_invasion"]), - vital_status=str(row["vital_status"]), - weight=None if "weight" not in row or row["weight"] is None else int(float(row["weight"])), - year_of_initial_pathologic_diagnosis=str(row["year_of_initial_pathologic_diagnosis"]) - ) - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, query_tuple) - sample_data = [] - for row in sample_cursor.fetchall(): - sample_data.append(row['sample_barcode']) - - aliquot_cursor = db.cursor(MySQLdb.cursors.DictCursor) - aliquot_cursor.execute(aliquot_query_str, query_tuple) - aliquot_data = [] - for row in aliquot_cursor.fetchall(): - aliquot_data.append(row['AliquotBarcode']) - - return PatientDetails(clinical_data=item, samples=sample_data, aliquots=aliquot_data) - except (IndexError, TypeError), e: - logger.info("Patient {} not found. Error: {}".format(patient_barcode, e)) - raise endpoints.NotFoundException("Patient {} not found.".format(patient_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tpatient query: {} {}\n\tsample query: {} {}\n\taliquot query: {} {}' \ - .format(e, clinical_query_str, query_tuple, sample_query_str, query_tuple, - aliquot_query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving patient, sample, or aliquot data. {}".format(msg)) - finally: - if clinical_cursor: clinical_cursor.close() - if sample_cursor: sample_cursor.close() - if aliquot_cursor: aliquot_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, SampleDetails, - path='sample_details', http_method='GET', name='cohorts.sample_details') - def sample_details(self, request): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated patient barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - - biospecimen_cursor = None - aliquot_cursor = None - patient_cursor = None - data_cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - biospecimen_query_str = 'select * ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s' - - query_tuple = (str(sample_barcode),) - extra_query_tuple = query_tuple - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' - - patient_query_str = 'select case_barcode ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s ' - - data_query_str = 'select ' \ - 'sample_barcode, ' \ - 'DataCenterName, ' \ - 'DataCenterType, ' \ - 'DataFileName, ' \ - 'DataFileNameKey, ' \ - 'DatafileUploaded, ' \ - 'DataLevel,' \ - 'Datatype,' \ - 'GenomeReference,' \ - 'GG_dataset_id, ' \ - 'GG_readgroupset_id, ' \ - 'Pipeline,' \ - 'Platform,' \ - 'platform_full_name,' \ - 'Project,' \ - 'Repository,' \ - 'SDRFFileName,' \ - 'SecurityProtocol ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' - - if request.get_assigned_value('platform') is not None: - platform = request.get_assigned_value('platform') - aliquot_query_str += ' and platform=%s ' - data_query_str += ' and platform=%s ' - extra_query_tuple += (str(platform),) - - if request.get_assigned_value('pipeline') is not None: - pipeline = request.get_assigned_value('pipeline') - aliquot_query_str += ' and pipeline=%s ' - data_query_str += ' and pipeline=%s ' - extra_query_tuple += (str(pipeline),) - - aliquot_query_str += ' group by AliquotBarcode' - patient_query_str += ' group by case_barcode' - - try: - db = sql_connection() - biospecimen_cursor = db.cursor(MySQLdb.cursors.DictCursor) - biospecimen_cursor.execute(biospecimen_query_str, query_tuple) - row = biospecimen_cursor.fetchone() - - item = MetadataItem( - avg_percent_lymphocyte_infiltration=None if "avg_percent_lymphocyte_infiltration" not in row or row[ - "avg_percent_lymphocyte_infiltration"] is None else float( - row["avg_percent_lymphocyte_infiltration"]), - avg_percent_monocyte_infiltration=None if "avg_percent_monocyte_infiltration" not in row or row[ - "avg_percent_monocyte_infiltration"] is None else float( - row["avg_percent_monocyte_infiltration"]), - avg_percent_necrosis=None if "avg_percent_necrosis" not in row or row[ - "avg_percent_necrosis"] is None else float( - row["avg_percent_necrosis"]), - avg_percent_neutrophil_infiltration=None if "avg_percent_neutrophil_infiltration" not in row or row[ - "avg_percent_neutrophil_infiltration"] is None else float( - row["avg_percent_neutrophil_infiltration"]), - avg_percent_normal_cells=None if "avg_percent_normal_cells" not in row or row[ - "avg_percent_normal_cells"] is None else float( - row["avg_percent_normal_cells"]), - avg_percent_stromal_cells=None if "avg_percent_stromal_cells" not in row or row[ - "avg_percent_stromal_cells"] is None else float( - row["avg_percent_stromal_cells"]), - avg_percent_tumor_cells=None if "avg_percent_tumor_cells" not in row or row[ - "avg_percent_tumor_cells"] is None else float( - row["avg_percent_tumor_cells"]), - avg_percent_tumor_nuclei=None if "avg_percent_tumor_nuclei" not in row or row[ - "avg_percent_tumor_nuclei"] is None else float( - row["avg_percent_tumor_nuclei"]), - batch_number=None if "batch_number" not in row or row["batch_number"] is None else int( - row["batch_number"]), - bcr=str(row["bcr"]), - days_to_collection=None if "days_to_collection" not in row or row[ - 'days_to_collection'] is None else int( - row["days_to_collection"]), - max_percent_lymphocyte_infiltration=None if "max_percent_lymphocyte_infiltration" not in row or row[ - "max_percent_lymphocyte_infiltration"] is None else int( - row["max_percent_lymphocyte_infiltration"]), # 46) - max_percent_monocyte_infiltration=None if "max_percent_monocyte_infiltration" not in row or row[ - "max_percent_monocyte_infiltration"] is None else int( - row["max_percent_monocyte_infiltration"]), # 47) - max_percent_necrosis=None if "max_percent_necrosis" not in row or row[ - "max_percent_necrosis"] is None else int( - row["max_percent_necrosis"]), # 48) - max_percent_neutrophil_infiltration=None if "max_percent_neutrophil_infiltration" not in row or row[ - "max_percent_neutrophil_infiltration"] is None else int( - row["max_percent_neutrophil_infiltration"]), # 49) - max_percent_normal_cells=None if "max_percent_normal_cells" not in row or row[ - "max_percent_normal_cells"] is None else int( - row["max_percent_normal_cells"]), # 50) - max_percent_stromal_cells=None if "max_percent_stromal_cells" not in row or row[ - "max_percent_stromal_cells"] is None else int( - row["max_percent_stromal_cells"]), # 51) - max_percent_tumor_cells=None if "max_percent_tumor_cells" not in row or row[ - "max_percent_tumor_cells"] is None else int( - row["max_percent_tumor_cells"]), # 52) - max_percent_tumor_nuclei=None if "max_percent_tumor_nuclei" not in row or row[ - "max_percent_tumor_nuclei"] is None else int( - row["max_percent_tumor_nuclei"]), # 53) - min_percent_lymphocyte_infiltration=None if "min_percent_lymphocyte_infiltration" not in row or row[ - "min_percent_lymphocyte_infiltration"] is None else int( - row["min_percent_lymphocyte_infiltration"]), # 55) - min_percent_monocyte_infiltration=None if "min_percent_monocyte_infiltration" not in row or row[ - "min_percent_monocyte_infiltration"] is None else int( - row["min_percent_monocyte_infiltration"]), # 56) - min_percent_necrosis=None if "min_percent_necrosis" not in row or row[ - "min_percent_necrosis"] is None else int( - row["min_percent_necrosis"]), # 57) - min_percent_neutrophil_infiltration=None if "min_percent_neutrophil_infiltration" not in row or row[ - "min_percent_neutrophil_infiltration"] is None else int( - row["min_percent_neutrophil_infiltration"]), # 58) - min_percent_normal_cells=None if "min_percent_normal_cells" not in row or row[ - "min_percent_normal_cells"] is None else int( - row["min_percent_normal_cells"]), # 59) - min_percent_stromal_cells=None if "min_percent_stromal_cells" not in row or row[ - "min_percent_stromal_cells"] is None else int( - row["min_percent_stromal_cells"]), # 60) - min_percent_tumor_cells=None if "min_percent_tumor_cells" not in row or row[ - "min_percent_tumor_cells"] is None else int( - row["min_percent_tumor_cells"]), # 61) - min_percent_tumor_nuclei=None if "min_percent_tumor_nuclei" not in row or row[ - "min_percent_tumor_nuclei"] is None else int( - row["min_percent_tumor_nuclei"]), # 62) - case_barcode=str(row["case_barcode"]), - Project=str(row["Project"]), - sample_barcode=str(row["sample_barcode"]), - Study=str(row["Study"]) - ) - aliquot_cursor = db.cursor(MySQLdb.cursors.DictCursor) - aliquot_cursor.execute(aliquot_query_str, extra_query_tuple) - aliquot_data = [] - for row in aliquot_cursor.fetchall(): - aliquot_data.append(row['AliquotBarcode']) - - patient_cursor = db.cursor(MySQLdb.cursors.DictCursor) - patient_cursor.execute(patient_query_str, query_tuple) - row = patient_cursor.fetchone() - if row is None: - aliquot_cursor.close() - patient_cursor.close() - biospecimen_cursor.close() - db.close() - error_message = "Sample barcode {} not found in metadata_biospecimen table.".format(sample_barcode) - return SampleDetails(biospecimen_data=None, aliquots=[], patient=None, data_details=[], - data_details_count=None, error=error_message) - patient_barcode = str(row["case_barcode"]) - - data_cursor = db.cursor(MySQLdb.cursors.DictCursor) - data_cursor.execute(data_query_str, extra_query_tuple) - data_data = [] - bad_repo_count = 0 - bad_repo_set = set() - for row in data_cursor.fetchall(): - if not row.get('DataFileNameKey'): - continue - if 'controlled' not in str(row['SecurityProtocol']).lower(): - cloud_storage_path = "gs://{}{}".format(settings.OPEN_DATA_BUCKET, row.get('DataFileNameKey')) - else: # not filtering on dbGaP_authorized: - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: # shouldn't ever happen - bad_repo_count += 1 - bad_repo_set.add(row['Repository']) - continue - cloud_storage_path = "gs://{}{}".format(bucket_name, row.get('DataFileNameKey')) - - data_item = DataDetails( - sample_barcode=str(row['sample_barcode']), - DataCenterName=str(row['DataCenterName']), - DataCenterType=str(row['DataCenterType']), - DataFileName=str(row['DataFileName']), - DataFileNameKey=str(row.get('DataFileNameKey')), - DatafileUploaded=str(row['DatafileUploaded']), - DataLevel=str(row['DataLevel']), - Datatype=str(row['Datatype']), - GenomeReference=str(row['GenomeReference']), - GG_dataset_id=str(row['GG_dataset_id']), - GG_readgroupset_id=str(row['GG_readgroupset_id']), - Pipeline=str(row['Pipeline']), - Platform=str(row['Platform']), - platform_full_name=str(row['platform_full_name']), - Project=str(row['Project']), - Repository=str(row['Repository']), - SDRFFileName=str(row['SDRFFileName']), - SecurityProtocol=str(row['SecurityProtocol']), - CloudStoragePath=cloud_storage_path - ) - data_data.append(data_item) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return SampleDetails(biospecimen_data=item, aliquots=aliquot_data, - patient=patient_barcode, data_details=data_data, - data_details_count=len(data_data)) - - except (IndexError, TypeError) as e: - logger.info("Sample details for barcode {} not found. Error: {}".format(sample_barcode, e)) - raise endpoints.NotFoundException( - "Sample details for barcode {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tbiospecimen query: {} {}\n\tpatient query: {} {}\n\tdata query: {} {}' \ - .format(e, biospecimen_query_str, query_tuple, patient_query_str, query_tuple, - data_query_str, extra_query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving biospecimen, patient, or other data. {}".format(msg)) - finally: - if biospecimen_cursor: biospecimen_cursor.close() - if aliquot_cursor: aliquot_cursor.close() - if patient_cursor: patient_cursor.close() - if data_cursor: data_cursor.close() - if db and db.open: db.close() - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - limit=messages.IntegerField(2), - platform=messages.StringField(3), - pipeline=messages.StringField(4), - token=messages.StringField(5)) - - @endpoints.method(GET_RESOURCE, DataFileNameKeyList, - path='datafilenamekey_list_from_cohort', http_method='GET', - name='cohorts.datafilenamekey_list_from_cohort') - def datafilenamekey_list_from_cohort(self, request): - """ - Takes a cohort id as a required parameter and returns cloud storage paths to files - associated with all the samples in that cohort, up to a default limit of 10,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - user_email = None - cursor = None - db = None - - limit = request.get_assigned_value('limit') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - cohort_id = request.get_assigned_value('cohort_id') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email: - django.setup() - - query_str = 'SELECT DataFileNameKey, SecurityProtocol, Repository ' \ - 'FROM metadata_data ' - - try: - user_id = Django_User.objects.get(email=user_email).id - django_cohort = Django_Cohort.objects.get(id=cohort_id) - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. Error: {}" \ - .format(user_email, cohort_id, e) - request_finished.send(self) - raise endpoints.UnauthorizedException(err_msg) - - query_str += 'JOIN cohorts_samples ON metadata_data.sample_barcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' \ - 'AND DataFileNameKey != "" AND DataFileNameKey is not null ' - query_tuple = (cohort_id,) - - if platform: - query_str += ' and metadata_data.Platform=%s ' - query_tuple += (platform,) - - if pipeline: - query_str += ' and metadata_data.Pipeline=%s ' - query_tuple += (pipeline,) - - query_str += ' GROUP BY DataFileNameKey, SecurityProtocol, Repository ' - if limit is None: - query_str += ' LIMIT 10000' - else: - query_str += ' LIMIT %s' - query_tuple += (limit,) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - datafilenamekeys = [] - bad_repo_count = 0 - bad_repo_set = set() - for row in cursor.fetchall(): - if not row.get('DataFileNameKey'): - continue - if 'controlled' not in str(row['SecurityProtocol']).lower(): - datafilenamekeys.append( - "gs://{}{}".format(settings.OPEN_DATA_BUCKET, row.get('DataFileNameKey'))) - else: # not filtering on dbGaP_authorized - bucket_name = '' - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: # shouldn't ever happen - bad_repo_count += 1 - bad_repo_set.add(row['Repository']) - continue - datafilenamekeys.append("gs://{}{}".format(bucket_name, row.get('DataFileNameKey'))) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return DataFileNameKeyList(datafilenamekeys=datafilenamekeys, count=len(datafilenamekeys)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\t query: {} {}'.format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, DataFileNameKeyList, - path='datafilenamekey_list_from_sample', http_method='GET', - name='cohorts.datafilenamekey_list_from_sample') - def datafilenamekey_list_from_sample(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - query_str = 'SELECT DataFileNameKey, SecurityProtocol, Repository ' \ - 'FROM metadata_data WHERE sample_barcode=%s ' - - query_tuple = (sample_barcode,) - - if platform: - query_str += ' and Platform=%s ' - query_tuple += (platform,) - - if pipeline: - query_str += ' and Pipeline=%s ' - query_tuple += (pipeline,) - - query_str += ' GROUP BY DataFileNameKey, SecurityProtocol, Repository' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - datafilenamekeys = [] - bad_repo_count = 0 - bad_repo_set = set() - for row in cursor.fetchall(): - if not row.get('DataFileNameKey'): - continue - if 'controlled' not in str(row['SecurityProtocol']).lower(): - datafilenamekeys.append("gs://{}{}".format(settings.OPEN_DATA_BUCKET, row.get('DataFileNameKey'))) - else: # not filtering on dbGaP_authorized - bucket_name = '' - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: # shouldn't ever happen - bad_repo_count += 0 - bad_repo_set.add(row['Repository']) - continue - datafilenamekeys.append("gs://{}{}".format(bucket_name, row.get('DataFileNameKey'))) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return DataFileNameKeyList(datafilenamekeys=datafilenamekeys, count=len(datafilenamekeys)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for sample {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\t query: {} {}'.format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, - name=messages.StringField(2, required=True), - token=messages.StringField(3)) - - @endpoints.method(POST_RESOURCE, Cohort, - path='save_cohort', http_method='POST', name='cohorts.save_cohort') - def save_cohort(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of patients and samples in that cohort. - """ - user_email = None - patient_cursor = None - sample_cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email: - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - request_finished.send(self) - raise endpoints.BadRequestException(err_msg) - - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name is not 'name' and k.name is not 'token' - and not k.name.endswith('_gte') - and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - sample_query_str = 'SELECT sample_barcode, IF(case_barcode="", LEFT(sample_barcode,12), case_barcode) AS case_barcode' \ - 'FROM metadata_samples ' \ - 'WHERE ' - value_tuple = () - - for key, value_list in query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - if isinstance(value_list, collections.Iterable) and "None" in value_list: - value_list.remove("None") - sample_query_str += ' ( {key} is null '.format(key=key) - if len(value_list) > 0: - sample_query_str += ' OR {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ') ' - else: - sample_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} >=%s '.format(key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} <=%s '.format(key) - value_tuple += (value,) - - sample_query_str += ' GROUP BY sample_barcode' - - patient_barcodes = [] - sample_barcodes = [] - try: - db = sql_connection() - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, value_tuple) - for row in sample_cursor.fetchall(): - sample_barcodes.append((row['sample_barcode'],row['case_barcode'],)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("Error retrieving samples or patients") - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tsample query: {} {}' \ - .format(e, sample_query_str, value_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error saving cohort. {}".format(msg)) - finally: - if patient_cursor: patient_cursor.close() - if sample_cursor: sample_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - cohort_name = request.get_assigned_value('name') - - # Validate the cohort name against a whitelist - whitelist = re.compile(WHITELIST_RE, re.UNICODE) - match = whitelist.search(unicode(cohort_name)) - if match: - # XSS risk, log and fail this cohort save - match = whitelist.findall(unicode(cohort_name)) - logger.error('[ERROR] While saving a cohort, saw a malformed name: ' + cohort_name + ', characters: ' + match.__str__()) - raise endpoints.BadRequestException("Your cohort's name contains invalid characters ("+match.__str__()+"); please choose another name.") - - # 1. create new cohorts_cohort with name, active=True, last_date_saved=now - created_cohort = Django_Cohort.objects.create(name=cohort_name, active=True, - last_date_saved=datetime.utcnow()) - created_cohort.save() - - # 2. insert samples into cohort_samples - sample_barcodes = list(set(sample_barcodes)) - sample_list = [Samples(cohort=created_cohort, sample_barcode=sample[0], case_barcode=sample[1]) for sample in sample_barcodes] - Samples.objects.bulk_create(sample_list) - - # 3. Set permission for user to be owner - perm = Cohort_Perms(cohort=created_cohort, user=django_user, perm=Cohort_Perms.OWNER) - perm.save() - - # 4. Create filters applied - filter_data = [] - for key, value_list in query_dict.items(): - for val in value_list: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - for key, val in [(k + '_lte', v) for k, v in lte_query_dict.items()] + [(k + '_gte', v) for k, v in gte_query_dict.items()]: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - # 5. Store cohort to BigQuery - project_id = settings.BQ_PROJECT_ID - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - bcs = BigQueryCohortSupport(project_id, cohort_settings.dataset_id, cohort_settings.table_id) - bcs.add_cohort_with_sample_barcodes(created_cohort.id, sample_barcodes) - - request_finished.send(self) - return Cohort(id=str(created_cohort.id), - name=cohort_name, - last_date_saved=str(datetime.utcnow()), - num_patients=str(len(patient_barcodes)), - num_samples=str(len(sample_barcodes)) - ) - - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - DELETE_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - token=messages.StringField(2)) - - @endpoints.method(DELETE_RESOURCE, ReturnJSON, - path='delete_cohort', http_method='POST', name='cohorts.delete') - def delete_cohort(self, request): - """ - Deletes a cohort. User must have owner permissions on the cohort. - """ - user_email = None - return_message = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email: - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_to_deactivate = Django_Cohort.objects.get(id=cohort_id) - if cohort_to_deactivate.active is True: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - if cohort_perm.perm == 'OWNER': - cohort_to_deactivate.active = False - cohort_to_deactivate.save() - return_message = 'Cohort %d successfully deactivated.' % cohort_id - else: - return_message = 'You do not have owner permission on cohort %d.' % cohort_id - else: - return_message = "Cohort %d was already deactivated." % cohort_id - - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Either cohort %d does not have an entry in the database " - "or you do not have owner or reader permissions on this cohort." % cohort_id) - finally: - request_finished.send(self) - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - return ReturnJSON(msg=return_message) - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem) - - @endpoints.method(POST_RESOURCE, CohortPatientsSamplesList, - path='preview_cohort', http_method='POST', name='cohorts.preview_cohort') - def preview_cohort(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of participant (aka patient) barcodes, and the list of sample barcodes. - Authentication is not required. - """ - # print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - patient_cursor = None - sample_cursor = None - db = None - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - raise endpoints.BadRequestException(err_msg) - - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and not k.name.endswith('_gte') and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - sample_query_str = 'SELECT sample_barcode,IF(case_barcode="", LEFT(sample_barcode,12), case_barcode) AS case_barcode ' \ - 'FROM metadata_samples ' \ - 'WHERE ' - - value_tuple = () - - for key, value_list in query_dict.iteritems(): - - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - if "None" in value_list: - value_list.remove("None") - sample_query_str += ' ( {key} is null '.format(key=key) - if len(value_list) > 0: - sample_query_str += ' OR {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ') ' - else: - sample_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} >=%s '.format(key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} <=%s '.format(key) - value_tuple += (value,) - - sample_query_str += ' GROUP BY sample_barcode' - - patient_barcodes = [] - sample_barcodes = [] - - try: - db = sql_connection() - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, value_tuple) - for row in sample_cursor.fetchall(): - sample_barcodes.append(row['sample_barcode']) - patient_barcodes.append(row['case_barcode']) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("Error retrieving samples or patients: {}".format(e)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tsample query: {} {}' \ - .format(e, sample_query_str, value_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error previewing cohort. {}".format(msg)) - finally: - if sample_cursor: sample_cursor.close() - if db and db.open: db.close() - - return CohortPatientsSamplesList(patients=patient_barcodes, - patient_count=len(patient_barcodes), - samples=sample_barcodes, - sample_count=len(sample_barcodes)) - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - token=messages.StringField(2)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, - # path='google_genomics_from_cohort', http_method='GET', name='cohorts.google_genomics_from_cohort') - def google_genomics_from_cohort(self, request): - """ - Returns a list of Google Genomics dataset and readgroupset ids associated with - all the samples in a specified cohort. - Authentication is required. User must have either READER or OWNER permissions on the cohort. - """ - cursor = None - db = None - user_email = None - cohort_id = request.get_assigned_value('cohort_id') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - django_cohort = Django_Cohort.objects.get(id=cohort_id) - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. Error: {}" \ - .format(user_email, cohort_id, e) - request_finished.send(self) - raise endpoints.UnauthorizedException(err_msg) - - query_str = 'SELECT sample_barcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'JOIN cohorts_samples ON metadata_data.sample_barcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY sample_barcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (cohort_id,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [] - for row in cursor.fetchall(): - google_genomics_items.append( - GoogleGenomicsItem( - sample_barcode=row['sample_barcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - ) - - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for cohort {} not found." - .format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for cohort. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, - # path='google_genomics_from_sample', http_method='GET', - # name='cohorts.google_genomics_from_sample') - def google_genomics_from_sample(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - # print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - query_str = 'SELECT sample_barcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE sample_barcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY sample_barcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [] - for row in cursor.fetchall(): - google_genomics_items.append( - GoogleGenomicsItem( - sample_barcode=row['sample_barcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - ) - - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() diff --git a/api/__init__.py b/api/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api/api_helpers.py b/api/api_helpers.py deleted file mode 100755 index 4a2af19d..00000000 --- a/api/api_helpers.py +++ /dev/null @@ -1,508 +0,0 @@ -""" - -Copyright 2018, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import os -import MySQLdb -import httplib2 -from oauth2client.client import GoogleCredentials, AccessTokenCredentials -from django.conf import settings -from googleapiclient.discovery import build - -debug = settings.DEBUG - -WHITELIST_RE = ur'([^\\\_\|\"\+~@:#\$%\^&\*=\-\.,\(\)0-9a-zA-Z\s\xc7\xfc\xe9\xe2\xe4\xe0\xe5\xe7\xea\xeb\xe8\xef\xee\xed\xec\xc4\xc5\xc9\xe6\xc6\xf4\xf6\xf2\xfb\xf9\xd6\xdc\xe1\xf3\xfa\xf1\xd1\xc0\xc1\xc2\xc3\xc8\xca\xcb\xcc\xcd\xce\xcf\xd0\xd2\xd3\xd4\xd5\xd8\xd9\xda\xdb\xdd\xdf\xe3\xf0\xf5\xf8\xfd\xfe\xff])' - -MOLECULAR_CATEGORIES = { - 'nonsilent': [ - 'Missense_Mutation', - 'Nonsense_Mutation', - 'Nonstop_Mutation', - 'Frame_Shift_Del', - 'Frame_Shift_Ins', - 'De_novo_Start_OutOfFrame', - 'In_Frame_Del', - 'In_Frame_Ins', - 'Start_Codon_SNP', - 'Start_Codon_Del', - ] -} - -# Database connection -def sql_connection(): - env = os.getenv('SERVER_SOFTWARE') - database = settings.DATABASES['default'] - connect_options = {} - - db = None - try: - if not settings.IS_DEV: - # Connecting from App Engine - connect_options['host'] = 'localhost' - connect_options['unix_socket'] = database['HOST'] - connect_options['db'] = database['NAME'] - connect_options['user'] = database['USER'] - connect_options['passwd'] = database['PASSWORD'] - else: - connect_options['host'] = '127.0.0.1' - connect_options['port'] = 3306 - connect_options['db'] = database['NAME'] - connect_options['user'] = database['USER'] - connect_options['passwd'] = database['PASSWORD'] - - db = MySQLdb.connect(**connect_options) - - except Exception as e: - print >> sys.stderr, "[ERROR] Exception in sql_connection(): " + e.message - print >> sys.stderr, traceback.format_exc() - - return db - - -def sql_bmi_by_ranges(value): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - if 'None' in value: - result += 'BMI is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == 'underweight': - result += ' (BMI < 18.5)' - elif str(val) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(val) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(val) == 'obese': - result += ' (BMI > 29.9)' - - else: - # value is a single range - if str(value) == 'underweight': - result += ' (BMI < 18.5)' - elif str(value) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(value) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(value) == 'obese': - result += ' (BMI > 29.9)' - - return result - - -def sql_age_by_ranges(value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - #value is a list of ranges - first = True - if 'None' in value: - result += 'age_at_initial_pathologic_diagnosis is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == '10 to 39': - result += ' (age_at_initial_pathologic_diagnosis >= 10 and age_at_initial_pathologic_diagnosis < 40)' - elif str(val) == '40 to 49': - result += ' (age_at_initial_pathologic_diagnosis >= 40 and age_at_initial_pathologic_diagnosis < 50)' - elif str(val) == '50 to 59': - result += ' (age_at_initial_pathologic_diagnosis >= 50 and age_at_initial_pathologic_diagnosis < 60)' - elif str(val) == '60 to 69': - result += ' (age_at_initial_pathologic_diagnosis >= 60 and age_at_initial_pathologic_diagnosis < 70)' - elif str(val) == '70 to 79': - result += ' (age_at_initial_pathologic_diagnosis >= 70 and age_at_initial_pathologic_diagnosis < 80)' - elif str(val).lower() == 'over 80': - result += ' (age_at_initial_pathologic_diagnosis >= 80)' - else: - #value is a single range - if str(value) == '10 to 39': - result += ' (age_at_initial_pathologic_diagnosis >= 10 and age_at_initial_pathologic_diagnosis < 40)' - elif str(value) == '40 to 49': - result += ' (age_at_initial_pathologic_diagnosis >= 40 and age_at_initial_pathologic_diagnosis < 50)' - elif str(value) == '50 to 59': - result += ' (age_at_initial_pathologic_diagnosis >= 50 and age_at_initial_pathologic_diagnosis < 60)' - elif str(value) == '60 to 69': - result += ' (age_at_initial_pathologic_diagnosis >= 60 and age_at_initial_pathologic_diagnosis < 70)' - elif str(value) == '70 to 79': - result += ' (age_at_initial_pathologic_diagnosis >= 70 and age_at_initial_pathologic_diagnosis < 80)' - elif str(value).lower() == 'over 80': - result += ' (age_at_initial_pathologic_diagnosis >= 80)' - elif str(value) == 'None': - result += ' age_at_initial_pathologic_diagnosis is null' - - return result - -def gql_age_by_ranges(q, key, value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - for val in value: - if first: - first = False - else: - result += ' or' - if str(val) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(val) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(val) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(val) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(val) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(val).lower() == 'over80': - result += ' (%s >= 80)' % key - else: - # value is a single range - if str(value) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(value) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(value) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(value) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(value) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(value).lower() == 'over80': - result += ' (%s >= 80)' % key - return result - - -def normalize_bmi(bmis): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - bmi_list = {'underweight': 0, 'normal weight': 0, 'overweight': 0, 'obese': 0, 'None': 0} - for bmi, count in bmis.items(): - if type(bmi) != dict: - if bmi and bmi != 'None': - fl_bmi = float(bmi) - if fl_bmi < 18.5: - bmi_list['underweight'] += int(count) - elif 18.5 <= fl_bmi <= 24.9: - bmi_list['normal weight'] += int(count) - elif 25 <= fl_bmi <= 29.9: - bmi_list['overweight'] += int(count) - elif fl_bmi >= 30: - bmi_list['obese'] += int(count) - else: - bmi_list['None'] += int(count) - - return bmi_list - - -def normalize_ages(ages): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age, count in ages.items(): - if type(age) != dict: - if age and age != 'None': - int_age = float(age) - if int_age < 40: - new_age_list['10 to 39'] += int(count) - elif int_age < 50: - new_age_list['40 to 49'] += int(count) - elif int_age < 60: - new_age_list['50 to 59'] += int(count) - elif int_age < 70: - new_age_list['60 to 69'] += int(count) - elif int_age < 80: - new_age_list['70 to 79'] += int(count) - else: - new_age_list['Over 80'] += int(count) - else: - new_age_list['None'] += int(count) - else: - print age - - return new_age_list - -def applyFilter(field, dict): -# this one gets called a lot... -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - query_dict = dict.copy() - if field in dict: - query_dict.pop(field, None) - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - else: - where_clause = None - else: - where_clause = build_where_clause(dict) - - return where_clause - - -def build_where_clause(filters, alt_key_map=False): -# this one gets called a lot -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - first = True - query_str = '' - big_query_str = '' # todo: make this work for non-string values -- use {}.format - value_tuple = () - key_order = [] - keyType = None - gene = None - - grouped_filters = None - - for key, value in filters.items(): - if isinstance(value, dict) and 'values' in value: - value = value['values'] - - if isinstance(value, list) and len(value) == 1: - value = value[0] - # Check if we need to map to a different column name for a given key - if alt_key_map and key in alt_key_map: - key = alt_key_map[key] - - # Multitable where's will come in with : in the name. Only grab the column piece for now - # TODO: Shouldn't throw away the entire key - elif ':' in key: - keyType = key.split(':')[0] - if keyType == 'MUT': - gene = key.split(':')[1] - key = key.split(':')[-1] - - # Multitable filter lists don't come in as string as they can contain arbitrary text in values - elif isinstance(value, basestring): - # If it's a list of values, split it into an array - if ',' in value: - value = value.split(',') - - key_order.append(key) - - # Bucket the grouped filter types (currently just certain has_ values, could be more) - if 'has_' in key and not key == 'has_Illumina_DNASeq' and not key == 'has_SNP6' and not key == 'has_RPPA': - if grouped_filters is None: - grouped_filters = {} - - if key == 'has_27k' or key == 'has_450k': - if 'DNA_methylation' not in grouped_filters: - grouped_filters['DNA_methylation'] = [] - grouped_filters['DNA_methylation'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_HiSeq_miRnaSeq' or key == 'has_GA_miRNASeq': - if 'miRNA_sequencing' not in grouped_filters: - grouped_filters['miRNA_sequencing'] = [] - grouped_filters['miRNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_UNC_HiSeq_RNASeq' or key == 'has_UNC_GA_RNASeq' or key == 'has_BCGSC_HiSeq_RNASeq' or key == 'has_BCGSC_GA_RNASeq': - if 'RNA_sequencing' not in grouped_filters: - grouped_filters['RNA_sequencing'] = [] - grouped_filters['RNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - # BQ-only format - elif keyType == 'MUT': - # If it's first in the list, don't append an "and" - params = {} - value_tuple += (params,) - - if first: - first = False - else: - big_query_str += ' AND' - - big_query_str += " %s = '{hugo_symbol}' AND " % 'Hugo_Symbol' - params['gene'] = gene - - if(key == 'category'): - if value == 'any': - big_query_str += '%s IS NOT NULL' % 'Variant_Classification' - params['var_class'] = '' - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = MOLECULAR_CATEGORIES[value] - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = value - - if value != 'any': - if isinstance(values, list): - j = 0 - for vclass in values: - if j == 0: - params['var_class'] = "'%s'" % vclass.replace("'", "\\'") - j = 1 - else: - params['var_class'] += ",'%s'" % vclass.replace("'", "\\'") - else: - params['var_class'] = "'%s'" % values.replace("'", "\\'") - - else: - # If it's first in the list, don't append an "and" - if first: - first = False - else: - query_str += ' and' - big_query_str += ' and' - - # If it's age ranges, give it special treament due to normalizations - if key == 'age_at_initial_pathologic_diagnosis': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_age_by_ranges(value) + ') ' - # If it's age ranges, give it special treament due to normalizations - elif key == 'BMI': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_bmi_by_ranges(value) + ') ' - # If it's a list of items for this key, create an or subclause - elif isinstance(value, list): - has_null = False - if 'None' in value: - has_null = True - query_str += ' (%s is null or' % key - big_query_str += ' (%s is null or' % key - value.remove('None') - query_str += ' %s in (' % key - big_query_str += ' %s in (' % key - i = 0 - for val in value: - value_tuple += (val.strip(),) if type(val) is unicode else (val,) - if i == 0: - query_str += '%s' - big_query_str += '"' + str(val) + '"' - i += 1 - else: - query_str += ',%s' - big_query_str += ',' + '"' + str(val) + '"' - query_str += ')' - big_query_str += ')' - if has_null: - query_str += ')' - big_query_str += ')' - - # If it's looking for None values - elif value == 'None': - query_str += ' %s is null' % key - big_query_str += ' %s is null' % key - - # For the general case - else: - if key == 'fl_archive_name': - big_query_str += ' %s like' % key - big_query_str += ' "%' + value + '%"' - elif key == 'fl_data_level': - big_query_str += ' %s=%s' % (key, value) - elif type(value) == bool: - big_query_str += ' %s=%r' % (key, value) - else: - query_str += ' %s=' % key - big_query_str += ' %s=' % key - query_str += '%s' - big_query_str += '"%s"' % value - value_tuple += (value.strip(),) if type(value) is unicode else (value,) - - # Handle our data buckets - if grouped_filters: - for bucket in grouped_filters: - if not query_str == '': - query_str += ' and ' - big_query_str += ' and ' - - query_str += '( ' - big_query_str += '( ' - - first = True - for filter in grouped_filters[bucket]: - if first: - first = False - else: - query_str += ' or ' - big_query_str += ' or ' - - query_str += ' %s=' % filter['filter'] - big_query_str += ' %s=' % filter['filter'] - query_str += '%s' - big_query_str += '"%s"' % filter['value'] - value_tuple += (filter['value'].strip(),) if type(filter['value']) is unicode else (filter['value'],) - - query_str += ' )' - big_query_str += ' )' - - return {'query_str': query_str, 'value_tuple': value_tuple, 'key_order': key_order, 'big_query_str': big_query_str} - - -def possible_future_authorization_function(): - # will put a decorator on this to ensure user has correct authorization before running - # such as if they are dbgap authorized - from oauth2client.client import flow_from_clientsecrets - from oauth2client.file import Storage - from oauth2client import tools - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - flow = flow_from_clientsecrets(settings.CLIENT_SECRETS, scope='https://www.googleapis.com/auth/bigquery') - ## in future, make storage file temporary somehow? - storage = Storage('bigquery_credentials.dat') - credentials = storage.get() - - if credentials is None or credentials.invalid: - credentials = tools.run_flow(flow, storage, tools.argparser.parse_args([])) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - return service - - -def authorize_credentials_with_Google(): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - # credentials = GoogleCredentials.get_application_default().create_scoped(SCOPES) - credentials = GoogleCredentials.from_stream(settings.GOOGLE_APPLICATION_CREDENTIALS).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - if debug: print >> sys.stderr,' big query authorization '+sys._getframe().f_code.co_name - return service - -# TODO refactor to remove duplicate code -def authorize_credentials_with_google_from_file(credentials_path): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - credentials = GoogleCredentials.from_stream(credentials_path).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - - return service - - -def get_user_email_from_token(access_token): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - user_email = None - credentials = AccessTokenCredentials(access_token, 'test-user') - http = credentials.authorize(httplib2.Http()) - user_info_service = build('oauth2', 'v2', http=http) - user_info = user_info_service.userinfo().get().execute() - if 'email' in user_info: - user_email = user_info['email'] - return user_email diff --git a/api/data_access.py b/api/data_access.py deleted file mode 100755 index a482033d..00000000 --- a/api/data_access.py +++ /dev/null @@ -1,542 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import json -import math -import traceback - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import BooleanField, EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.feature_value_types import ValueType, is_log_transformable -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_tcga_only, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess -from bq_data_access.utils import DurationLogged -from bq_data_access.data_access import FeatureIdQueryDescription - -from api.pairwise import PairwiseInputVector, Pairwise -from api.pairwise_api import PairwiseResults, PairwiseResultVector, PairwiseFilterMessage -from api.api_helpers import sql_connection - -from projects.models import Project - -import sys - -logger = logging.getLogger(__name__) - -VIZ_UNIT_DATADICTIONARY = { - 'BMI': 'kg/m^2', -} - -ISB_CGC_PROJECTS = { - 'list': [] -} - -# DUPLICATE METHOD -# Due to the way sql connections are done, it's easiest to duplicate this method and the static variable -# it creates. The original is in Cohorts/views, and all changes will happen there first. -# -# Generate the ISB_CGC_PROJECTS['list'] value set based on the get_isbcgc_project_set sproc -def fetch_isbcgc_project_set(): - try: - cursor = None - db = sql_connection() - if not ISB_CGC_PROJECTS['list'] or len(ISB_CGC_PROJECTS['list']) <= 0: - cursor = db.cursor() - cursor.execute("SELECT COUNT(SPECIFIC_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE SPECIFIC_NAME = 'get_isbcgc_project_set';") - # Only try to fetch the study set if the sproc exists - if cursor.fetchall()[0][0] > 0: - cursor.execute("CALL get_isbcgc_project_set();") - ISB_CGC_PROJECTS['list'] = [] - for row in cursor.fetchall(): - ISB_CGC_PROJECTS['list'].append(row[0]) - else: - # Otherwise just warn - logger.warn("[WARNING] Stored procedure get_isbcgc_project_set was not found!") - - return ISB_CGC_PROJECTS['list'] - except Exception as e: - logger.error(e) - logger.error(traceback.format_exc()) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - -def get_axis_units(xAttr, yAttr): - units = {'x': '', 'y': ''} - - checkUnits = {} - if xAttr is not None: - checkUnits[xAttr] = 'x' - if yAttr is not None: - checkUnits[yAttr] = 'y' - - for attr in checkUnits: - - if '_age' in attr or 'age_' in attr or 'year_' in attr: - units[checkUnits[attr]] = 'years' - elif '_days' in attr or 'days_' in attr: - units[checkUnits[attr]] = 'days' - elif 'percent' in attr: - units[checkUnits[attr]] = 'percent' - elif 'CNVR:' in attr: - units[checkUnits[attr]] = 'log(CN/2)' - elif 'RPPA:' in attr: - units[checkUnits[attr]] = 'protein expression' - elif 'METH:' in attr: - units[checkUnits[attr]] = 'beta value' - elif 'GEXP:' in attr or 'MIRN:' in attr or ('GNAB:' in attr and "num_mutations" in attr): - units[checkUnits[attr]] = 'count' - elif attr.split(':')[1] in VIZ_UNIT_DATADICTIONARY: - units[checkUnits[attr]] = VIZ_UNIT_DATADICTIONARY[attr.split(':')[1]] - - return units - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, required=True) - - -class DataPoint(Message): - patient_id = StringField(1) - sample_id = StringField(2) - aliquot_id = StringField(3) - value = StringField(4) - - -class DataPointList(Message): - type = EnumField(ValueType, 1) - items = MessageField(DataPoint, 2, repeated=True) - - -class PlotDataRequest(Message): - x_id = StringField(1, required=True) - y_id = StringField(2, required=False) - c_id = StringField(3, required=False) - log_transform = StringField(4, required=False) - cohort_id = IntegerField(5, repeated=True) - pairwise = BooleanField(6, required=False) - - -class PlotDataPointCohortMemberships(Message): - ids = IntegerField(1, repeated=True) - - -class PlotDataCohortInfo(Message): - id = IntegerField(1, required=True) - name = StringField(2, required=True) - -DATAPOINT_COHORT_THRESHOLD = 1 - - -class PlotDataPoint(Message): - sample_id = StringField(1) - case_id = StringField(2) - x = StringField(3) - y = StringField(4) - c = StringField(5) - cohort = IntegerField(6, repeated=True) - - -class PlotDataTypes(Message): - x = EnumField(ValueType, 1) - y = EnumField(ValueType, 2) - c = EnumField(ValueType, 3) - - -class PlotDataFeatureLabels(Message): - x = StringField(1) - y = StringField(2) - c = StringField(3) - - -class PlotDatapointCohortSet(Message): - datapoint_id = StringField(1, required=True) - - -class PlotDatapointCount(Message): - total_num_patients = IntegerField(1, required=True) - total_num_samples = IntegerField(2, required=True) - num_patients_w_xy = IntegerField(3, required=True) - num_samples_w_xy = IntegerField(4, required=True) - num_patients_wo_x = IntegerField(5, required=True) - num_samples_wo_x = IntegerField(6, required=True) - num_patients_wo_y = IntegerField(7, required=True) - num_samples_wo_y = IntegerField(8, required=True) - - -class PlotDataResponse(Message): - types = MessageField(PlotDataTypes, 1, required=True) - labels = MessageField(PlotDataFeatureLabels, 2, required=True) - items = MessageField(PlotDataPoint, 3, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - counts = MessageField(PlotDatapointCount, 5) - pairwise_result = MessageField(PairwiseResults, 6, required=False) - xUnits = StringField(7, required=False) - yUnits = StringField(8, required=False) - - -FeatureDataEndpointsAPI = endpoints_api(name='feature_data_api', version='v1', - description='Endpoints for feature data used by the web application.') - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class FeatureDataEndpoints(remote.Service): - def get_counts(self, data): - total_num_patients = [] - total_num_samples = [] - num_samples_w_xy = [] - num_patients_w_xy = [] - num_samples_wo_x = [] - num_samples_wo_y = [] - num_patients_wo_x = [] - num_patients_wo_y = [] - - result = {} - - for item in data: - total_num_samples.append(item['sample_id']) - total_num_patients.append(item['sample_id'][:12]) - - if item['x'] != 'NA' and item['y'] != 'NA': - num_samples_w_xy.append(item['sample_id']) - num_patients_w_xy.append(item['sample_id'][:12]) - else: - if item['x'] == 'NA': - num_samples_wo_x.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_x.append(item['sample_id'][:12]) - elif item['y'] == 'NA': - num_samples_wo_y.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_y.append(item['sample_id'][:12]) - - result['total_num_patients'] = len(set(total_num_patients)) - result['total_num_samples'] = len(set(total_num_samples)) - result['num_patients_w_xy'] = len(set(num_patients_w_xy)) - result['num_samples_w_xy'] = len(set(num_samples_w_xy)) - result['num_patients_wo_x'] = len(set(num_patients_wo_x)) - result['num_samples_wo_x'] = len(set(num_samples_wo_x)) - result['num_patients_wo_y'] = len(set(num_patients_wo_y)) - result['num_samples_wo_y'] = len(set(num_samples_wo_y)) - - return result - - # TODO refactor to separate module - @DurationLogged('PAIRWISE', 'GET') - def get_pairwise_result(self, feature_array): - # Format the feature vectors for pairwise - input_vectors = Pairwise.prepare_feature_vector(feature_array) - outputs = None - results = None - - try: - outputs = Pairwise.run_pairwise(input_vectors) - - if outputs is not None: - results = PairwiseResults(result_vectors=[], filter_messages=[]) - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - except Exception as e: - outputs = None - results = None - logger.error(traceback.format_exc()) - - return results - - @DurationLogged('FEATURE', 'VECTOR_MERGE') - def get_merged_dict_timed(self, vms): - return vms.get_merged_dict() - - # TODO refactor missing value logic out of this module - @DurationLogged('FEATURE', 'GET_VECTORS') - def get_merged_feature_vectors(self, x_id, y_id, c_id, cohort_id_array, logTransform, study_id_array): - """ - Fetches and merges data for two or three feature vectors (see parameter documentation below). - The vectors have to be an array of dictionaries, with each dictionary containing a 'value' field - (other fields are ignored): - [ - { - 'value': 0.5 - }, - { - 'value': 1.0 - } - ] - The merged result: - [ - { - 'patient_id': - 'x': - 'y': - 'c': - }, - { - 'patient_id': - 'x': - 'y': - 'c': - } - ... - ] - - :param x_id: Feature identifier for x-axis e.g. 'CLIN:age_at_initial_pathologic_diagnosis' - :param y_id: Feature identifier for y-axis. If None, values for 'y' in the response will be marked as missing. - :param c_id: Feature identifier for color-by. If None, values for 'c' in the response will be marked as missing. - :param cohort_id_array: Cohort identifier array. - - :return: PlotDataResponse - """ - - async_params = [FeatureIdQueryDescription(x_id, cohort_id_array, study_id_array)] - - c_type, c_vec = ValueType.STRING, [] - y_type, y_vec = ValueType.STRING, [] - - units = get_axis_units(x_id, y_id) - - if c_id is not None: - async_params.append(FeatureIdQueryDescription(c_id, cohort_id_array, study_id_array)) - if y_id is not None: - async_params.append(FeatureIdQueryDescription(y_id, cohort_id_array, study_id_array)) - - async_result = get_feature_vectors_tcga_only(async_params) - - if c_id is not None: - c_type, c_vec = async_result[c_id]['type'], async_result[c_id]['data'] - if y_id is not None: - y_type, y_vec = async_result[y_id]['type'], async_result[y_id]['data'] - if logTransform is not None and logTransform['y'] and y_vec and is_log_transformable(y_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum y-value - ''' - yvals = [] - for yd in y_vec: - if 'value' in yd and yd['value'] is not None and yd['value'] != "NA" and yd['value'] != "None": - yvals.append(float(yd['value'])) - y_min = min(yvals) - ''' - for ydata in y_vec: - if 'value' in ydata and ydata['value'] is not None and ydata['value'] != "NA" and ydata['value'] != "None": - if float(ydata['value']) < 0: - ydata['value'] = "NA" - elif logTransform['yBase'] == 10: - ydata['value'] = str(math.log10((float(ydata['value']) + 1))) - elif logTransform['yBase'] == 'e': - ydata['value'] = str(math.log((float(ydata['value']) + 1))) - elif type(logTransform['yBase']) is int: - ydata['value'] = str(math.log((float(ydata['value']) + 1), logTransform['yBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - x_type, x_vec = async_result[x_id]['type'], async_result[x_id]['data'] - - if logTransform is not None and logTransform['x'] and x_vec and is_log_transformable(x_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum x-value - ''' - xvals = [] - for xd in x_vec: - if 'value' in xd and xd['value'] is not None and xd['value'] != "NA" and xd['value'] != "None": - xvals.append(float(xd['value'])) - x_min = min(xvals) - ''' - - for xdata in x_vec: - if 'value' in xdata and xdata['value'] is not None and xdata['value'] != "NA" and xdata['value'] != "None": - if float(xdata['value']) < 0: - xdata['value'] = "NA" - elif logTransform['xBase'] == 10: - xdata['value'] = str(math.log10((float(xdata['value']) + 1))) - elif logTransform['xBase'] == 'e': - xdata['value'] = str(math.log((float(xdata['value']) + 1))) - elif type(logTransform['xBase']) is int: - xdata['value'] = str(math.log((float(xdata['value']) + 1), logTransform['xBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', ['x', 'y', 'c']) # changed so that it plots per sample not patient - vms.add_dict_array(x_vec, 'x', 'value') - vms.add_dict_array(y_vec, 'y', 'value') - vms.add_dict_array(c_vec, 'c', 'value') - merged = self.get_merged_dict_timed(vms) - - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(id=item['id'], name=item['name'])) - - items = [] - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - - if len(cohort_set) >= DATAPOINT_COHORT_THRESHOLD: - value_bundle['cohort'] = cohort_set - - items.append(PlotDataPoint(**value_bundle)) - - counts = self.get_counts(merged) - count_message = PlotDatapointCount(**counts) - - type_message = PlotDataTypes(x=x_type, y=y_type, c=c_type) - - # TODO assign label for y if y_id is None, as in that case the y-field will be missing from the response - label_message = PlotDataFeatureLabels(x=x_id, y=y_id, c=c_id) - - # TODO Refactor pairwise call to separate function - # Include pairwise results - input_vectors = [PairwiseInputVector(x_id, x_type, x_vec)] - if c_id is not None: - input_vectors.append(PairwiseInputVector(c_id, c_type, c_vec)) - if y_id is not None: - input_vectors.append(PairwiseInputVector(y_id, y_type, y_vec)) - - - pairwise_result = None - - if len(input_vectors) > 1: - pairwise_result = self.get_pairwise_result(input_vectors) - - if pairwise_result is None: - logger.warn("[WARNING] Pairwise results not included in returned object") - - return PlotDataResponse(types=type_message, labels=label_message, items=items, - cohort_set=cohort_info_obj_array, - counts=count_message, pairwise_result=pairwise_result, xUnits=units['x'], yUnits=units['y']) - - def get_feature_id_validity_for_array(self, feature_id_array): - """ - For each feature identifier in an array, check whether or not the identifier is - valid. - - Args: - feature_id_array: - - Returns: - Array of tuples - (feature identifier, ) - """ - result = [] - for feature_id in feature_id_array: - result.append((feature_id, is_valid_feature_identifier(feature_id))) - - return result - - @endpoints_method(PlotDataRequest, PlotDataResponse, - path='feature_data_plot', http_method='GET', name='feature_access.getFeatureDataForPlot') - def data_access_for_plot(self, request): - """ Used by the web application.""" - try: - x_id = request.x_id - y_id = request.y_id - c_id = request.c_id - logTransform = json.loads(request.log_transform) - cohort_id_array = request.cohort_id - - # Check that all requested feature identifiers are valid. Do not check for y_id if it is not - # supplied in the request. - feature_ids_to_check = [x_id] - if c_id is not None: - feature_ids_to_check.append(c_id) - if y_id is not None: - feature_ids_to_check.append(y_id) - - valid_features = self.get_feature_id_validity_for_array(feature_ids_to_check) - - for feature_id, is_valid in valid_features: - logging.info((feature_id, is_valid)) - if not is_valid: - logging.error("Invalid internal feature ID '{}'".format(feature_id)) - raise NotFoundException() - - # Get the project IDs these cohorts' samples come from - cohort_vals = () - cohort_params = "" - - for cohort in cohort_id_array: - cohort_params += "%s," - cohort_vals += (cohort,) - - cohort_params = cohort_params[:-1] - - db = sql_connection() - cursor = db.cursor() - - tcga_studies = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN ("+cohort_params+");",cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_study_ids = [] - unconfirmed_study_ids = [] - - for row in cursor.fetchall(): - if row[0] in tcga_studies: - if row[0] not in confirmed_study_ids: - confirmed_study_ids.append(row[0]) - elif row[0] not in unconfirmed_study_ids: - unconfirmed_study_ids.append(row[0]) - - if len(unconfirmed_study_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_study_ids) - - for project in projects: - if project.get_my_root_and_depth()['root'] in tcga_studies: - confirmed_study_ids.append(project.id) - - return self.get_merged_feature_vectors(x_id, y_id, c_id, cohort_id_array, logTransform, confirmed_study_ids) - except NotFoundException as nfe: - # Pass through NotFoundException so that it is not handled as Exception below. - raise nfe - except Exception as e: - logger.exception(e) - raise InternalServerErrorException() - diff --git a/api/feature_access.py b/api/feature_access.py deleted file mode 100755 index 1e987916..00000000 --- a/api/feature_access.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import BadRequestException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import Message, MessageField, StringField - -from bq_data_access.feature_search.common import BackendException, EmptyQueryException, InvalidFieldException, InvalidDataTypeException -from bq_data_access.gexp_data import GEXP_FEATURE_TYPE -from bq_data_access.clinical_data import CLINICAL_FEATURE_TYPE -from bq_data_access.methylation_data import METH_FEATURE_TYPE -from bq_data_access.copynumber_data import CNVR_FEATURE_TYPE -from bq_data_access.protein_data import RPPA_FEATURE_TYPE -from bq_data_access.mirna_data import MIRN_FEATURE_TYPE -from bq_data_access.gnab_data import GNAB_FEATURE_TYPE -from bq_data_access.feature_search.gexp_searcher import GEXPSearcher -from bq_data_access.feature_search.clinical_searcher import ClinicalSearcher -from bq_data_access.feature_search.methylation_searcher import METHSearcher -from bq_data_access.feature_search.copynumber_search import CNVRSearcher -from bq_data_access.feature_search.protein import RPPASearcher -from bq_data_access.feature_search.microrna_searcher import MIRNSearcher -from bq_data_access.feature_search.gnab_searcher import GNABSearcher - -class ClinicalFeatureType(Message): - feature_type = StringField(1) - gene = StringField(2) - label = StringField(3) - internal_id = StringField(4) - -class FeatureTypesRequest(Message): - keyword = StringField(1, required=True) - -class FeatureDataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = StringField(2, required=True) - -class FeatureTypeSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=False) - gene_name = StringField(3, required=False) - platform = StringField(4, required=False) - center = StringField(5, required=False) - protein_name = StringField(6, required=False) - value_field = StringField(7, required=False) - probe_name = StringField(8, required=False) - relation_to_gene = StringField(9, required=False) - relation_to_island = StringField(10, required=False) - mirna_name = StringField(11, required=False) - -class FeatureSearchResult(Message): - feature_type = StringField(1) - internal_feature_id = StringField(2) - label = StringField(3) - type = StringField(4) - -class FeatureTypeList(Message): - items = MessageField(FeatureSearchResult, 1, repeated=True) - -class FeatureTypeFieldSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=True) - field = StringField(3, required=True) - -class FeatureFieldSearchResult(Message): - values = StringField(1, repeated=True) - - -class FeatureDefinitionSearcherFactory(object): - @classmethod - def build_from_datatype(cls, datatype): - if datatype == CLINICAL_FEATURE_TYPE: - return ClinicalSearcher() - elif datatype == GEXP_FEATURE_TYPE: - return GEXPSearcher() - elif datatype == METH_FEATURE_TYPE: - return METHSearcher() - elif datatype == CNVR_FEATURE_TYPE: - return CNVRSearcher() - elif datatype == RPPA_FEATURE_TYPE: - return RPPASearcher() - elif datatype == MIRN_FEATURE_TYPE: - return MIRNSearcher() - elif datatype == GNAB_FEATURE_TYPE: - return GNABSearcher() - #TODO build a full search on all features - #elif datatype == ALL: - # return FullSearcher() - raise InvalidDataTypeException("Invalid datatype '{datatype}'".format(datatype=datatype)) - -FeatureAccessEndpointsAPI = endpoints_api(name='feature_type_api', version='v1', - description='Endpoints used by the web application to return features.') -@FeatureAccessEndpointsAPI.api_class(resource_name='feature_type_endpoints') -class FeatureAccessEndpoints(remote.Service): - @endpoints_method(FeatureTypeSearchRequest, FeatureTypeList, - path='feature_search', http_method='GET', name='feature_access.FeatureSearch') - def feature_search(self, request): - """ Used by the web application.""" - try: - datatype = request.datatype - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - parameters = {} - for message in request.all_fields(): - field_name = message.name - if field_name != 'datatype': - value = request.get_assigned_value(field_name) - if value is not None: - parameters[field_name] = value - - result = searcher.search(parameters) - items = [] - fields = ['label', 'internal_feature_id', 'feature_type'] - for row in result: - obj = {key: row[key] for key in fields} - if obj['feature_type'] == 'CLIN': - obj['type'] = row['type'] - items.append(obj) - - return FeatureTypeList(items=items) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except EmptyQueryException as e: - logging.error("Empty query: %s", str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error("Invalid field: %s", str(e)) - raise BadRequestException(str(e)) - except BackendException: - logging.exception("feature_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - - @endpoints_method(FeatureTypeFieldSearchRequest, FeatureFieldSearchResult, - path='feature_field_search', http_method='GET', name='feature_access.getFeatureFieldSearch') - def feature_field_search(self, request): - """ Used by the web application.""" - try: - datatype, keyword, field = request.datatype, request.keyword, request.field - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - result = searcher.field_value_search(keyword, field) - return FeatureFieldSearchResult(values=result) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error(str(e)) - raise BadRequestException() - except BackendException: - logging.exception("feature_field_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api/isb_cgc_api/__init__.py b/api/isb_cgc_api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/api/isb_cgc_api/aliquots_annotations.py b/api/isb_cgc_api/aliquots_annotations.py deleted file mode 100644 index 890bcd82..00000000 --- a/api/isb_cgc_api/aliquots_annotations.py +++ /dev/null @@ -1,149 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message -from message_classes import MetadataAnnotationItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class MetadataAnnotationList(messages.Message): - items = messages.MessageField(MetadataAnnotationItem, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -class AliquotsAnnotationsQueryBuilder(object): - - @staticmethod - def build_query(item_type_name=None): - query_str = 'select * ' \ - 'from metadata_annotation ' \ - 'where AliquotBarcode=%s ' - if len(item_type_name) > 0: - query_str += 'and itemTypeName in (' + ', '.join(['%s']*len(item_type_name)) + ')' - - - return query_str - - @staticmethod - def build_metadata_samples_query(): - query_str = 'select * ' \ - 'from metadata_data ' \ - 'where AliquotBarcode=%s ' - - return query_str - -@ISB_CGC_Endpoints.api_class(resource_name='aliquots') -class AliquotsAnnotationAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(aliquot_barcode=messages.StringField(1, required=True), - item_type_name=messages.StringField(2, repeated=True)) - - @endpoints.method(GET_RESOURCE, MetadataAnnotationList, - path='aliquots/{aliquot_barcode}/annotations', http_method='GET') - def annotations(self, request): - """ - Returns TCGA annotations about a specific aliquot, - Takes a aliquot barcode (of length 28 *eg* TCGA-01-0628-11A-01D-0358-06) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - aliquot_barcode = request.get_assigned_value('aliquot_barcode') - query_tuple = (str(aliquot_barcode),) - # check to make sure aliquot_barcode is in correct form - try: - parts = aliquot_barcode.split('-') - assert len(parts) == 7 - assert len(parts[0]) == 4 - assert len(parts[1]) == 2 - assert len(parts[2]) == 4 - assert len(parts[3]) == 3 - assert len(parts[4]) in [2, 3] - assert len(parts[5]) == 4 - assert len(parts[6]) == 2 - except AssertionError: - raise endpoints.BadRequestException('{} is not the correct format for a aliquot barcode. ' - 'Aliquot barcodes must be of the form XXXX-XX-XXXX-XXX-XXX-XXXX-XX ' - 'or XXXX-XX-XXXX-XXX-XX-XXXX-XX.'.format(aliquot_barcode)) - - item_type_name = request.get_assigned_value('item_type_name') - - # check to make sure each item_type_name is valid - if len(item_type_name) > 0: - for itm in item_type_name: - itm = itm.strip() - if itm.lower() not in ['patient', 'aliquot', 'analyte', 'shipped portion', 'portion', 'slide', 'sample']: - raise endpoints.BadRequestException("'{}' is not a valid entry for item_type_name. " - "Valid entries include 'Patient', 'Aliquot', 'Analyte', 'Shipped Portion', " - "'Portion', 'Slide', and 'Sample'".format(itm)) - query_tuple += (itm,) - - query_str = AliquotsAnnotationsQueryBuilder().build_query(item_type_name=item_type_name) - metadata_samples_query_str = AliquotsAnnotationsQueryBuilder().build_metadata_samples_query() - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build annotation message - cursor.execute(query_str, query_tuple) - rows = cursor.fetchall() - cursor.execute(metadata_samples_query_str, (str(aliquot_barcode),)) - metadata_sample_rows = cursor.fetchall() - if len(rows) == 0: - cursor.close() - db.close() - if len(metadata_sample_rows) == 0: - msg = "Aliquot barcode {} not found in the database.".format(aliquot_barcode) - logger.info(msg) - else: - msg = "No annotations found for aliquot barcode {}".format(aliquot_barcode) - if item_type_name is not None: - msg += " and item type name {}. Item type name must be one of the following: " \ - "'Patient', 'Aliquot', 'Analyte', 'Shipped Portion', 'Portion', 'Slide', 'Sample'.".format(item_type_name) - logger.info(msg) - raise endpoints.NotFoundException(msg) - - items = [] - for row in rows: - constructor_dict = build_constructor_dict_for_message(MetadataAnnotationItem(), row) - items.append(MetadataAnnotationItem(**constructor_dict)) - - return MetadataAnnotationList(items=items, count=len(items)) - - except (IndexError, TypeError), e: - logger.info("Aliquot {} not found. Error: {}".format(aliquot_barcode, e)) - raise endpoints.NotFoundException("Aliquot {} not found.".format(aliquot_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_cloudstoragefilepaths.py b/api/isb_cgc_api/cohorts_cloudstoragefilepaths.py deleted file mode 100644 index 849c9477..00000000 --- a/api/isb_cgc_api/cohorts_cloudstoragefilepaths.py +++ /dev/null @@ -1,115 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb -import django - -from django.conf import settings -from django.core.signals import request_finished -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsSamplesFilesQueryBuilder, CohortsSamplesFilesMessageBuilder -from api.api_helpers import sql_connection -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GCSFilePathList(messages.Message): - cloud_storage_file_paths = messages.StringField(1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsCloudStorageFilePathsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - limit=messages.IntegerField(2), - platform=messages.StringField(3), - pipeline=messages.StringField(4)) - - @endpoints.method(GET_RESOURCE, GCSFilePathList, http_method='GET', - path='cohorts/{cohort_id}/cloud_storage_file_paths') - def cloud_storage_file_paths(self, request): - """ - Takes a cohort id as a required parameter and returns cloud storage paths to files - associated with all the samples in that cohort, up to a default limit of 10,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - user_email = None - cursor = None - db = None - - limit = request.get_assigned_value('limit') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - cohort_id = request.get_assigned_value('cohort_id') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - Django_Cohort.objects.get(id=cohort_id) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. " \ - "Error: {}".format(user_email, cohort_id, e) - raise endpoints.UnauthorizedException(err_msg) - finally: - request_finished.send(self) - - query_str, query_tuple = CohortsSamplesFilesQueryBuilder().build_query( - platform=platform, pipeline=pipeline, limit=limit, cohort_id=cohort_id) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - cursor_rows = cursor.fetchall() - bad_repo_count, bad_repo_set = CohortsSamplesFilesMessageBuilder().get_GCS_file_paths_and_bad_repos(cursor_rows) - cloud_storage_path_list = [row['cloud_storage_path'] for row in cursor_rows] - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return GCSFilePathList(cloud_storage_file_paths=cloud_storage_path_list, count=len(cloud_storage_path_list)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving file paths. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_create.py b/api/isb_cgc_api/cohorts_create.py deleted file mode 100644 index f7a2b3e0..00000000 --- a/api/isb_cgc_api/cohorts_create.py +++ /dev/null @@ -1,174 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import re -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages -from datetime import datetime -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsCreatePreviewQueryBuilder, \ - are_there_bad_keys, are_there_no_acceptable_keys, construct_parameter_error_message - -from message_classes import MetadataRangesItem - -from api.api_helpers import sql_connection, WHITELIST_RE -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms, Samples, Filters -from bq_data_access.cohort_bigquery import BigQueryCohortSupport - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -class CreatedCohort(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - filters = messages.MessageField(FilterDetails, 4, repeated=True) - patient_count = messages.IntegerField(5, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(6, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsCreateAPI(remote.Service): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, - name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of patients and the number - of samples in that cohort. - """ - cursor = None - db = None - - user = endpoints.get_current_user() - user_email = user.email() if user else None - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - request_finished.send(self) - raise endpoints.BadRequestException(err_msg) - - query_dict, gte_query_dict, lte_query_dict = CohortsCreatePreviewQueryBuilder().build_query_dictionaries(request) - - patient_query_str, sample_query_str, value_tuple = CohortsCreatePreviewQueryBuilder().build_query( - query_dict, gte_query_dict, lte_query_dict) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, value_tuple) - # TODO: We need to adjust this to pull the correct project ID as well - sample_barcodes = [{'sample_barcode': row['sample_barcode'], 'case_barcode': row['case_barcode'], 'project_id': None,} for row in cursor.fetchall()] - - except (IndexError, TypeError), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("Error retrieving samples or patients") - except MySQLdb.ProgrammingError as e: - logger.warn("Error saving cohort. {}".format(e)) - request_finished.send(self) - raise endpoints.BadRequestException("Error saving cohort. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - cohort_name = request.get_assigned_value('name') - - # Validate the cohort name against a whitelist - whitelist = re.compile(WHITELIST_RE, re.UNICODE) - match = whitelist.search(unicode(cohort_name)) - if match: - # XSS risk, log and fail this cohort save - match = whitelist.findall(unicode(cohort_name)) - logger.error( - '[ERROR] While saving a cohort, saw a malformed name: ' + cohort_name + ', characters: ' + match.__str__()) - raise endpoints.BadRequestException( - "Your cohort's name contains invalid characters (" + match.__str__() + "); please choose another name.") - - if len(sample_barcodes) == 0: - raise endpoints.BadRequestException( - "The cohort could not be saved because no samples meet the specified parameters.") - - # todo: maybe create all objects first, then save them all at the end? - # 1. create new cohorts_cohort with name, active=True, last_date_saved=now - created_cohort = Django_Cohort.objects.create(name=cohort_name, active=True, - last_date_saved=datetime.utcnow()) - created_cohort.save() - - # 2. insert samples into cohort_samples - sample_list = [Samples(cohort=created_cohort, sample_barcode=sample['sample_barcode'], case_barcode=sample['case_barcode'], project_id=sample['project_id']) for sample in sample_barcodes] - Samples.objects.bulk_create(sample_list) - - # 3. Set permission for user to be owner - perm = Cohort_Perms(cohort=created_cohort, user=django_user, perm=Cohort_Perms.OWNER) - perm.save() - - # 4. Create filters applied - filter_data = [] - for key, value_list in query_dict.items(): - for val in value_list: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - for key, val in [(k + '_lte', v) for k, v in lte_query_dict.items()] + [(k + '_gte', v) for k, v in gte_query_dict.items()]: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - # 5. Store cohort to BigQuery - project_id = settings.BQ_PROJECT_ID - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - bcs = BigQueryCohortSupport(project_id, cohort_settings.dataset_id, cohort_settings.table_id) - bcs.add_cohort_to_bq(created_cohort.id, sample_barcodes) - - request_finished.send(self) - - return CreatedCohort(id=str(created_cohort.id), - name=cohort_name, - last_date_saved=str(datetime.utcnow()), - filters=filter_data, - patient_count=created_cohort.case_size(), - sample_count=len(sample_barcodes) - ) diff --git a/api/isb_cgc_api/cohorts_delete.py b/api/isb_cgc_api/cohorts_delete.py deleted file mode 100644 index efd6108b..00000000 --- a/api/isb_cgc_api/cohorts_delete.py +++ /dev/null @@ -1,89 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import endpoints -import logging - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class ReturnJSON(messages.Message): - message = messages.StringField(1) - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsDeleteAPI(remote.Service): - DELETE_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - @endpoints.method(DELETE_RESOURCE, ReturnJSON, http_method='DELETE', path='cohorts/{cohort_id}') - def delete(self, request): - """ - Deletes a cohort. User must have owner permissions on the cohort. - """ - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_to_deactivate = Django_Cohort.objects.get(id=cohort_id) - if cohort_to_deactivate.active is True: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - if cohort_perm.perm == 'OWNER': - cohort_to_deactivate.active = False - cohort_to_deactivate.save() - return_message = 'Cohort %d successfully deleted.' % cohort_id - else: - return_message = 'You do not have owner permission on cohort %d.' % cohort_id - else: - return_message = "Cohort %d was already deleted." % cohort_id - return ReturnJSON(message=return_message) - - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Either cohort %d does not have an entry in the database " - "or you do not have owner or reader permissions on this cohort." % cohort_id) - finally: - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_get.py b/api/isb_cgc_api/cohorts_get.py deleted file mode 100644 index 0573f00c..00000000 --- a/api/isb_cgc_api/cohorts_get.py +++ /dev/null @@ -1,161 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import endpoints -import logging -import MySQLdb -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsGetListQueryBuilder, \ - CohortsGetListMessageBuilder, FilterDetails -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class CohortDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - patient_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - patients = messages.StringField(13, repeated=True) - samples = messages.StringField(14, repeated=True) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsGetAPI(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - @endpoints.method(GET_RESOURCE, CohortDetails, http_method='GET', path='cohorts/{cohort_id}') - def get(self, request): - """ - Returns information about a specific cohort the user has READER or OWNER permission on - when given a cohort ID. Authentication is required. - """ - user_email = None - cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - cohort_id = request.get_assigned_value('cohort_id') - query_dict = {'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1'), - 'cohorts_cohort.id': cohort_id} - - query_str, query_tuple = CohortsGetListQueryBuilder().build_cohort_query(query_dict) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - row = cursor.fetchone() - - if row is None: - raise endpoints.NotFoundException( - "Cohort {id} not found. Either it never existed, it was deleted, " - "or {user_email} does not have permission to view it.".format( - id=cohort_id, user_email=user_email)) - - # get the filters used for this cohort - filter_query_str, filter_query_tuple = CohortsGetListQueryBuilder().build_filter_query( - {'cohorts_filters.resulting_cohort_id': str(row['id'])}) - cursor.execute(filter_query_str, filter_query_tuple) - filter_data = CohortsGetListMessageBuilder().make_filter_details_from_cursor( - cursor.fetchall()) - - # getting the parent_id's for this cohort is a separate query - # since a single cohort may have multiple parent cohorts - parent_query_str, parent_query_tuple = CohortsGetListQueryBuilder().build_parent_query( - {'cohort_id': str(row['id'])}) - cursor.execute(parent_query_str, parent_query_tuple) - parent_id_data = CohortsGetListMessageBuilder().make_parent_id_list_from_cursor( - cursor.fetchall(), row) - - # get list of samples and cases in this cohort - sample_query_str, sample_query_tuple = CohortsGetListQueryBuilder().build_samples_query( - {'cohort_id': str(row['id'])}) - cursor.execute(sample_query_str, sample_query_tuple) - sample_list = [] - patient_list = [] - for s_row in cursor.fetchall(): - sample_list.append(s_row['sample_barcode']) - if s_row['case_barcode']: - patient_list.append(s_row['case_barcode']) - - if len(sample_list) == 0: - sample_list = ["None"] - if len(patient_list) == 0: - patient_list = ["None"] - - return CohortDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - patient_count=len(patient_list), - sample_count=len(sample_list), - patients=patient_list, - samples=sample_list - ) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "Cohort {} for user {} not found. {}: {}".format(cohort_id, user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_googlegenomics.py b/api/isb_cgc_api/cohorts_googlegenomics.py deleted file mode 100644 index ec0e6bbb..00000000 --- a/api/isb_cgc_api/cohorts_googlegenomics.py +++ /dev/null @@ -1,127 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb -import django - -from django.conf import settings -from django.core.signals import request_finished -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from api.api_helpers import sql_connection -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsGoogleGenomicssAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='cohorts/{cohort_id}/googlegenomics') - def googlegenomics(self, request): - """ - Returns a list of Google Genomics dataset and readgroupset ids associated with - all the samples in a specified cohort. - Authentication is required. User must have either READER or OWNER permissions on the cohort. - """ - cursor = None - db = None - user_email = None - cohort_id = request.get_assigned_value('cohort_id') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - Django_Cohort.objects.get(id=cohort_id) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. Error: {}" \ - .format(user_email, cohort_id, e) - request_finished.send(self) - raise endpoints.UnauthorizedException(err_msg) - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'JOIN cohorts_samples ON metadata_data.SampleBarcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (cohort_id,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for cohort {} not found." - .format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for cohort. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_list.py b/api/isb_cgc_api/cohorts_list.py deleted file mode 100644 index e7075d00..00000000 --- a/api/isb_cgc_api/cohorts_list.py +++ /dev/null @@ -1,157 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages, message_types - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsGetListQueryBuilder, \ - CohortsGetListMessageBuilder, FilterDetails -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class CohortDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - patient_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - - -class CohortDetailsList(messages.Message): - items = messages.MessageField(CohortDetails, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsListAPI(remote.Service): - @endpoints.method(message_types.VoidMessage, CohortDetailsList, http_method='GET', path='cohorts') - def list(self, unused_request): - """ - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. Optionally takes a cohort id as a parameter to - only list information about one cohort. - """ - user_email = None - cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - query_str, query_tuple = CohortsGetListQueryBuilder().build_cohort_query({ - 'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1') - }) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - data = [] - - for row in cursor.fetchall(): - # get filters for each cohort - filter_query_str, filter_query_tuple = CohortsGetListQueryBuilder().build_filter_query( - {'cohorts_filters.resulting_cohort_id': str(row['id'])}) - cursor.execute(filter_query_str, filter_query_tuple) - filter_data = CohortsGetListMessageBuilder().make_filter_details_from_cursor( - cursor.fetchall()) - - # getting the parent_id's for each cohort is a separate query - # since a single cohort may have multiple parent cohorts - parent_query_str, parent_query_tuple = CohortsGetListQueryBuilder().build_parent_query( - {'cohort_id': str(row['id'])}) - cursor.execute(parent_query_str, parent_query_tuple) - parent_id_data = CohortsGetListMessageBuilder().make_parent_id_list_from_cursor( - cursor.fetchall(), row) - - # get number of patients for each cohort - patient_query_str, patient_query_tuple = CohortsGetListQueryBuilder().build_patients_query( - {'cohort_id': str(row['id'])}) - cursor.execute(patient_query_str, patient_query_tuple) - patient_count = len(cursor.fetchall()) - - # get number of samples for each cohort - sample_query_str, sample_query_tuple = CohortsGetListQueryBuilder().build_samples_query( - {'cohort_id': str(row['id'])}) - cursor.execute(sample_query_str, sample_query_tuple) - sample_count = len(cursor.fetchall()) - - data.append(CohortDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - patient_count=patient_count, - sample_count=sample_count - )) - - if len(data) == 0: - raise endpoints.NotFoundException("{} has no active cohorts.".format(user_email)) - - return CohortDetailsList(items=data, count=len(data)) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "User {}'s cohorts not found. {}: {}".format(user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api/isb_cgc_api/cohorts_preview.py b/api/isb_cgc_api/cohorts_preview.py deleted file mode 100644 index f272e36c..00000000 --- a/api/isb_cgc_api/cohorts_preview.py +++ /dev/null @@ -1,98 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsCreatePreviewQueryBuilder, \ - are_there_bad_keys, are_there_no_acceptable_keys, construct_parameter_error_message - -from message_classes import MetadataRangesItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class CohortPatientsSamplesList(messages.Message): - patients = messages.StringField(1, repeated=True) - patient_count = messages.IntegerField(2, variant=messages.Variant.INT32) - samples = messages.StringField(3, repeated=True) - sample_count = messages.IntegerField(4, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsPreviewAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(**{field.name: field for field in MetadataRangesItem.all_fields()}) - - @endpoints.method(GET_RESOURCE, CohortPatientsSamplesList, path='cohorts/preview', http_method='GET') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of participant (aka patient) barcodes, and the list of sample barcodes. - Authentication is not required. - """ - patient_cursor = None - sample_cursor = None - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - raise endpoints.BadRequestException(err_msg) - - query_dict, gte_query_dict, lte_query_dict = CohortsCreatePreviewQueryBuilder().build_query_dictionaries(request) - - patient_query_str, sample_query_str, value_tuple = CohortsCreatePreviewQueryBuilder().build_query( - query_dict, gte_query_dict, lte_query_dict) - - patient_barcodes = [] - sample_barcodes = [] - - try: - db = sql_connection() - patient_cursor = db.cursor(MySQLdb.cursors.DictCursor) - patient_cursor.execute(patient_query_str, value_tuple) - for row in patient_cursor.fetchall(): - patient_barcodes.append(row['case_barcode']) - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, value_tuple) - for row in sample_cursor.fetchall(): - sample_barcodes.append(row['sample_barcode']) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("Error retrieving samples or patients: {}".format(e)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tpatient query: {} {}\n\tsample query: {} {}' \ - .format(e, patient_query_str, value_tuple, sample_query_str, value_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error previewing cohort. {}".format(msg)) - finally: - if patient_cursor: patient_cursor.close() - if sample_cursor: sample_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - return CohortPatientsSamplesList(patients=patient_barcodes, - patient_count=len(patient_barcodes), - samples=sample_barcodes, - sample_count=len(sample_barcodes)) diff --git a/api/isb_cgc_api/isb_cgc_api_helpers.py b/api/isb_cgc_api/isb_cgc_api_helpers.py deleted file mode 100644 index fdc48dca..00000000 --- a/api/isb_cgc_api/isb_cgc_api_helpers.py +++ /dev/null @@ -1,393 +0,0 @@ -import endpoints -from django.conf import settings -from protorpc import messages -import logging - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -BUILTIN_ENDPOINTS_PARAMETERS = [ - 'alt', - 'fields', - 'enum', - 'enumDescriptions', - 'key', - 'oauth_token', - 'prettyPrint', - 'quotaUser', - 'userIp' -] - -ISB_CGC_Endpoints = endpoints.api(name='isb_cgc_api', version='v2', - description="Get information about cohorts, patients, and samples. Create and delete cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v2', - title="ISB-CGC API") - - -def are_there_bad_keys(request): - ''' - Checks for unrecognized fields in an endpoint request - :param request: the request object from the endpoint - :return: boolean indicating True if bad (unrecognized) fields are present in the request - ''' - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - return unrecognized_param_dict != {} - - -def are_there_no_acceptable_keys(request): - """ - Checks for a lack of recognized fields in an endpoints request. Used in save_cohort and preview_cohort endpoints. - :param request: the request object from the endpoint - :return: boolean indicating True if there are no recognized fields in the request. - """ - param_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - } - return param_dict == {} - - -def construct_parameter_error_message(request, filter_required): - err_msg = '' - sorted_acceptable_keys = sorted([k.name for k in request.all_fields()], key=lambda s: s.lower()) - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - if unrecognized_param_dict: - bad_key_str = "'" + "', '".join(unrecognized_param_dict.keys()) + "'" - err_msg += "The following filters were not recognized: {}. ".format(bad_key_str) - if filter_required: - err_msg += "You must specify at least one of the following " \ - "case-sensitive filters: {}".format(sorted_acceptable_keys) - else: - err_msg += "Acceptable filters are: {}".format(sorted_acceptable_keys) - - return err_msg - - -class CohortsGetListQueryBuilder(object): - - def build_cohort_query(self, query_dict): - """ - Builds the query that will select cohort id, name, last_date_saved, - perms, comments, source type, and source notes - :param query_dict: should contain {'cohorts_cohort_perms.user_id': user_id, 'cohorts_cohort.active': unicode('1')} - :return: query_str, query_tuple - """ - query_str = 'SELECT cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'cohorts_cohort_comments.content AS comments, ' \ - 'cohorts_source.type AS source_type, ' \ - 'cohorts_source.notes AS source_notes ' \ - 'FROM cohorts_cohort_perms ' \ - 'JOIN cohorts_cohort ' \ - 'ON cohorts_cohort.id=cohorts_cohort_perms.cohort_id ' \ - 'JOIN auth_user ' \ - 'ON auth_user.id=cohorts_cohort_perms.user_id ' \ - 'LEFT JOIN cohorts_cohort_comments ' \ - 'ON cohorts_cohort_comments.user_id=cohorts_cohort_perms.user_id ' \ - 'AND cohorts_cohort_comments.cohort_id=cohorts_cohort.id ' \ - 'LEFT JOIN cohorts_source ' \ - 'ON cohorts_source.cohort_id=cohorts_cohort_perms.cohort_id ' - - query_tuple = () - if query_dict: - query_str += ' WHERE ' + '=%s and '.join(key for key in query_dict.keys()) + '=%s ' - query_tuple = tuple(value for value in query_dict.values()) - - query_str += 'GROUP BY ' \ - 'cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'comments, ' \ - 'source_type, ' \ - 'source_notes ' - - return query_str, query_tuple - - def build_filter_query(self, filter_query_dict): - """ - Builds the query that selects the filter name and value for a particular cohort - :param filter_query_dict: should be {'cohorts_filters.resulting_cohort_id:': id} - :return: filter_query_str, filter_query_tuple - """ - filter_query_str = 'SELECT name, value ' \ - 'FROM cohorts_filters ' - - filter_query_str += ' WHERE ' + '=%s AND '.join(key for key in filter_query_dict.keys()) + '=%s ' - filter_query_tuple = tuple(value for value in filter_query_dict.values()) - - return filter_query_str, filter_query_tuple - - def build_parent_query(self, parent_query_dict): - """ - Builds the query that selects parent_ids for a particular cohort - :param parent_query_dict: should be {'cohort_id': str(row['id'])} - :return: parent_query_str, parent_query_tuple - """ - parent_query_str = 'SELECT parent_id ' \ - 'FROM cohorts_source ' - parent_query_str += ' WHERE ' + '=%s AND '.join(key for key in parent_query_dict.keys()) + '=%s ' - parent_query_tuple = tuple(value for value in parent_query_dict.values()) - - return parent_query_str, parent_query_tuple - - def build_patients_query(self, patient_query_dict): - """ - Builds the query that selects the case count for a particular cohort - :param patient_query_dict: should be {'cohort_id': str(row['id])} - :return: patient_query_str, patient_query_tuple - """ - patients_query_str = 'SELECT case_barcode ' \ - 'FROM cohorts_samples ' - - patients_query_str += ' WHERE ' + '=%s AND '.join(key for key in patient_query_dict.keys()) + '=%s ' - patient_query_tuple = tuple(value for value in patient_query_dict.values()) - - return patients_query_str, patient_query_tuple - - def build_samples_query(self, sample_query_dict): - """ - Builds the query that selects the sample count for a particular cohort - :param sample_query_dict: should be {'cohort_id': str(row['id])} - :return: sample_query_str, sample_query_tuple - """ - samples_query_str = 'SELECT sample_barcode, case_barcode ' \ - 'FROM cohorts_samples ' - - samples_query_str += ' WHERE ' + '=%s AND '.join(key for key in sample_query_dict.keys()) + '=%s ' - sample_query_tuple = tuple(value for value in sample_query_dict.values()) - - return samples_query_str, sample_query_tuple - - -class CohortsCreatePreviewQueryBuilder(object): - def build_query_dictionaries(self, request): - """ - Builds the query dictionaries for create and preview cohort endpoints. - Returns query_dict, gte_query_dict, lte_query_dict. - """ - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - and k.name is not 'name' - and not k.name.endswith('_gte') - and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - return query_dict, gte_query_dict, lte_query_dict - - def build_query(self, query_dict, gte_query_dict, lte_query_dict): - """ - Builds the queries that selects the patient and sample barcodes - that meet the criteria specified in the request body. - Returns patient query string, sample query string, value tuple. - """ - - patient_query_str = 'SELECT DISTINCT(IF(case_barcode="", LEFT(sample_barcode,12), case_barcode)) ' \ - 'AS case_barcode ' \ - 'FROM metadata_samples ' \ - 'WHERE ' - - sample_query_str = 'SELECT sample_barcode, case_barcode ' \ - 'FROM metadata_samples ' \ - 'WHERE ' - value_tuple = () - - for key, value_list in query_dict.iteritems(): - patient_query_str += ' AND ' if not patient_query_str.endswith('WHERE ') else '' - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - if "None" in value_list: - value_list.remove("None") - patient_query_str += ' ( {key} is null '.format(key=key) - sample_query_str += ' ( {key} is null '.format(key=key) - if len(value_list) > 0: - patient_query_str += ' OR {key} IN ({vals}) '.format( - key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ' OR {key} IN ({vals}) '.format( - key=key, vals=', '.join(['%s'] * len(value_list))) - patient_query_str += ') ' - sample_query_str += ') ' - else: - patient_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - patient_query_str += ' AND ' if not patient_query_str.endswith('WHERE ') else '' - patient_query_str += ' {} >=%s '.format(key) - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} >=%s '.format(key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - patient_query_str += ' AND ' if not patient_query_str.endswith('WHERE ') else '' - patient_query_str += ' {} <=%s '.format(key) - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} <=%s '.format(key) - value_tuple += (value,) - - sample_query_str += ' GROUP BY sample_barcode' - - return patient_query_str, sample_query_str, value_tuple - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -class CohortsGetListMessageBuilder(object): - def make_filter_details_from_cursor(self, filter_cursor_dict): - """ - Returns list of FilterDetails from a dictionary of results - from a filter query. - """ - filter_data = [] - for filter_row in filter_cursor_dict: - filter_data.append(FilterDetails( - name=str(filter_row['name']), - value=str(filter_row['value']) - )) - - if len(filter_data) == 0: - filter_data.append(FilterDetails( - name="None", - value="None" - )) - - return filter_data - - def make_parent_id_list_from_cursor(self, parent_cursor_dict, row): - """ - Returns list of parent_id's from a dictionary of results - from a parent id query. - """ - parent_id_data = [str(p_row['parent_id']) for p_row in parent_cursor_dict if row.get('parent_id')] - if len(parent_id_data) == 0: - parent_id_data.append("None") - - return parent_id_data - - -class CohortsSamplesFilesQueryBuilder(object): - - def build_query(self, platform=None, pipeline=None, limit=None, cohort_id=None, sample_barcode=None): - - query_str = 'SELECT DataFileNameKey, SecurityProtocol, Repository ' \ - 'FROM metadata_data ' - - if cohort_id is None: - query_str += 'WHERE sample_barcode=%s ' - else: - query_str += 'JOIN cohorts_samples ON metadata_data.sample_barcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' - - query_str += 'AND DataFileNameKey != "" AND DataFileNameKey is not null ' - query_str += ' and metadata_data.Platform=%s ' if platform is not None else '' - query_str += ' and metadata_data.Pipeline=%s ' if pipeline is not None else '' - query_str += ' GROUP BY DataFileNameKey, SecurityProtocol, Repository ' - query_str += ' LIMIT %s' if limit is not None else ' LIMIT 10000' - - query_tuple = (cohort_id,) if cohort_id is not None else (sample_barcode,) - query_tuple += (platform,) if platform is not None else () - query_tuple += (pipeline,) if pipeline is not None else () - query_tuple += (limit,) if limit is not None else () - - return query_str, query_tuple - - -class CohortsSamplesFilesMessageBuilder(object): - - def get_GCS_file_paths_and_bad_repos(self, cursor_rows): - """ - Used in cohorts.datafilenamekeys, samples.datafilenamekeys, samples.get. - Modifies cursor_rows to add the cloud storage path to each row representing a file. - A count of bad repositories and a set of bad repositories is returned in case - there are any errors with the data repository information in the row. - :param cursor_rows: list of dictionaries resulting from a database query. - Each dictionary with the key 'DataFileNameKey' must also have a 'SecurityProtocol' key. - Each dictionary with 'controlled' in the value for 'SecurityProtocol' must also - have the key 'Repository'. - :return: bad_repo_count, bad_repo_set - - """ - bad_repo_count = 0 - bad_repo_set = set() - for row in cursor_rows: - if not row.get('DataFileNameKey'): - continue - - if 'controlled' not in str(row['SecurityProtocol']).lower(): - # this may only be necessary for the vagrant db - path = row.get('DataFileNameKey') if row.get('DataFileNameKey') is None \ - else row.get('DataFileNameKey').replace('gs://' + settings.OPEN_DATA_BUCKET, '') - row['cloud_storage_path'] = "gs://{}{}".format(settings.OPEN_DATA_BUCKET, path) - else: - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: - bad_repo_count += 1 - bad_repo_set.add(row['Repository']) - continue - # this may only be necessary for the vagrant db - path = row.get('DataFileNameKey') if row.get('DataFileNameKey') is None \ - else row.get('DataFileNameKey').replace('gs://' + bucket_name, '') - - row['cloud_storage_path'] = "gs://{}{}".format(bucket_name, path) - return bad_repo_count, bad_repo_set - - -def build_constructor_dict_for_message(message_class, row): - """ - Takes an instance of a message class and a dictionary of values from a database query - and first validates the values in the dictionary against the message class fields - and then returns a dictionary of all the validated key-value pairs in the database query. - This will only work if the headers in the database query have the same name as the names of - fields in the message class. - """ - constructor_dict = {} - metadata_item_dict = {field.name: field for field in message_class.all_fields()} - for name, field in metadata_item_dict.iteritems(): - if row.get(name) is not None: - try: - field.validate(row[name]) - constructor_dict[name] = row[name] - except messages.ValidationError, e: - constructor_dict[name] = None - logger.warn('{name}: {value} was not validated while constructing kwargs for {message_class}. Error: {e}' - .format(name=name, value=str(row[name]), message_class=str(message_class), e=e)) - else: - constructor_dict[name] = None - - return constructor_dict \ No newline at end of file diff --git a/api/isb_cgc_api/message_classes.py b/api/isb_cgc_api/message_classes.py deleted file mode 100644 index 2a60db14..00000000 --- a/api/isb_cgc_api/message_classes.py +++ /dev/null @@ -1,366 +0,0 @@ -from protorpc import messages - - -class MetadataRangesItem(messages.Message): - - age_at_initial_pathologic_diagnosis = messages.IntegerField(1, repeated=True, variant=messages.Variant.INT32) - age_at_initial_pathologic_diagnosis_lte = messages.IntegerField(2, variant=messages.Variant.INT32) - age_at_initial_pathologic_diagnosis_gte = messages.IntegerField(3, variant=messages.Variant.INT32) - - anatomic_neoplasm_subdivision = messages.StringField(4, repeated=True) - - avg_percent_lymphocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_lymphocyte_infiltration_lte = messages.FloatField(6) - avg_percent_lymphocyte_infiltration_gte = messages.FloatField(7) - - avg_percent_monocyte_infiltration = messages.FloatField(8, repeated=True) - avg_percent_monocyte_infiltration_lte = messages.FloatField(9) - avg_percent_monocyte_infiltration_gte = messages.FloatField(10) - - avg_percent_necrosis = messages.FloatField(11, repeated=True) - avg_percent_necrosis_lte = messages.FloatField(12) - avg_percent_necrosis_gte = messages.FloatField(13) - - avg_percent_neutrophil_infiltration = messages.FloatField(14, repeated=True) - avg_percent_neutrophil_infiltration_lte = messages.FloatField(15) - avg_percent_neutrophil_infiltration_gte = messages.FloatField(16) - - avg_percent_normal_cells = messages.FloatField(17, repeated=True) - avg_percent_normal_cells_lte = messages.FloatField(18) - avg_percent_normal_cells_gte = messages.FloatField(19) - - avg_percent_stromal_cells = messages.FloatField(20, repeated=True) - avg_percent_stromal_cells_lte = messages.FloatField(21) - avg_percent_stromal_cells_gte = messages.FloatField(22) - - avg_percent_tumor_cells = messages.FloatField(23, repeated=True) - avg_percent_tumor_cells_lte = messages.FloatField(24) - avg_percent_tumor_cells_gte = messages.FloatField(25) - - avg_percent_tumor_nuclei = messages.FloatField(26, repeated=True) - avg_percent_tumor_nuclei_lte = messages.FloatField(27) - avg_percent_tumor_nuclei_gte = messages.FloatField(28) - - batch_number = messages.IntegerField(29, repeated=True, variant=messages.Variant.INT32) - batch_number_lte = messages.IntegerField(30, variant=messages.Variant.INT32) - batch_number_gte = messages.IntegerField(31, variant=messages.Variant.INT32) - - bcr = messages.StringField(32, repeated=True) - - BMI = messages.FloatField(33, repeated=True) - BMI_lte = messages.FloatField(34) - BMI_gte = messages.FloatField(35) - - clinical_M = messages.StringField(36, repeated=True) - clinical_N = messages.StringField(37, repeated=True) - clinical_stage = messages.StringField(38, repeated=True) - clinical_T = messages.StringField(39, repeated=True) - colorectal_cancer = messages.StringField(40, repeated=True) - country = messages.StringField(41, repeated=True) - - days_to_birth = messages.IntegerField(42, repeated=True, variant=messages.Variant.INT32) - days_to_birth_lte = messages.IntegerField(43, variant=messages.Variant.INT32) - days_to_birth_gte = messages.IntegerField(44, variant=messages.Variant.INT32) - - days_to_collection = messages.IntegerField(45, repeated=True, variant=messages.Variant.INT32) - days_to_collection_lte = messages.IntegerField(46, variant=messages.Variant.INT32) - days_to_collection_gte = messages.IntegerField(47, variant=messages.Variant.INT32) - - days_to_death = messages.IntegerField(48, repeated=True, variant=messages.Variant.INT32) - days_to_death_lte = messages.IntegerField(49, variant=messages.Variant.INT32) - days_to_death_gte = messages.IntegerField(50, variant=messages.Variant.INT32) - - days_to_initial_pathologic_diagnosis = messages.IntegerField(51, repeated=True, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_lte = messages.IntegerField(52, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_gte = messages.IntegerField(53, variant=messages.Variant.INT32) - - days_to_last_followup = messages.IntegerField(54, repeated=True, variant=messages.Variant.INT32) - days_to_last_followup_lte = messages.IntegerField(55, variant=messages.Variant.INT32) - days_to_last_followup_gte = messages.IntegerField(56, variant=messages.Variant.INT32) - - days_to_last_known_alive = messages.IntegerField(57, repeated=True, variant=messages.Variant.INT32) - days_to_last_known_alive_lte = messages.IntegerField(58, variant=messages.Variant.INT32) - days_to_last_known_alive_gte = messages.IntegerField(59, variant=messages.Variant.INT32) - - days_to_submitted_specimen_dx = messages.IntegerField(60, repeated=True, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_lte = messages.IntegerField(61, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_gte = messages.IntegerField(62, variant=messages.Variant.INT32) - - ethnicity = messages.StringField(63, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(64, repeated=True) - gender = messages.StringField(65, repeated=True) - - gleason_score_combined = messages.IntegerField(66, repeated=True, variant=messages.Variant.INT32) - gleason_score_combined_lte = messages.IntegerField(67, variant=messages.Variant.INT32) - gleason_score_combined_gte = messages.IntegerField(68, variant=messages.Variant.INT32) - - has_27k = messages.BooleanField(69, repeated=True) - has_450k = messages.BooleanField(70, repeated=True) - has_BCGSC_GA_RNASeq = messages.BooleanField(71, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.BooleanField(72, repeated=True) - has_GA_miRNASeq = messages.BooleanField(73, repeated=True) - has_HiSeq_miRnaSeq = messages.BooleanField(74, repeated=True) - has_Illumina_DNASeq = messages.BooleanField(75, repeated=True) - has_RPPA = messages.BooleanField(76, repeated=True) - has_SNP6 = messages.BooleanField(77, repeated=True) - has_UNC_GA_RNASeq = messages.BooleanField(78, repeated=True) - has_UNC_HiSeq_RNASeq = messages.BooleanField(79, repeated=True) - - height = messages.IntegerField(80, repeated=True, variant=messages.Variant.INT32) - height_lte = messages.IntegerField(81, variant=messages.Variant.INT32) - height_gte = messages.IntegerField(82, variant=messages.Variant.INT32) - - histological_type = messages.StringField(83, repeated=True) - history_of_colon_polyps = messages.StringField(84, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(85, repeated=True) - history_of_prior_malignancy = messages.StringField(86, repeated=True) - hpv_calls = messages.StringField(87, repeated=True) - hpv_status = messages.StringField(88, repeated=True) - icd_10 = messages.StringField(89, repeated=True) - icd_o_3_histology = messages.StringField(90, repeated=True) - icd_o_3_site = messages.StringField(91, repeated=True) - lymphatic_invasion = messages.StringField(92, repeated=True) - lymphnodes_examined = messages.StringField(93, repeated=True) - lymphovascular_invasion_present = messages.StringField(94, repeated=True) - - max_percent_lymphocyte_infiltration = messages.FloatField(95, repeated=True) - max_percent_lymphocyte_infiltration_lte = messages.FloatField(96) - max_percent_lymphocyte_infiltration_gte = messages.FloatField(97) - - max_percent_monocyte_infiltration = messages.FloatField(98, repeated=True) - max_percent_monocyte_infiltration_lte = messages.FloatField(99) - max_percent_monocyte_infiltration_gte = messages.FloatField(100) - - max_percent_necrosis = messages.FloatField(101, repeated=True) - max_percent_necrosis_lte = messages.FloatField(102) - max_percent_necrosis_gte = messages.FloatField(103) - - max_percent_neutrophil_infiltration = messages.FloatField(104, repeated=True) - max_percent_neutrophil_infiltration_lte = messages.FloatField(105) - max_percent_neutrophil_infiltration_gte = messages.FloatField(106) - - max_percent_normal_cells = messages.FloatField(107, repeated=True) - max_percent_normal_cells_lte = messages.FloatField(108) - max_percent_normal_cells_gte = messages.FloatField(109) - - max_percent_stromal_cells = messages.FloatField(110, repeated=True) - max_percent_stromal_cells_lte = messages.FloatField(111) - max_percent_stromal_cells_gte = messages.FloatField(112) - - max_percent_tumor_cells = messages.FloatField(113, repeated=True) - max_percent_tumor_cells_lte = messages.FloatField(114) - max_percent_tumor_cells_gte = messages.FloatField(115) - - max_percent_tumor_nuclei = messages.FloatField(116, repeated=True) - max_percent_tumor_nuclei_lte = messages.FloatField(117) - max_percent_tumor_nuclei_gte = messages.FloatField(118) - - menopause_status = messages.StringField(119, repeated=True) - - min_percent_lymphocyte_infiltration = messages.FloatField(120, repeated=True) - min_percent_lymphocyte_infiltration_lte = messages.FloatField(121) - min_percent_lymphocyte_infiltration_gte = messages.FloatField(122) - - min_percent_monocyte_infiltration = messages.FloatField(123, repeated=True) - min_percent_monocyte_infiltration_lte = messages.FloatField(124) - min_percent_monocyte_infiltration_gte = messages.FloatField(125) - - min_percent_necrosis = messages.FloatField(126, repeated=True) - min_percent_necrosis_lte = messages.FloatField(127) - min_percent_necrosis_gte = messages.FloatField(128) - - min_percent_neutrophil_infiltration = messages.FloatField(129, repeated=True) - min_percent_neutrophil_infiltration_lte = messages.FloatField(130) - min_percent_neutrophil_infiltration_gte = messages.FloatField(131) - - min_percent_normal_cells = messages.FloatField(132, repeated=True) - min_percent_normal_cells_lte = messages.FloatField(133) - min_percent_normal_cells_gte = messages.FloatField(134) - - min_percent_stromal_cells = messages.FloatField(135, repeated=True) - min_percent_stromal_cells_lte = messages.FloatField(136) - min_percent_stromal_cells_gte = messages.FloatField(137) - - min_percent_tumor_cells = messages.FloatField(138, repeated=True) - min_percent_tumor_cells_lte = messages.FloatField(139) - min_percent_tumor_cells_gte = messages.FloatField(140) - - min_percent_tumor_nuclei = messages.FloatField(141, repeated=True) - min_percent_tumor_nuclei_lte = messages.FloatField(142) - min_percent_tumor_nuclei_gte = messages.FloatField(143) - - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(144, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(145, repeated=True) - neoplasm_histologic_grade = messages.StringField(146, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(147, repeated=True) - - number_of_lymphnodes_examined = messages.IntegerField(148, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_lte = messages.IntegerField(149, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_gte = messages.IntegerField(150, variant=messages.Variant.INT32) - - number_of_lymphnodes_positive_by_he = messages.IntegerField(151, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_lte = messages.IntegerField(152, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_gte = messages.IntegerField(153, variant=messages.Variant.INT32) - - number_pack_years_smoked = messages.IntegerField(154, repeated=True, variant=messages.Variant.INT32) - number_pack_years_smoked_lte = messages.IntegerField(155, variant=messages.Variant.INT32) - number_pack_years_smoked_gte = messages.IntegerField(156, variant=messages.Variant.INT32) - - case_barcode = messages.StringField(157, repeated=True) - pathologic_M = messages.StringField(158, repeated=True) - pathologic_N = messages.StringField(159, repeated=True) - pathologic_stage = messages.StringField(160, repeated=True) - pathologic_T = messages.StringField(161, repeated=True) - person_neoplasm_cancer_status = messages.StringField(162, repeated=True) - pregnancies = messages.StringField(163, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(164, repeated=True) - primary_therapy_outcome_success = messages.StringField(165, repeated=True) - prior_dx = messages.StringField(166, repeated=True) - Project = messages.StringField(167, repeated=True) - - psa_value = messages.FloatField(168, repeated=True) - psa_value_lte = messages.FloatField(169) - psa_value_gte = messages.FloatField(170) - - race = messages.StringField(171, repeated=True) - residual_tumor = messages.StringField(172, repeated=True) - sample_barcode = messages.StringField(173, repeated=True) - SampleTypeCode = messages.StringField(174, repeated=True) - disease_code = messages.StringField(175, repeated=True) - tobacco_smoking_history = messages.StringField(176, repeated=True) - TSSCode = messages.StringField(177, repeated=True) - tumor_tissue_site = messages.StringField(178, repeated=True) - tumor_type = messages.StringField(179, repeated=True) - vital_status = messages.StringField(180, repeated=True) - - weight = messages.IntegerField(181, repeated=True, variant=messages.Variant.INT32) - weight_lte = messages.IntegerField(182, variant=messages.Variant.INT32) - weight_gte = messages.IntegerField(183, variant=messages.Variant.INT32) - - weiss_venous_invasion = messages.StringField(184, repeated=True) - - year_of_initial_pathologic_diagnosis = messages.IntegerField(185, repeated=True, variant=messages.Variant.INT32) - year_of_initial_pathologic_diagnosis_lte = messages.IntegerField(186, variant=messages.Variant.INT32) - year_of_initial_pathologic_diagnosis_gte = messages.IntegerField(187, variant=messages.Variant.INT32) - - -class MetadataItem(messages.Message): - age_at_initial_pathologic_diagnosis = messages.IntegerField(1, variant=messages.Variant.INT32) - anatomic_neoplasm_subdivision = messages.StringField(2) - avg_percent_lymphocyte_infiltration = messages.FloatField(3) - avg_percent_monocyte_infiltration = messages.FloatField(4) - avg_percent_necrosis = messages.FloatField(5) - avg_percent_neutrophil_infiltration = messages.FloatField(6) - avg_percent_normal_cells = messages.FloatField(7) - avg_percent_stromal_cells = messages.FloatField(8) - avg_percent_tumor_cells = messages.FloatField(9) - avg_percent_tumor_nuclei = messages.FloatField(10) - batch_number = messages.IntegerField(11, variant=messages.Variant.INT32) - bcr = messages.StringField(12) - BMI = messages.FloatField(13) - clinical_M = messages.StringField(14) - clinical_N = messages.StringField(15) - clinical_stage = messages.StringField(16) - clinical_T = messages.StringField(17) - colorectal_cancer = messages.StringField(18) - country = messages.StringField(19) - days_to_birth = messages.IntegerField(20, variant=messages.Variant.INT32) - days_to_collection = messages.IntegerField(21, variant=messages.Variant.INT32) - days_to_death = messages.IntegerField(22, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis = messages.IntegerField(23, variant=messages.Variant.INT32) - days_to_last_followup = messages.IntegerField(24, variant=messages.Variant.INT32) - days_to_last_known_alive = messages.IntegerField(25, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx = messages.IntegerField(26, variant=messages.Variant.INT32) - ethnicity = messages.StringField(27) - frozen_specimen_anatomic_site = messages.StringField(28) - gender = messages.StringField(29) - gleason_score_combined = messages.IntegerField(30, variant=messages.Variant.INT32) - has_27k = messages.BooleanField(31) - has_450k = messages.BooleanField(32) - has_BCGSC_GA_RNASeq = messages.BooleanField(33) - has_BCGSC_HiSeq_RNASeq = messages.BooleanField(34) - has_GA_miRNASeq = messages.BooleanField(35) - has_HiSeq_miRnaSeq = messages.BooleanField(36) - has_Illumina_DNASeq = messages.BooleanField(37) - has_RPPA = messages.BooleanField(38) - has_SNP6 = messages.BooleanField(39) - has_UNC_GA_RNASeq = messages.BooleanField(40) - has_UNC_HiSeq_RNASeq = messages.BooleanField(41) - height = messages.IntegerField(42, variant=messages.Variant.INT32) - histological_type = messages.StringField(43) - history_of_colon_polyps = messages.StringField(44) - history_of_neoadjuvant_treatment = messages.StringField(45) - history_of_prior_malignancy = messages.StringField(46) - hpv_calls = messages.StringField(47) - hpv_status = messages.StringField(48) - icd_10 = messages.StringField(49) - icd_o_3_histology = messages.StringField(50) - icd_o_3_site = messages.StringField(51) - lymphatic_invasion = messages.StringField(52) - lymphnodes_examined = messages.StringField(53) - lymphovascular_invasion_present = messages.StringField(54) - max_percent_lymphocyte_infiltration = messages.FloatField(55) - max_percent_monocyte_infiltration = messages.FloatField(56) - max_percent_necrosis = messages.FloatField(57) - max_percent_neutrophil_infiltration = messages.FloatField(58) - max_percent_normal_cells = messages.FloatField(59) - max_percent_stromal_cells = messages.FloatField(60) - max_percent_tumor_cells = messages.FloatField(61) - max_percent_tumor_nuclei = messages.FloatField(62) - menopause_status = messages.StringField(63) - min_percent_lymphocyte_infiltration = messages.FloatField(64) - min_percent_monocyte_infiltration = messages.FloatField(65) - min_percent_necrosis = messages.FloatField(66) - min_percent_neutrophil_infiltration = messages.FloatField(67) - min_percent_normal_cells = messages.FloatField(68) - min_percent_stromal_cells = messages.FloatField(69) - min_percent_tumor_cells = messages.FloatField(70) - min_percent_tumor_nuclei = messages.FloatField(71) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(72) - mononucleotide_marker_panel_analysis_status = messages.StringField(73) - neoplasm_histologic_grade = messages.StringField(74) - new_tumor_event_after_initial_treatment = messages.StringField(75) - number_of_lymphnodes_examined = messages.IntegerField(76, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he = messages.IntegerField(77, variant=messages.Variant.INT32) - number_pack_years_smoked = messages.IntegerField(78, variant=messages.Variant.INT32) - case_barcode = messages.StringField(79) - pathologic_M = messages.StringField(80) - pathologic_N = messages.StringField(81) - pathologic_stage = messages.StringField(82) - pathologic_T = messages.StringField(83) - person_neoplasm_cancer_status = messages.StringField(84) - pregnancies = messages.StringField(85) - primary_neoplasm_melanoma_dx = messages.StringField(86) - primary_therapy_outcome_success = messages.StringField(87) - prior_dx = messages.StringField(88) - program_name = messages.StringField(89) - psa_value = messages.FloatField(90) - race = messages.StringField(91) - residual_tumor = messages.StringField(92) - sample_barcode = messages.StringField(93) - SampleTypeCode = messages.StringField(94) - disease_code = messages.StringField(95) - tobacco_smoking_history = messages.StringField(96) - TSSCode = messages.StringField(97) - tumor_tissue_site = messages.StringField(98) - tumor_type = messages.StringField(99) - vital_status = messages.StringField(100) - weight = messages.IntegerField(101, variant=messages.Variant.INT32) - weiss_venous_invasion = messages.StringField(102) - year_of_initial_pathologic_diagnosis = messages.IntegerField(103, variant=messages.Variant.INT32) - - -class MetadataAnnotationItem(messages.Message): - AliquotBarcode = messages.StringField(1) - annotationCategoryId = messages.IntegerField(2, variant=messages.Variant.INT32) - annotationCategoryName = messages.StringField(3) - annotationClassification = messages.StringField(4) - annotationClassificationId = messages.IntegerField(5, variant=messages.Variant.INT32) - annotationId = messages.IntegerField(6, variant=messages.Variant.INT32) - annotationNoteText = messages.StringField(7) - itemBarcode = messages.StringField(8) - itemTypeId = messages.IntegerField(9, variant=messages.Variant.INT32) - itemTypeName = messages.StringField(10) - case_barcode = messages.StringField(11) - sample_barcode = messages.StringField(12) - disease_code = messages.StringField(13) \ No newline at end of file diff --git a/api/isb_cgc_api/message_generator.py b/api/isb_cgc_api/message_generator.py deleted file mode 100644 index dfefbf6b..00000000 --- a/api/isb_cgc_api/message_generator.py +++ /dev/null @@ -1,134 +0,0 @@ - -from argparse import ArgumentParser -import MySQLdb - -def get_sql_connection(args): - try: - connect_options = { - 'host': args.host, - 'db': args.database, - 'user': args.user, - 'passwd': args.password, - 'ssl': { - 'ca': args.ssl_ca, - 'key': args.ssl_key, - 'cert': args.ssl_cert - } - } - db = MySQLdb.connect(**connect_options) - except: - print 'failed' - raise - return db - -FIELD_TYPES ={ - 'varchar': 'StringField', - 'float': 'FloatField', - 'tinyint': 'BooleanField', - 'int': 'IntegerField' -} - - -def write_metadata_file(rows, write_file=False): - - ranges_text = 'class MetadataRangesItem(messages.Message):\n ' - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if field_type == 'IntegerField' or field_type == 'FloatField' and i > 1: - ranges_text += '\n ' - - ranges_text += '%-65s = messages.%s(%d, repeated=True' % (row['COLUMN_NAME'], field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if field_type == 'IntegerField' or field_type == 'FloatField': - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_lte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_gte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '\n ' - - item_text = '\nclass MetadataItem(messages.Message):\n ' - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - item_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME'], field_type, i) - item_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if write_file is True: - with open('message_classes.py', 'w') as f: - f.write('from protorpc import messages\n\n\n') - f.write(ranges_text) - f.write(item_text) - else: - print ranges_text - print '\n\n' - print item_text - - -def write_annotation_file(rows, write_file=False): - item_text = '\nclass MetadataAnnotationItem(messages.Message):\n ' - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if field_type.lower() == 'datetime': continue - item_text += '%-30s = messages.%s(%d' % (row['COLUMN_NAME'], field_type, i) - item_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if write_file is True: - with open('message_classes.py', 'a') as f: - f.write('\n\n\n') - f.write(item_text) - else: - print item_text - - -def main(args): - db = get_sql_connection(args) - cursor = db.cursor(MySQLdb.cursors.DictCursor) - query_str = 'SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'FROM INFORMATION_SCHEMA.COLUMNS ' - if args.table == 'metadata_samples': - query_str += 'WHERE table_name = "metadata_samples" ' \ - 'AND COLUMN_NAME != "metadata_samples_id" ' - - elif args.table == 'metadata_annotation': - query_str += 'WHERE table_name = "metadata_annotation" ' \ - 'AND COLUMN_NAME != "metadata_annotation_id" ' - - query_str += 'group by COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'ORDER BY column_name' - - cursor.execute(query_str) - rows = cursor.fetchall() - cursor.close() - db.close() - - write_file = not bool(args.dry_run) - - if args.table == 'metadata_samples': - write_metadata_file(rows, write_file=write_file) - if args.table == 'metadata_annotation': - write_annotation_file(rows, write_file=write_file) - -if __name__ == '__main__': - parser = ArgumentParser() - parser.add_argument('--host', help='database host') - parser.add_argument('--database', '-d', help='database name') - parser.add_argument('--table', '-t', help='database table name') - parser.add_argument('--user', '-u', help='database username') - parser.add_argument('--password', '-p', help='database password') - parser.add_argument('--ssl_key') - parser.add_argument('--ssl_cert') - parser.add_argument('--ssl_ca') - parser.add_argument('--dry_run', dest='dry_run', action='store_true') - parser.add_argument('--write_file', dest='dry_run', action='store_false') - parser.set_defaults(feature=True) - args = parser.parse_args() - - main(args) \ No newline at end of file diff --git a/api/isb_cgc_api/patients_annotations.py b/api/isb_cgc_api/patients_annotations.py deleted file mode 100644 index c8231a16..00000000 --- a/api/isb_cgc_api/patients_annotations.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message -from message_classes import MetadataAnnotationItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class MetadataAnnotationList(messages.Message): - items = messages.MessageField(MetadataAnnotationItem, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -class PatientsAnnotationsQueryBuilder(object): - - @staticmethod - def build_query(item_type_name=None): - query_str = 'select * ' \ - 'from metadata_annotation ' \ - 'where case_barcode=%s ' - if len(item_type_name) > 0: - query_str += 'and itemTypeName in (' + ', '.join(['%s']*len(item_type_name)) + ')' - - - return query_str - - @staticmethod - def build_metadata_samples_query(): - query_str = 'select * ' \ - 'from metadata_samples ' \ - 'where case_barcode=%s ' - - return query_str - -@ISB_CGC_Endpoints.api_class(resource_name='patients') -class PatientsAnnotationAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(patient_barcode=messages.StringField(1, required=True), - item_type_name=messages.StringField(2, repeated=True)) - - @endpoints.method(GET_RESOURCE, MetadataAnnotationList, - path='patients/{patient_barcode}/annotations', http_method='GET') - def annotations(self, request): - """ - Returns TCGA annotations about a specific patient, - Takes a patient barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - patient_barcode = request.get_assigned_value('patient_barcode') - query_tuple = (str(patient_barcode),) - # check to make sure patient_barcode is in correct form - try: - parts = patient_barcode.split('-') - assert len(parts) == 3 - assert len(parts[0]) == 4 - assert len(parts[1]) == 2 - assert len(parts[2]) == 4 - except AssertionError: - raise endpoints.BadRequestException('{} is not the correct format for a patient barcode. ' - 'Patient barcodes must be of the form XXXX-XX-XXXX'.format(patient_barcode)) - - item_type_name = request.get_assigned_value('item_type_name') - - # check to make sure each item_type_name is valid - if len(item_type_name) > 0: - for itm in item_type_name: - itm = itm.strip() - if itm.lower() not in ['patient', 'aliquot', 'analyte', 'shipped portion', 'portion', 'slide', 'sample']: - raise endpoints.BadRequestException("'{}' is not a valid entry for item_type_name. " - "Valid entries include 'Patient', 'Aliquot', 'Analyte', 'Shipped Portion', " - "'Portion', 'Slide', and 'Sample'".format(itm)) - query_tuple += (itm,) - - query_str = PatientsAnnotationsQueryBuilder().build_query(item_type_name=item_type_name) - metadata_samples_query_str = PatientsAnnotationsQueryBuilder().build_metadata_samples_query() - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build annotation message - cursor.execute(query_str, query_tuple) - rows = cursor.fetchall() - cursor.execute(metadata_samples_query_str, (str(patient_barcode),)) - metadata_sample_rows = cursor.fetchall() - if len(rows) == 0: - cursor.close() - db.close() - if len(metadata_sample_rows) == 0: - msg = "Patient barcode {} not found in the database.".format(patient_barcode) - logger.info(msg) - else: - msg = "No annotations found for patient barcode {}".format(patient_barcode) - if item_type_name is not None: - msg += " and item type name {}. Item type name must be one of the following: " \ - "'Patient', 'Aliquot', 'Analyte', 'Shipped Portion', 'Portion', 'Slide', 'Sample'.".format(item_type_name) - logger.info(msg) - raise endpoints.NotFoundException(msg) - - items = [] - for row in rows: - constructor_dict = build_constructor_dict_for_message(MetadataAnnotationItem(), row) - items.append(MetadataAnnotationItem(**constructor_dict)) - - return MetadataAnnotationList(items=items, count=len(items)) - - except (IndexError, TypeError), e: - logger.info("Patient {} not found. Error: {}".format(patient_barcode, e)) - raise endpoints.NotFoundException("Patient {} not found.".format(patient_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving patient data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving patient data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/patients_get.py b/api/isb_cgc_api/patients_get.py deleted file mode 100644 index b618de22..00000000 --- a/api/isb_cgc_api/patients_get.py +++ /dev/null @@ -1,114 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message -from message_classes import MetadataItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class PatientsGetQueryBuilder(object): - - def build_queries(self): - clinical_query_str = 'select * ' \ - 'from metadata_clinical ' \ - 'where case_barcode=%s' - - sample_query_str = 'select sample_barcode ' \ - 'from metadata_biospecimen ' \ - 'where case_barcode=%s' - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where case_barcode=%s ' \ - 'group by AliquotBarcode' - - return clinical_query_str, sample_query_str, aliquot_query_str - - -class PatientDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - - -@ISB_CGC_Endpoints.api_class(resource_name='patients') -class PatientsGetAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(patient_barcode=messages.StringField(1, required=True)) - - @endpoints.method(GET_RESOURCE, PatientDetails, - path='patients/{patient_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific patient, - including a list of samples and aliquots derived from this patient. - Takes a patient barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - patient_barcode = request.get_assigned_value('patient_barcode') - query_tuple = (str(patient_barcode),) - clinical_query_str, sample_query_str, aliquot_query_str = PatientsGetQueryBuilder().build_queries() - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build clinical data message - cursor.execute(clinical_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - logger.warn("Patient barcode {} not found in metadata_clinical table.".format(patient_barcode)) - raise endpoints.NotFoundException("Patient barcode {} not found".format(patient_barcode)) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - clinical_data_item = MetadataItem(**constructor_dict) - - # get list of samples - cursor.execute(sample_query_str, query_tuple) - sample_list = [row['sample_barcode'] for row in cursor.fetchall()] - - # get list of aliquots - cursor.execute(aliquot_query_str, query_tuple) - aliquot_list = [row['AliquotBarcode'] for row in cursor.fetchall()] - - return PatientDetails(clinical_data=clinical_data_item, samples=sample_list, aliquots=aliquot_list) - - except (IndexError, TypeError), e: - logger.info("Patient {} not found. Error: {}".format(patient_barcode, e)) - raise endpoints.NotFoundException("Patient {} not found.".format(patient_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving patient, sample, or aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving patient, sample, or aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/samples_annotations.py b/api/isb_cgc_api/samples_annotations.py deleted file mode 100644 index df51455b..00000000 --- a/api/isb_cgc_api/samples_annotations.py +++ /dev/null @@ -1,145 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message -from message_classes import MetadataAnnotationItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class MetadataAnnotationList(messages.Message): - items = messages.MessageField(MetadataAnnotationItem, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -class SamplesAnnotationsQueryBuilder(object): - - @staticmethod - def build_query(item_type_name=None): - query_str = 'select * ' \ - 'from metadata_annotation ' \ - 'where SampleBarcode=%s' - if len(item_type_name) > 0: - query_str += 'and itemTypeName in (' + ', '.join(['%s']*len(item_type_name)) + ')' - - return query_str - - @staticmethod - def build_metadata_samples_query(): - query_str = 'select * ' \ - 'from metadata_samples ' \ - 'where sample_barcode=%s ' - - return query_str - - -@ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesAnnotationAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - item_type_name=messages.StringField(2, repeated=True)) - - @endpoints.method(GET_RESOURCE, MetadataAnnotationList, - path='samples/{sample_barcode}/annotations', http_method='GET') - def annotations(self, request): - """ - Returns TCGA annotations about a specific sample, - Takes a patient barcode (of length , *eg* TCGA-01-0628-11A) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - query_tuple = (str(sample_barcode),) - # check to make sure sample_barcode is in correct form - try: - parts = sample_barcode.split('-') - assert len(parts) == 4 - assert len(parts[0]) == 4 - assert len(parts[1]) == 2 - assert len(parts[2]) == 4 - assert len(parts[3]) == 3 - except AssertionError: - raise endpoints.BadRequestException('{} is not the correct format for a sample barcode. ' - 'Sample barcodes must be of the form XXXX-XX-XXXX-XXX'.format(sample_barcode)) - - item_type_name = request.get_assigned_value('item_type_name') - # check to make sure item_type_name is valid - # check to make sure each item_type_name is valid - if len(item_type_name) > 0: - for itm in item_type_name: - itm = itm.strip() - if itm.lower() not in ['patient', 'aliquot', 'analyte', 'shipped portion', 'portion', 'slide', 'sample']: - raise endpoints.BadRequestException("'{}' is not a valid entry for item_type_name. " - "Valid entries include 'Patient', 'Aliquot', 'Analyte', 'Shipped Portion', " - "'Portion', 'Slide', and 'Sample'".format(itm)) - query_tuple += (itm,) - - query_str = SamplesAnnotationsQueryBuilder().build_query(item_type_name=item_type_name) - metadata_samples_query_str = SamplesAnnotationsQueryBuilder().build_metadata_samples_query() - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build annotation message - cursor.execute(query_str, query_tuple) - rows = cursor.fetchall() - cursor.execute(metadata_samples_query_str, (str(sample_barcode),)) - metadata_sample_rows = cursor.fetchall() - if len(rows) == 0: - cursor.close() - db.close() - if len(metadata_sample_rows) == 0: - msg = "Sample barcode {} not found in the database.".format(sample_barcode) - logger.info(msg) - else: - msg = "No annotations found for sample barcode {}".format(sample_barcode) - if item_type_name is not None: - msg += " and item type name {}. Item type name must be one of the following: " \ - "'Patient', 'Aliquot', 'Analyte', 'Shipped Portion', 'Portion', 'Slide', 'Sample'.".format(item_type_name) - logger.info(msg) - raise endpoints.NotFoundException(msg) - - items = [] - for row in rows: - constructor_dict = build_constructor_dict_for_message(MetadataAnnotationItem(), row) - items.append(MetadataAnnotationItem(**constructor_dict)) - - return MetadataAnnotationList(items=items, count=len(items)) - - except (IndexError, TypeError), e: - logger.info("Patient {} not found. Error: {}".format(sample_barcode, e)) - raise endpoints.NotFoundException("Patient {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving patient data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving patient data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/samples_cloudstoragefilepaths.py b/api/isb_cgc_api/samples_cloudstoragefilepaths.py deleted file mode 100644 index a5faeba2..00000000 --- a/api/isb_cgc_api/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,90 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, are_there_bad_keys, construct_parameter_error_message, \ - CohortsSamplesFilesQueryBuilder, CohortsSamplesFilesMessageBuilder -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class GCSFilePathList(messages.Message): - cloud_storage_file_paths = messages.StringField(1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesCloudStorageFilePathsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, GCSFilePathList, - path='samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - query_str, query_tuple = CohortsSamplesFilesQueryBuilder().build_query( - platform=platform, pipeline=pipeline, sample_barcode=sample_barcode) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - cursor_rows = cursor.fetchall() - # add 'cloud_storage_path' to cursor_rows - bad_repo_count, bad_repo_set = CohortsSamplesFilesMessageBuilder().get_GCS_file_paths_and_bad_repos(cursor_rows) - cloud_storage_path_list = [row['cloud_storage_path'] for row in cursor_rows] - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return GCSFilePathList(cloud_storage_file_paths=cloud_storage_path_list, count=len(cloud_storage_path_list)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for sample {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\t query: {} {}'.format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/samples_get.py b/api/isb_cgc_api/samples_get.py deleted file mode 100644 index d6e260c4..00000000 --- a/api/isb_cgc_api/samples_get.py +++ /dev/null @@ -1,214 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message, \ - CohortsSamplesFilesMessageBuilder -from message_classes import MetadataItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class SamplesGetQueryBuilder(object): - - def build_aliquot_query(self, platform=None, pipeline=None): - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' - - aliquot_query_str += ' and platform=%s ' if platform is not None else '' - aliquot_query_str += ' and pipeline=%s ' if pipeline is not None else '' - aliquot_query_str += ' group by AliquotBarcode' - - return aliquot_query_str - - def build_biospecimen_query(self): - - biospecimen_query_str = 'select * ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s' - - return biospecimen_query_str - - def build_data_query(self, platform=None, pipeline=None): - - data_query_str = 'select ' \ - 'sample_barcode, ' \ - 'DataCenterName, ' \ - 'DataCenterType, ' \ - 'DataFileName, ' \ - 'DataFileNameKey, ' \ - 'DatafileUploaded, ' \ - 'DataLevel,' \ - 'Datatype,' \ - 'GenomeReference,' \ - 'Pipeline,' \ - 'Platform,' \ - 'platform_full_name,' \ - 'program_name,' \ - 'Repository,' \ - 'SDRFFileName,' \ - 'SecurityProtocol ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' \ - 'and DataFileNameKey is not null and DataFileNameKey !=""' - - data_query_str += ' and platform=%s ' if platform is not None else '' - data_query_str += ' and pipeline=%s ' if pipeline is not None else '' - - return data_query_str - - def build_patient_query(self): - - patient_query_str = 'select case_barcode ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s ' \ - 'group by case_barcode' - - return patient_query_str - - -class DataDetails(messages.Message): - sample_Barcode = messages.StringField(1) - DataCenterName = messages.StringField(2) - DataCenterType = messages.StringField(3) - DataFileName = messages.StringField(4) - DataFileNameKey = messages.StringField(5) - DatafileUploaded = messages.StringField(6) - DataLevel = messages.StringField(7) - Datatype = messages.StringField(8) - GenomeReference = messages.StringField(9) - Pipeline = messages.StringField(10) - Platform = messages.StringField(11) - platform_full_name = messages.StringField(12) - Project = messages.StringField(13) - Repository = messages.StringField(14) - SDRFFileName = messages.StringField(15) - SecurityProtocol = messages.StringField(16) - cloud_storage_path = messages.StringField(17) - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(MetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - patient = messages.StringField(3) - data_details = messages.MessageField(DataDetails, 4, repeated=True) - data_details_count = messages.IntegerField(5, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesGetAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, SampleDetails, - path='samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated patient barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - pipeline = request.get_assigned_value('pipeline') - platform = request.get_assigned_value('platform') - - aliquot_query_str = SamplesGetQueryBuilder().build_aliquot_query(platform=platform, pipeline=pipeline) - biospecimen_query_str = SamplesGetQueryBuilder().build_biospecimen_query() - data_query_str = SamplesGetQueryBuilder().build_data_query(platform=platform, pipeline=pipeline) - patient_query_str = SamplesGetQueryBuilder().build_patient_query() - - query_tuple = (str(sample_barcode),) - extra_query_tuple = query_tuple - if pipeline is not None: extra_query_tuple += (pipeline,) - if platform is not None: extra_query_tuple += (platform,) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build biospecimen data message - cursor.execute(biospecimen_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - error_message = "Sample barcode {} not found in metadata_biospecimen table.".format(sample_barcode) - raise endpoints.NotFoundException(error_message) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - biospecimen_data_item = MetadataItem(**constructor_dict) - - # get list of aliquots - cursor.execute(aliquot_query_str, extra_query_tuple) - aliquot_list = [row['AliquotBarcode'] for row in cursor.fetchall()] - - # get patient barcode (superfluous?) - cursor.execute(patient_query_str, query_tuple) - row = cursor.fetchone() - patient_barcode = str(row["case_barcode"]) - - # prepare to build list of data details messages - cursor.execute(data_query_str, extra_query_tuple) - cursor_rows = cursor.fetchall() - # update every dictionary in cursor_rows to contain the full cloud_storage_path for each sample - bad_repo_count, bad_repo_set = \ - CohortsSamplesFilesMessageBuilder().get_GCS_file_paths_and_bad_repos(cursor_rows) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - - # build a data details message for each row returned from metadata_data table - data_details_list = [] - for row in cursor_rows: - constructor_dict = build_constructor_dict_for_message(DataDetails(), row) - data_details_item = DataDetails(**constructor_dict) - data_details_list.append(data_details_item) - - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - - return SampleDetails(aliquots=aliquot_list, - biospecimen_data=biospecimen_data_item, - data_details=data_details_list, - data_details_count=len(data_details_list), - patient=patient_barcode) - - except (IndexError, TypeError) as e: - logger.info("Sample details for barcode {} not found. Error: {}".format(sample_barcode, e)) - raise endpoints.NotFoundException( - "Sample details for barcode {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn(e) - raise endpoints.BadRequestException("Error retrieving biospecimen, patient, or other data. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api/isb_cgc_api/samples_googlegenomics.py b/api/isb_cgc_api/samples_googlegenomics.py deleted file mode 100644 index ec1b3ea1..00000000 --- a/api/isb_cgc_api/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api/isb_cgc_api/users_get.py b/api/isb_cgc_api/users_get.py deleted file mode 100644 index 458bd370..00000000 --- a/api/isb_cgc_api/users_get.py +++ /dev/null @@ -1,143 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import django -import pytz -import datetime -import logging -from protorpc import remote, messages, message_types -from googleapiclient.errors import HttpError - -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from django.conf import settings - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from google_helpers.directory_service import get_directory_resource -from accounts.models import NIH_User - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - - -def is_dbgap_authorized(user_email): - directory_service, http_auth = get_directory_resource() - try: - directory_service.members().get(groupKey="", - memberKey=user_email).execute(http=http_auth) - return True - except HttpError, e: - return False - - -class ReturnJSON(messages.Message): - message = messages.StringField(1) - dbGaP_authorized = messages.BooleanField(2) - - -@ISB_CGC_Endpoints.api_class(resource_name='users') -class UserGetAPI(remote.Service): - - @endpoints.method(message_types.VoidMessage, ReturnJSON, http_method='GET', path='users') - def get(self, request): - ''' - Returns the dbGaP authorization status of the user. - ''' - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException("Authentication unsuccessful.") - - # this checks the controlled-access google group - am_dbgap_authorized = is_dbgap_authorized(user_email) - - if not am_dbgap_authorized: - return ReturnJSON(message="{} is not on the controlled-access google group.".format(user_email), - dbGaP_authorized=False) - - django.setup() - - # all the following five situations should never happen - - # 1. check to make sure they have an entry in auth_user - try: - django_user = Django_User.objects.get(email=user_email) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error("Email {} is in {} group but did not have a unique entry in auth_user table. Error: {}" - .format(user_email, "", e)) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but does not have an entry in the user database." - .format(user_email)) - - # 2. check to make sure they have an entry in accounts_nih_user - try: - nih_user = NIH_User.objects.get(user_id=django_user.id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error("Email {} is in {} group but did not have a unique entry in " - "accounts_nih_user table. Error: {}" - .format(user_email, "", e)) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but does not have an entry in the nih_user database." - .format(user_email)) - - # 3. check if their entry in accounts_nih_user is currently active - if not nih_user.active: - logger.error("Email {} is in {} group but their entry in accounts_nih_user table is inactive." - .format(user_email, "")) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but has an inactive entry in the nih_user database." - .format(user_email)) - - # 4. check if their entry in accounts_nih_user is dbGaP_authorized - if not nih_user.dbGaP_authorized: - logger.error("Email {} is in {} group but their entry in accounts_nih_user table " - "is not dbGaP_authorized." - .format(user_email, "")) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but their entry in the nih_user database is not dbGaP_authorized." - .format(user_email)) - - # 5. check if their dbgap authorization has expired - expire_time = nih_user.NIH_assertion_expiration - expire_time = expire_time if expire_time.tzinfo is not None else pytz.utc.localize(expire_time) - now_in_utc = pytz.utc.localize(datetime.datetime.now()) - - if (expire_time - now_in_utc).total_seconds() <= 0: - logger.error("Email {} is in {} group but their entry in accounts_nih_user table " - "is expired." - .format(user_email, "")) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but their entry in the nih_user database is expired." - .format(user_email)) - - # all checks have passed - request_finished.send(self) - return ReturnJSON(message="{} has dbGaP authorization and is a member of the controlled-access google group." - .format(user_email), - dbGaP_authorized=True) \ No newline at end of file diff --git a/api/maf.py b/api/maf.py deleted file mode 100755 index 3c61ee0d..00000000 --- a/api/maf.py +++ /dev/null @@ -1,104 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -from protorpc import remote -from protorpc.messages import Message, MessageField, IntegerField, StringField, Variant -from MySQLdb.cursors import DictCursor - -from api_helpers import * - - -# Generated code - in scripts/maf_import in Python do: -# -# > import import_maf -# > repr(import_maf.FIELD_NAMES_AND_TYPES) -# -FIELD_NAMES_AND_TYPES = [ - {'type': 'varchar(128)', 'name': 'hugo_symbol'}, - {'type': 'int', 'name': 'amino_acid_position'}, - {'type': 'varchar(128)', 'name': 'tumor_type'}, - {'type': 'varchar(12)', 'name': 'uniprot_id'}, - {'type': 'varchar(128)', 'name': 'variant_classification'}, - {'type': 'varchar(128)', 'name': 'c_position'}, - {'type': 'varchar(128)', 'name': 'tumor_sample_barcode'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'reference_allele'}, - {'type': 'varchar(128)', 'name': 'variant_type'}, - {'type': 'varchar(128)', 'name': 'ucsc_cons'} -] - - -class MAFRecord(Message): - hugo_symbol = StringField(1) - tumor_type = StringField(3) - amino_acid_position = IntegerField(2, variant=Variant.INT32) - uniprot_id = StringField(4) - variant_classification = StringField(5) - c_position = StringField(6) - tumor_sample_barcode = StringField(7) - match_norm_seq_allele2 = StringField(8) - tumor_seq_allele1 = StringField(9) - tumor_seq_allele2 = StringField(10) - match_norm_seq_allele1 = StringField(11) - reference_allele = StringField(12) - variant_type = StringField(13) - ucsc_cons = StringField(14) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - - -class MAFRequest(Message): - gene = StringField(1, required=True) - tumor = StringField(2, repeated=True) - -MAFEndpointsAPI = endpoints.api(name='maf_api', version='v1') - - -@MAFEndpointsAPI .api_class(resource_name='maf_endpoints') -class MAFEndpointsAPI(remote.Service): - @endpoints.method(MAFRequest, MAFRecordList, - path='maf_search', http_method='GET', name='maf.getMAF') - def maf_search(self, request): - gene = request.gene - tumor_type_list = request.tumor - tumor_set_template = ', '.join(['%s' for x in range(len(tumor_type_list))]) - query = 'SELECT * FROM maf WHERE hugo_symbol=%s AND tumor_type IN ({0})'.format(tumor_set_template) - - values = [gene] - values.extend(tumor_type_list) - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - data_list = [] - for row in cursor.fetchall(): - data_list.append(MAFRecord(**row)) - - cursor.close() - db.close() - return MAFRecordList(items=data_list) - - except: - raise endpoints.NotFoundException('MAF query error') \ No newline at end of file diff --git a/api/metadata.py b/api/metadata.py deleted file mode 100755 index 1eff4889..00000000 --- a/api/metadata.py +++ /dev/null @@ -1,2754 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import time -from protorpc import messages -from protorpc import message_types -from protorpc import remote -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from allauth.socialaccount.models import SocialToken, SocialAccount -from accounts.models import NIH_User -from cohorts.models import Cohort_Perms, Cohort as Django_Cohort, Samples, Filters -from projects.models import Project, User_Feature_Definitions, User_Feature_Counts, User_Data_Tables -from django.core.signals import request_finished -from time import sleep -import django -import sys -import logging -import re -import json -import traceback -import copy -from uuid import uuid4 - -from api.api_helpers import * - -logger = logging.getLogger(__name__) - -debug = settings.DEBUG - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID -OPEN_DATA_BUCKET = settings.OPEN_DATA_BUCKET -#CONTROLLED_DATA_BUCKET = settings.CONTROLLED_DATA_BUCKET - -METADATA_SHORTLIST = [ - # 'adenocarcinoma_invasion', - 'age_at_initial_pathologic_diagnosis', - # 'anatomic_neoplasm_subdivision', - # 'avg_percent_lymphocyte_infiltration', - # 'avg_percent_monocyte_infiltration', - # 'avg_percent_necrosis', - # 'avg_percent_neutrophil_infiltration', - # 'avg_percent_normal_cells', - # 'avg_percent_stromal_cells', - # 'avg_percent_tumor_cells', - # 'avg_percent_tumor_nuclei', - # 'batch_number', - # 'bcr', - 'BMI', - # 'clinical_M', - # 'clinical_N', - # 'clinical_stage', - # 'clinical_T', - # 'colorectal_cancer', - # 'country', - # 'country_of_procurement', - # 'days_to_birth', - # 'days_to_collection', - # 'days_to_death', - # 'days_to_initial_pathologic_diagnosis', - # 'days_to_last_followup', - # 'days_to_submitted_specimen_dx', - 'Study', - # 'ethnicity', - # 'frozen_specimen_anatomic_site', - 'gender', - # 'gleason_score_combined', - 'has_27k', - 'has_450k', - 'has_BCGSC_GA_RNASeq', - 'has_BCGSC_HiSeq_RNASeq', - 'has_GA_miRNASeq', - 'has_HiSeq_miRnaSeq', - 'has_Illumina_DNASeq', - 'has_RPPA', - 'has_SNP6', - 'has_UNC_GA_RNASeq', - 'has_UNC_HiSeq_RNASeq', - # 'height', - 'histological_type', - # 'history_of_colon_polyps', - # 'history_of_neoadjuvant_treatment', - # 'history_of_prior_malignancy', - # 'hpv_calls', - 'hpv_status', - 'icd_10', - 'icd_o_3_histology', - 'icd_o_3_site', - # 'lymph_node_examined_count', - # 'lymphatic_invasion', - # 'lymphnodes_examined', - # 'lymphovascular_invasion_present', - # 'max_percent_lymphocyte_infiltration', - # 'max_percent_monocyte_infiltration', - # 'max_percent_necrosis', - # 'max_percent_neutrophil_infiltration', - # 'max_percent_normal_cells', - # 'max_percent_stromal_cells', - # 'max_percent_tumor_cells', - # 'max_percent_tumor_nuclei', - # 'menopause_status', - # 'min_percent_lymphocyte_infiltration', - # 'min_percent_monocyte_infiltration', - # 'min_percent_necrosis', - # 'min_percent_neutrophil_infiltration', - # 'min_percent_normal_cells', - # 'min_percent_stromal_cells', - # 'min_percent_tumor_cells', - # 'min_percent_tumor_nuclei', - # 'mononucleotide_and_dinucleotide_marker_panel_analysis_status', - # 'mononucleotide_marker_panel_analysis_status', - 'neoplasm_histologic_grade', - 'new_tumor_event_after_initial_treatment', - # 'number_of_lymphnodes_examined', - # 'number_of_lymphnodes_positive_by_he', - # 'number_pack_years_smoked', - # 'other_dx', - # 'case_barcode', - # 'pathologic_M', - # 'pathologic_N', - 'pathologic_stage', - # 'pathologic_T', - 'person_neoplasm_cancer_status', - # 'pregnancies', - # 'preservation_method', - # 'primary_neoplasm_melanoma_dx', - # 'primary_therapy_outcome_success', - 'Project', - # 'psa_value', - # 'race', - 'residual_tumor', - # 'sample_barcode', - 'SampleTypeCode', - # 'Study', - 'tobacco_smoking_history', - # 'total_number_of_pregnancies', - # 'tumor_pathology', - 'tumor_tissue_site', - 'tumor_type', - # 'weiss_venous_invasion', - 'vital_status' - # 'weight', - # 'year_of_initial_pathologic_diagnosis', -] - -metadata_dict = { - 'age_at_initial_pathologic_diagnosis': 'INTEGER', - 'anatomic_neoplasm_subdivision': 'VARCHAR(63)', - 'avg_percent_lymphocyte_infiltration': 'FLOAT', - 'avg_percent_monocyte_infiltration': 'FLOAT', - 'avg_percent_necrosis': 'FLOAT', - 'avg_percent_neutrophil_infiltration': 'FLOAT', - 'avg_percent_normal_cells': 'FLOAT', - 'avg_percent_stromal_cells': 'FLOAT', - 'avg_percent_tumor_cells': 'FLOAT', - 'avg_percent_tumor_nuclei': 'FLOAT', - 'batch_number': 'INTEGER', - 'bcr': 'VARCHAR(63)', - 'BMI': 'FLOAT', - 'clinical_M': 'VARCHAR(12)', - 'clinical_N': 'VARCHAR(12)', - 'clinical_T': 'VARCHAR(12)', - 'clinical_stage': 'VARCHAR(12)', - 'colorectal_cancer': 'VARCHAR(10)', - 'country': 'VARCHAR(63)', - 'days_to_birth': 'INTEGER', - 'days_to_collection': 'INTEGER', - 'days_to_death': 'INTEGER', - 'days_to_initial_pathologic_diagnosis': 'INTEGER', - 'days_to_last_followup': 'INTEGER', - 'days_to_submitted_specimen_dx': 'INTEGER', - 'Study': 'VARCHAR(4)', - 'ethnicity': 'VARCHAR(20)', - 'frozen_specimen_anatomic_site': 'VARCHAR(63)', - 'gender': 'VARCHAR(15)', - 'gleason_score_combined': 'INTEGER', - 'height': 'INTEGER', - 'histological_type': 'VARCHAR(63)', - 'history_of_colon_polyps': 'VARCHAR(8)', - 'history_of_neoadjuvant_treatment': 'VARCHAR(63)', - 'history_of_prior_malignancy': 'VARCHAR(25)', - 'hpv_calls': 'VARCHAR(20)', - 'hpv_status': 'VARCHAR(20)', - 'icd_10': 'VARCHAR(8)', - 'icd_o_3_histology': 'VARCHAR(10)', - 'icd_o_3_site': 'VARCHAR(8)', - 'lymphatic_invasion': 'VARCHAR(8)', - 'lymphnodes_examined': 'VARCHAR(8)', - 'lymphovascular_invasion_present': 'VARCHAR(63)', - 'max_percent_lymphocyte_infiltration': 'INTEGER', - 'max_percent_monocyte_infiltration': 'INTEGER', - 'max_percent_necrosis': 'INTEGER', - 'max_percent_neutrophil_infiltration': 'INTEGER', - 'max_percent_normal_cells': 'INTEGER', - 'max_percent_stromal_cells': 'INTEGER', - 'max_percent_tumor_cells': 'INTEGER', - 'max_percent_tumor_nuclei': 'INTEGER', - 'menopause_status': 'VARCHAR(30)', - 'min_percent_lymphocyte_infiltration': 'INTEGER', - 'min_percent_monocyte_infiltration': 'INTEGER', - 'min_percent_necrosis': 'INTEGER', - 'min_percent_neutrophil_infiltration': 'INTEGER', - 'min_percent_normal_cells': 'INTEGER', - 'min_percent_stromal_cells': 'INTEGER', - 'min_percent_tumor_cells': 'INTEGER', - 'min_percent_tumor_nuclei': 'INTEGER', - 'mononucleotide_and_dinucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'mononucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'neoplasm_histologic_grade': 'VARCHAR(15)', - 'new_tumor_event_after_initial_treatment': 'VARCHAR(8)', - 'number_of_lymphnodes_examined': 'INTEGER', - 'number_of_lymphnodes_positive_by_he': 'INTEGER', - 'number_pack_years_smoked': 'INTEGER', - 'case_barcode': 'VARCHAR(12)', - 'pathologic_M': 'VARCHAR(5)', - 'pathologic_N': 'VARCHAR(5)', - 'pathologic_T': 'VARCHAR(5)', - 'pathologic_stage': 'VARCHAR(10)', - 'person_neoplasm_cancer_status': 'VARCHAR(15)', - 'pregnancies': 'VARCHAR(35)', - 'primary_neoplasm_melanoma_dx': 'VARCHAR(10)', - 'primary_therapy_outcome_success': 'VARCHAR(35)', - 'prior_dx': 'VARCHAR(50)', - 'Project': 'VARCHAR(4)', - 'psa_value': 'FLOAT', - 'race': 'VARCHAR(30)', - 'residual_tumor': 'VARCHAR(5)', - 'sample_barcode': 'VARCHAR(16)', - 'Study': 'VARCHAR(4)', - 'tobacco_smoking_history': 'VARCHAR(30)', - 'tumor_tissue_site': 'VARCHAR(20)', - 'tumor_type': 'VARCHAR(4)', - 'weiss_venous_invasion': 'VARCHAR(63)', - 'vital_status': 'VARCHAR(63)', - 'weight': 'VARCHAR(63)', - 'year_of_initialPY_pathologic_diagnosis': 'VARCHAR(63)', - 'SampleTypeCode': 'VARCHAR(3)', - 'has_Illumina_DNASeq': 'TINYINT', - 'has_BCGSC_HiSeq_RNASeq': 'TINYINT', - 'has_UNC_HiSeq_RNASeq': 'TINYINT', - 'has_BCGSC_GA_RNASeq': 'TINYINT', - 'has_UNC_GA_RNASeq': 'TINYINT', - 'has_HiSeq_miRnaSeq': 'TINYINT', - 'has_GA_miRNASeq': 'TINYINT', - 'has_RPPA': 'TINYINT', - 'has_SNP6': 'TINYINT', - 'has_27k': 'TINYINT', - 'has_450k': 'TINYINT', -} - - -class MetaValueListCount(messages.Message): - value = messages.StringField(1) # note: this means converting booleans to strings - count = messages.IntegerField(2) - - -class MetaAttrValuesList(messages.Message): - adenocarcinoma_invasion = messages.MessageField(MetaValueListCount, 1, repeated=True) - age_at_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 2, repeated=True) - anatomic_neoplasm_subdivision = messages.MessageField(MetaValueListCount, 3, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_necrosis = messages.FloatField(6, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(7, repeated=True) - avg_percent_normal_cells = messages.FloatField(8, repeated=True) - avg_percent_stromal_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_cells = messages.FloatField(10, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(11, repeated=True) - batch_number = messages.MessageField(MetaValueListCount, 12, repeated=True) - bcr = messages.MessageField(MetaValueListCount, 13, repeated=True) - clinical_M = messages.MessageField(MetaValueListCount, 14, repeated=True) - clinical_N = messages.MessageField(MetaValueListCount, 15, repeated=True) - clinical_stage = messages.MessageField(MetaValueListCount, 16, repeated=True) - clinical_T = messages.MessageField(MetaValueListCount, 17, repeated=True) - colorectal_cancer = messages.MessageField(MetaValueListCount, 18, repeated=True) - country = messages.MessageField(MetaValueListCount, 19, repeated=True) - country_of_procurement = messages.MessageField(MetaValueListCount, 20, repeated=True) - days_to_birth = messages.MessageField(MetaValueListCount, 21, repeated=True) - days_to_collection = messages.MessageField(MetaValueListCount, 22, repeated=True) - days_to_death = messages.MessageField(MetaValueListCount, 23, repeated=True) - days_to_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 24, repeated=True) - days_to_last_followup = messages.MessageField(MetaValueListCount, 25, repeated=True) - days_to_submitted_specimen_dx = messages.MessageField(MetaValueListCount, 26, repeated=True) - Study = messages.MessageField(MetaValueListCount, 27, repeated=True) - ethnicity = messages.MessageField(MetaValueListCount, 28, repeated=True) - frozen_specimen_anatomic_site = messages.MessageField(MetaValueListCount, 29, repeated=True) - gender = messages.MessageField(MetaValueListCount, 30, repeated=True) - height = messages.MessageField(MetaValueListCount, 31, repeated=True) - histological_type = messages.MessageField(MetaValueListCount, 32, repeated=True) - history_of_colon_polyps = messages.MessageField(MetaValueListCount, 33, repeated=True) - history_of_neoadjuvant_treatment = messages.MessageField(MetaValueListCount, 34, repeated=True) - history_of_prior_malignancy = messages.MessageField(MetaValueListCount, 35, repeated=True) - hpv_calls = messages.MessageField(MetaValueListCount, 36, repeated=True) - hpv_status = messages.MessageField(MetaValueListCount, 37, repeated=True) - icd_10 = messages.MessageField(MetaValueListCount, 38, repeated=True) - icd_o_3_histology = messages.MessageField(MetaValueListCount, 39, repeated=True) - icd_o_3_site = messages.MessageField(MetaValueListCount, 40, repeated=True) - lymph_node_examined_count = messages.MessageField(MetaValueListCount, 41, repeated=True) - lymphatic_invasion = messages.MessageField(MetaValueListCount, 42, repeated=True) - lymphnodes_examined = messages.MessageField(MetaValueListCount, 43, repeated=True) - lymphovascular_invasion_present = messages.MessageField(MetaValueListCount, 44, repeated=True) - max_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 45, repeated=True) - max_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 46, repeated=True) - max_percent_necrosis = messages.MessageField(MetaValueListCount, 47, repeated=True) - max_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 48, repeated=True) - max_percent_normal_cells = messages.MessageField(MetaValueListCount, 49, repeated=True) - max_percent_stromal_cells = messages.MessageField(MetaValueListCount, 50, repeated=True) - max_percent_tumor_cells = messages.MessageField(MetaValueListCount, 51, repeated=True) - max_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 52, repeated=True) - menopause_status = messages.MessageField(MetaValueListCount, 53, repeated=True) - min_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 54, repeated=True) - min_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 55, repeated=True) - min_percent_necrosis = messages.MessageField(MetaValueListCount, 56, repeated=True) - min_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 57, repeated=True) - min_percent_normal_cells = messages.MessageField(MetaValueListCount, 58, repeated=True) - min_percent_stromal_cells = messages.MessageField(MetaValueListCount, 59, repeated=True) - min_percent_tumor_cells = messages.MessageField(MetaValueListCount, 60, repeated=True) - min_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 61, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 62, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 63, repeated=True) - neoplasm_histologic_grade = messages.MessageField(MetaValueListCount, 64, repeated=True) - new_tumor_event_after_initial_treatment = messages.MessageField(MetaValueListCount, 65, repeated=True) - number_of_lymphnodes_examined = messages.MessageField(MetaValueListCount, 66, repeated=True) - number_of_lymphnodes_positive_by_he = messages.MessageField(MetaValueListCount, 67, repeated=True) - case_barcode = messages.MessageField(MetaValueListCount, 68, repeated=True) - pathologic_M = messages.MessageField(MetaValueListCount, 69, repeated=True) - pathologic_N = messages.MessageField(MetaValueListCount, 70, repeated=True) - pathologic_stage = messages.MessageField(MetaValueListCount, 71, repeated=True) - pathologic_T = messages.MessageField(MetaValueListCount, 72, repeated=True) - person_neoplasm_cancer_status = messages.MessageField(MetaValueListCount, 73, repeated=True) - pregnancies = messages.MessageField(MetaValueListCount, 74, repeated=True) - preservation_method = messages.MessageField(MetaValueListCount, 75, repeated=True) - primary_neoplasm_melanoma_dx = messages.MessageField(MetaValueListCount, 76, repeated=True) - primary_therapy_outcome_success = messages.MessageField(MetaValueListCount, 77, repeated=True) - prior_dx = messages.MessageField(MetaValueListCount, 78, repeated=True) - Project = messages.MessageField(MetaValueListCount, 79, repeated=True) - psa_value = messages.FloatField(80, repeated=True) - race = messages.MessageField(MetaValueListCount, 81, repeated=True) - residual_tumor = messages.MessageField(MetaValueListCount, 82, repeated=True) - sample_barcode = messages.MessageField(MetaValueListCount, 83, repeated=True) - tobacco_smoking_history = messages.MessageField(MetaValueListCount, 86, repeated=True) - total_number_of_pregnancies = messages.MessageField(MetaValueListCount, 87, repeated=True) - tumor_tissue_site = messages.MessageField(MetaValueListCount, 88, repeated=True) - tumor_pathology = messages.MessageField(MetaValueListCount, 89, repeated=True) - tumor_type = messages.MessageField(MetaValueListCount, 90, repeated=True) - weiss_venous_invasion = messages.MessageField(MetaValueListCount, 91, repeated=True) - vital_status = messages.MessageField(MetaValueListCount, 92, repeated=True) - weight = messages.MessageField(MetaValueListCount, 93, repeated=True) - year_of_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 94, repeated=True) - SampleTypeCode = messages.MessageField(MetaValueListCount, 95, repeated=True) - has_Illumina_DNASeq = messages.MessageField(MetaValueListCount, 96, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 97, repeated=True) - has_UNC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 98, repeated=True) - has_BCGSC_GA_RNASeq = messages.MessageField(MetaValueListCount, 99, repeated=True) - has_UNC_GA_RNASeq = messages.MessageField(MetaValueListCount, 100, repeated=True) - has_HiSeq_miRnaSeq = messages.MessageField(MetaValueListCount, 101, repeated=True) - has_GA_miRNASeq = messages.MessageField(MetaValueListCount, 102, repeated=True) - has_RPPA = messages.MessageField(MetaValueListCount, 103, repeated=True) - has_SNP6 = messages.MessageField(MetaValueListCount, 104, repeated=True) - has_27k = messages.MessageField(MetaValueListCount, 105, repeated=True) - has_450k = messages.MessageField(MetaValueListCount, 106, repeated=True) - BMI = messages.FloatField(107, repeated=True) - - -class MetadataItem(messages.Message): - adenocarcinoma_invasion = messages.StringField(1) - age_at_initial_pathologic_diagnosis = messages.IntegerField(2) - anatomic_neoplasm_subdivision = messages.StringField(3) - avg_percent_lymphocyte_infiltration = messages.FloatField(4) - avg_percent_monocyte_infiltration = messages.FloatField(5) - avg_percent_necrosis = messages.FloatField(6) - avg_percent_neutrophil_infiltration = messages.FloatField(7) - avg_percent_normal_cells = messages.FloatField(8) - avg_percent_stromal_cells = messages.FloatField(9) - avg_percent_tumor_cells = messages.FloatField(10) - avg_percent_tumor_nuclei = messages.FloatField(11) - batch_number = messages.IntegerField(12) - bcr = messages.StringField(13) - clinical_M = messages.StringField(14) - clinical_N = messages.StringField(15) - clinical_stage = messages.StringField(16) - clinical_T = messages.StringField(17) - colorectal_cancer = messages.StringField(18) - country = messages.StringField(19) - country_of_procurement = messages.StringField(20) - days_to_birth = messages.IntegerField(21) - days_to_collection = messages.IntegerField(22) - days_to_death = messages.IntegerField(23) - days_to_initial_pathologic_diagnosis = messages.IntegerField(24) - days_to_last_followup = messages.IntegerField(25) - days_to_submitted_specimen_dx = messages.IntegerField(26) - Study = messages.StringField(27) - ethnicity = messages.StringField(28) - frozen_specimen_anatomic_site = messages.StringField(29) - gender = messages.StringField(30) - height = messages.IntegerField(31) - histological_type = messages.StringField(32) - history_of_colon_polyps = messages.StringField(33) - history_of_neoadjuvant_treatment = messages.StringField(34) - history_of_prior_malignancy = messages.StringField(35) - hpv_calls = messages.StringField(36) - hpv_status = messages.StringField(37) - icd_10 = messages.StringField(38) - icd_o_3_histology = messages.StringField(39) - icd_o_3_site = messages.StringField(40) - lymph_node_examined_count = messages.IntegerField(41) - lymphatic_invasion = messages.StringField(42) - lymphnodes_examined = messages.StringField(43) - lymphovascular_invasion_present = messages.StringField(44) - max_percent_lymphocyte_infiltration = messages.IntegerField(45) - max_percent_monocyte_infiltration = messages.IntegerField(46) - max_percent_necrosis = messages.IntegerField(47) - max_percent_neutrophil_infiltration = messages.IntegerField(48) - max_percent_normal_cells = messages.IntegerField(49) - max_percent_stromal_cells = messages.IntegerField(50) - max_percent_tumor_cells = messages.IntegerField(51) - max_percent_tumor_nuclei = messages.IntegerField(52) - menopause_status = messages.StringField(53) - min_percent_lymphocyte_infiltration = messages.IntegerField(54) - min_percent_monocyte_infiltration = messages.IntegerField(55) - min_percent_necrosis = messages.IntegerField(56) - min_percent_neutrophil_infiltration = messages.IntegerField(57) - min_percent_normal_cells = messages.IntegerField(58) - min_percent_stromal_cells = messages.IntegerField(59) - min_percent_tumor_cells = messages.IntegerField(60) - min_percent_tumor_nuclei = messages.IntegerField(61) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(62) - mononucleotide_marker_panel_analysis_status = messages.StringField(63) - neoplasm_histologic_grade = messages.StringField(64) - new_tumor_event_after_initial_treatment = messages.StringField(65) - number_of_lymphnodes_examined = messages.IntegerField(66) - number_of_lymphnodes_positive_by_he = messages.IntegerField(67) - case_barcode = messages.StringField(68) - pathologic_M = messages.StringField(69) - pathologic_N = messages.StringField(70) - pathologic_stage = messages.StringField(71) - pathologic_T = messages.StringField(72) - person_neoplasm_cancer_status = messages.StringField(73) - pregnancies = messages.StringField(74) - preservation_method = messages.StringField(75) - primary_neoplasm_melanoma_dx = messages.StringField(76) - primary_therapy_outcome_success = messages.StringField(77) - prior_dx = messages.StringField(78) - Project = messages.StringField(79) - psa_value = messages.FloatField(80) - race = messages.StringField(81) - residual_tumor = messages.StringField(82) - sample_barcode = messages.StringField(83) - tobacco_smoking_history = messages.StringField(86) - total_number_of_pregnancies = messages.IntegerField(87) - tumor_tissue_site = messages.StringField(88) - tumor_pathology = messages.StringField(89) - tumor_type = messages.StringField(90) - weiss_venous_invasion = messages.StringField(91) - vital_status = messages.StringField(92) - weight = messages.IntegerField(93) - year_of_initial_pathologic_diagnosis = messages.StringField(94) - SampleTypeCode = messages.StringField(95) - has_Illumina_DNASeq = messages.StringField(96) - has_BCGSC_HiSeq_RNASeq = messages.StringField(97) - has_UNC_HiSeq_RNASeq = messages.StringField(98) - has_BCGSC_GA_RNASeq = messages.StringField(99) - has_UNC_GA_RNASeq = messages.StringField(100) - has_HiSeq_miRnaSeq = messages.StringField(101) - has_GA_miRNASeq = messages.StringField(102) - has_RPPA = messages.StringField(103) - has_SNP6 = messages.StringField(104) - has_27k = messages.StringField(105) - has_450k = messages.StringField(106) - BMI = messages.FloatField(107) - -''' -Incoming object needs to use age and BMI that's a string (eg. 10_to_39) -''' -class IncomingMetadataItem(messages.Message): - age_at_initial_pathologic_diagnosis = messages.StringField(1, repeated=True) - anatomic_neoplasm_subdivision = messages.StringField(2, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(3, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_necrosis = messages.FloatField(5, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(6, repeated=True) - avg_percent_normal_cells = messages.FloatField(7, repeated=True) - avg_percent_stromal_cells = messages.FloatField(8, repeated=True) - avg_percent_tumor_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(10, repeated=True) - batch_number = messages.IntegerField(11, repeated=True) - bcr = messages.StringField(12, repeated=True) - clinical_M = messages.StringField(13, repeated=True) - clinical_N = messages.StringField(14, repeated=True) - clinical_stage = messages.StringField(15, repeated=True) - clinical_T = messages.StringField(16, repeated=True) - colorectal_cancer = messages.StringField(17, repeated=True) - country = messages.StringField(18, repeated=True) - days_to_birth = messages.IntegerField(19, repeated=True) - days_to_collection = messages.IntegerField(20, repeated=True) - days_to_death = messages.IntegerField(21, repeated=True) - days_to_initial_pathologic_diagnosis = messages.IntegerField(22, repeated=True) - days_to_last_followup = messages.IntegerField(23, repeated=True) - days_to_submitted_specimen_dx = messages.IntegerField(24, repeated=True) - Study = messages.StringField(25, repeated=True) - ethnicity = messages.StringField(26, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(27, repeated=True) - gender = messages.StringField(28, repeated=True) - height = messages.IntegerField(29, repeated=True) - histological_type = messages.StringField(30, repeated=True) - history_of_colon_polyps = messages.StringField(31, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(32, repeated=True) - history_of_prior_malignancy = messages.StringField(33, repeated=True) - hpv_calls = messages.StringField(34, repeated=True) - hpv_status = messages.StringField(35, repeated=True) - icd_10 = messages.StringField(36, repeated=True) - icd_o_3_histology = messages.StringField(37, repeated=True) - icd_o_3_site = messages.StringField(38, repeated=True) - lymphatic_invasion = messages.StringField(39, repeated=True) - lymphnodes_examined = messages.StringField(40, repeated=True) - lymphovascular_invasion_present = messages.StringField(41, repeated=True) - max_percent_lymphocyte_infiltration = messages.IntegerField(42, repeated=True) - max_percent_monocyte_infiltration = messages.IntegerField(43, repeated=True) - max_percent_necrosis = messages.IntegerField(44, repeated=True) - max_percent_neutrophil_infiltration = messages.IntegerField(45, repeated=True) - max_percent_normal_cells = messages.IntegerField(46, repeated=True) - max_percent_stromal_cells = messages.IntegerField(47, repeated=True) - max_percent_tumor_cells = messages.IntegerField(48, repeated=True) - max_percent_tumor_nuclei = messages.IntegerField(49, repeated=True) - menopause_status = messages.StringField(50, repeated=True) - min_percent_lymphocyte_infiltration = messages.IntegerField(51, repeated=True) - min_percent_monocyte_infiltration = messages.IntegerField(52, repeated=True) - min_percent_necrosis = messages.IntegerField(53, repeated=True) - min_percent_neutrophil_infiltration = messages.IntegerField(54, repeated=True) - min_percent_normal_cells = messages.IntegerField(55, repeated=True) - min_percent_stromal_cells = messages.IntegerField(56, repeated=True) - min_percent_tumor_cells = messages.IntegerField(57, repeated=True) - min_percent_tumor_nuclei = messages.IntegerField(58, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(59, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(60, repeated=True) - neoplasm_histologic_grade = messages.StringField(61, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(62, repeated=True) - number_of_lymphnodes_examined = messages.IntegerField(63, repeated=True) - number_of_lymphnodes_positive_by_he = messages.IntegerField(64, repeated=True) - case_barcode = messages.StringField(65, repeated=True) - pathologic_M = messages.StringField(66, repeated=True) - pathologic_N = messages.StringField(67, repeated=True) - pathologic_stage = messages.StringField(68, repeated=True) - pathologic_T = messages.StringField(69, repeated=True) - person_neoplasm_cancer_status = messages.StringField(70, repeated=True) - pregnancies = messages.StringField(71, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(72, repeated=True) - primary_therapy_outcome_success = messages.StringField(73, repeated=True) - prior_dx = messages.StringField(74, repeated=True) - Project = messages.StringField(75, repeated=True) - psa_value = messages.FloatField(76, repeated=True) - race = messages.StringField(77, repeated=True) - residual_tumor = messages.StringField(78, repeated=True) - sample_barcode = messages.StringField(79, repeated=True) - tobacco_smoking_history = messages.StringField(80, repeated=True) - tumor_tissue_site = messages.StringField(81, repeated=True) - tumor_type = messages.StringField(82, repeated=True) - weiss_venous_invasion = messages.StringField(83, repeated=True) - vital_status = messages.StringField(84, repeated=True) - weight = messages.IntegerField(85, repeated=True) - year_of_initial_pathologic_diagnosis = messages.StringField(86, repeated=True) - SampleTypeCode = messages.StringField(87, repeated=True) - has_Illumina_DNASeq = messages.StringField(88, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(89, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(90, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(91, repeated=True) - has_UNC_GA_RNASeq = messages.StringField(92, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(93, repeated=True) - has_GA_miRNASeq = messages.StringField(94, repeated=True) - has_RPPA = messages.StringField(95, repeated=True) - has_SNP6 = messages.StringField(96, repeated=True) - has_27k = messages.StringField(97, repeated=True) - has_450k = messages.StringField(98, repeated=True) - BMI = messages.StringField(99, repeated=True) - -class MetadataAttributeValues(messages.Message): - name = messages.StringField(1) - id = messages.StringField(2) - values = messages.MessageField(MetaValueListCount, 3, repeated=True) - total = messages.IntegerField(4) - -class MetadataItemList(messages.Message): - items = messages.MessageField(MetadataItem, 1, repeated=True) - count = messages.MessageField(MetaAttrValuesList, 2) - total = messages.IntegerField(3) - -class MetadataCountsItem(messages.Message): - count = messages.MessageField(MetadataAttributeValues, 1, repeated=True) - total = messages.IntegerField(2) - -class MetaDomainsList(messages.Message): - gender = messages.StringField(1, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(2, repeated=True) - country = messages.StringField(3, repeated=True) - Study = messages.StringField(4, repeated=True) - ethnicity = messages.StringField(5, repeated=True) - histological_type = messages.StringField(6, repeated=True) - icd_10 = messages.StringField(7, repeated=True) - icd_o_3_histology = messages.StringField(8, repeated=True) - icd_o_3_site = messages.StringField(9, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(10, repeated=True) - neoplasm_histologic_grade = messages.StringField(11, repeated=True) - pathologic_N = messages.StringField(12, repeated=True) - pathologic_T = messages.StringField(13, repeated=True) - pathologic_stage = messages.StringField(14, repeated=True) - person_neoplasm_cancer_status = messages.StringField(15, repeated=True) - prior_dx = messages.StringField(16, repeated=True) - Project = messages.StringField(17, repeated=True) - race = messages.StringField(18, repeated=True) - residual_tumor = messages.StringField(19, repeated=True) - SampleTypeCode = messages.StringField(20, repeated=True) - tumor_tissue_site = messages.StringField(21, repeated=True) - tumor_type = messages.StringField(22, repeated=True) - vital_status = messages.StringField(23, repeated=True) - has_Illumina_DNASeq = messages.StringField(24, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(25, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(26, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(27, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(28, repeated=True) - has_GA_miRNASeq = messages.StringField(29, repeated=True) - has_RPPA = messages.StringField(30, repeated=True) - has_SNP6 = messages.StringField(31, repeated=True) - has_27k = messages.StringField(32, repeated=True) - has_450k = messages.StringField(33, repeated=True) - - -class SampleBarcodeItem(messages.Message): - sample_barcode = messages.StringField(1) - study_id = messages.IntegerField(2) - -class MetadataAttr(messages.Message): - attribute = messages.StringField(1) - code = messages.StringField(2) - spec = messages.StringField(3) - key = messages.StringField(4) - - -class MetadataAttrList(messages.Message): - items = messages.MessageField(MetadataAttr, 1, repeated=True) - count = messages.IntegerField(2) - - -class SampleBarcodeList(messages.Message): - items = messages.MessageField(SampleBarcodeItem, 1, repeated=True) - count = messages.IntegerField(2) - - -class MetadataPlatformItem(messages.Message): - DNAseq_data = messages.StringField(1) - cnvrPlatform = messages.StringField(2) - gexpPlatform = messages.StringField(3) - methPlatform = messages.StringField(4) - mirnPlatform = messages.StringField(5) - rppaPlatform = messages.StringField(6) - - -class MetadataPlatformItemList(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - - -class MetadataCountsPlatformItem(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - count = messages.MessageField(MetadataAttributeValues, 2, repeated=True) - participants = messages.IntegerField(3) - total = messages.IntegerField(4) - - -def createDataItem(data, selectors): - if len(selectors): - item = MetadataItem() - for attr in selectors: - attr = attr.encode('utf-8') - if data[attr] is not None: - if type(data[attr]) is not long and type(data[attr]) is not int: - item.__setattr__(attr, data[attr].encode('utf-8')) - if attr.startswith('has_'): - item.__setattr__(attr, str(bool(data[attr]))) - else: - item.__setattr__(attr, data[attr]) - else: - item.__setattr__(attr, None) - return item - - -class MetadataDomainItem(messages.Message): - attribute = messages.StringField(1) - domains = messages.StringField(2, repeated=True) - - -def submit_bigquery_job(bq_service, project_id, query_body, batch=False): - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid4()) - }, - 'configuration': { - 'query': { - 'query': query_body, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - - return bq_service.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=5) - - -def is_bigquery_job_finished(bq_service, project_id, job_id): - job = bq_service.jobs().get(projectId=project_id, - jobId=job_id).execute() - - return job['status']['state'] == 'DONE' - - -def get_bq_job_results(bq_service, job_reference): - result = [] - page_token = None - - while True: - page = bq_service.jobs().getQueryResults( - pageToken=page_token, - **job_reference).execute(num_retries=2) - - if int(page['totalRows']) == 0: - break - - rows = page['rows'] - result.extend(rows) - - page_token = page.get('pageToken') - if not page_token: - break - - return result - - -def generateSQLQuery(request): - db = sql_connection() - query_dict = {} - select = '*' - sample_ids = () - selector_list = [] - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s AND project_id IS NULL;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - cursor.close() - - except (TypeError, IndexError) as e: - print e - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - if request.__getattribute__('selectors') is not None and len(request.__getattribute__('selectors')): - select = ','.join(request.selectors) - selector_list = select.split(',') # request.selectors - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None: - if key.startswith('has_'): - query_dict[key] = 1 if request.__getattribute__(key) == 'True' else 0 - else: - query_dict[key] = request.__getattribute__(key).replace('_', ' ') # values coming in with _ replaced with spaces - - query_str = 'SELECT %s FROM metadata_samples' % select - value_tuple = () - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - if request.__getattribute__('limit') is not None: - query_str += ' LIMIT %s' % request.__getattribute__('limit') - - query_str += ';' - db.close() - - return query_str, value_tuple, selector_list - - -class MetadataDomainList(messages.Message): - items = messages.MessageField(MetadataDomainItem, 1, repeated=True) - - -def normalize_metadata_ages(ages): - result = [] - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age in ages: - if type(age) != dict: - - if age.value != 'None': - int_age = float(age.value) - if int_age < 40: - new_age_list['10 to 39'] += int(age.count) - elif int_age < 50: - new_age_list['40 to 49'] += int(age.count) - elif int_age < 60: - new_age_list['50 to 59'] += int(age.count) - elif int_age < 70: - new_age_list['60 to 69'] += int(age.count) - elif int_age < 80: - new_age_list['70 to 79'] += int(age.count) - else: - new_age_list['Over 80'] += int(age.count) - else: - new_age_list['None'] += int(age.count) - - for key, value in new_age_list.items(): - result.append({'count': value, 'value': key}) - return result - - -class PlatformCount(messages.Message): - platform = messages.StringField(1) - count = messages.IntegerField(2) - - -class FileDetails(messages.Message): - sample = messages.StringField(1) - filename = messages.StringField(2) - pipeline = messages.StringField(3) - platform = messages.StringField(4) - datalevel = messages.StringField(5) - datatype = messages.StringField(6) - gg_readgroupset_id = messages.StringField(7) - cloudstorage_location = messages.StringField(8) - access = messages.StringField(9) - - -class SampleFiles(messages.Message): - total_file_count = messages.IntegerField(1) - page = messages.IntegerField(2) - platform_count_list = messages.MessageField(PlatformCount, 3, repeated=True) - file_list = messages.MessageField(FileDetails, 4, repeated=True) - - -class SampleFileCount(messages.Message): - sample_id = messages.StringField(1) - count = messages.IntegerField(2) - - -class CohortFileCountSampleList(messages.Message): - sample_list = messages.MessageField(SampleFileCount, 1, repeated=True) - - -class IncomingPlatformSelection(messages.Message): - ABSOLiD_DNASeq = messages.StringField(1) - Genome_Wide_SNP_6 = messages.StringField(2) - HumanMethylation27 = messages.StringField(3) - HumanMethylation450 = messages.StringField(4) - IlluminaGA_DNASeq = messages.StringField(5) - IlluminaGA_DNASeq_automated = messages.StringField(6) - IlluminaGA_DNASeq_Cont_automated = messages.StringField(7) - IlluminaGA_DNASeq_curated = messages.StringField(8) - IlluminaGA_miRNASeq = messages.StringField(9) - IlluminaGA_None = messages.StringField(10) - IlluminaGA_RNASeq = messages.StringField(11) - IlluminaGA_RNASeqV2 = messages.StringField(12) - IlluminaHiSeq_DNASeq = messages.StringField(13) - IlluminaHiSeq_DNASeq_automated = messages.StringField(14) - IlluminaHiSeq_DNASeq_Cont_automated = messages.StringField(15) - IlluminaHiSeq_miRNASeq = messages.StringField(16) - IlluminaHiSeq_None = messages.StringField(17) - IlluminaHiSeq_RNASeq = messages.StringField(18) - IlluminaHiSeq_RNASeqV2 = messages.StringField(19) - IlluminaHiSeq_TotalRNASeqV2 = messages.StringField(20) - IlluminaMiSeq_DNASeq = messages.StringField(21) - IlluminaMiSeq_None = messages.StringField(22) - LifeIonTorrentPGM_None = messages.StringField(23) - LifeIonTorrentProton_None = messages.StringField(24) - MDA_RPPA_Core = messages.StringField(25) - microsat_i = messages.StringField(26) - Mixed_DNASeq_Cont = messages.StringField(27) - Mixed_DNASeq_Cont_curated = messages.StringField(28) - Mixed_DNASeq_curated = messages.StringField(29) - RocheGSFLX_DNASeq = messages.StringField(30) - - -class IncomingMetadataCount(messages.Message): - filters = messages.StringField(1) - cohort_id = messages.IntegerField(2) - token = messages.StringField(3) - - -def get_current_user(request): - """ - Returns a Django_User object from either endpoints.get_current_user() or an access token. - Anytime this is used in an endpoint, request_finished.send(self) must be used - """ - user_email = None - access_token = request.__getattribute__('token') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - django.setup() - try: - if access_token is not None and user_email is None: - social_account_id = SocialToken.objects.get(token=access_token).account_id - user_id = SocialAccount.objects.get(id=social_account_id).user_id - return Django_User.objects.get(id=user_id) - elif user_email is not None: - return Django_User.objects.get(email=user_email) - else: - return None - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - return None - - -# TODO: needs to be refactored to use other samples tables -def get_participant_list(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - return results - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_list') - -# TODO: needs to be refactored to use other samples tables -def get_participant_count(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT COUNT(DISTINCT case_barcode) AS ParticipantCount FROM metadata_samples WHERE sample_barcode IN (' - first = True - samples = () - for barcode in sample_ids: - samples += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - count = 0; - cursor.execute(participant_query, samples) - for row in cursor.fetchall(): - count = row['ParticipantCount'] - - return count - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_count') - - -def count_metadata(user, cohort_id=None, sample_ids=None, filters=None): - counts_and_total = {} - sample_tables = {} - valid_attrs = {} - project_ids = () - table_key_map = {} - - if filters is None: - filters = {} - - if sample_ids is None: - sample_ids = {} - - for key in sample_ids: - samples_by_project = sample_ids[key] - sample_ids[key] = { - 'sample_barcode': build_where_clause({'sample_barcode': samples_by_project}), - } - - db = sql_connection() - django.setup() - - try: - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'sample_ids': None} - if sample_ids and None in sample_ids: - sample_tables['metadata_samples']['sample_ids'] = sample_ids[None] - - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - if row['attribute'] in METADATA_SHORTLIST: - valid_attrs[row['spec'] + ':' + row['attribute']] = { - 'name': row['attribute'], - 'tables': ('metadata_samples',), - 'sample_ids': None - } - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = {'sample_ids': None} - if sample_ids and project.id in sample_ids: - sample_tables[tables.metadata_samples_table]['sample_ids'] = sample_ids[project.id] - - features = User_Feature_Definitions.objects.filter(project__in=project_ids) - for feature in features: - if ' ' in feature.feature_name: - # It is not a column name and comes from molecular data, ignore it - continue - - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name, 'tables': (), 'sample_ids': None} - - for tables in User_Data_Tables.objects.filter(project_id=feature.project_id): - valid_attrs[key]['tables'] += (tables.metadata_samples_table,) - - if tables.metadata_samples_table not in table_key_map: - table_key_map[tables.metadata_samples_table] = {} - table_key_map[tables.metadata_samples_table][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - - if sample_ids and feature.project_id in sample_ids: - valid_attrs[key]['sample_ids'] = sample_ids[feature.project_id] - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - # For filters with no tables at this point, assume its the TCGA metadata_samples table - for key, obj in filters.items(): - if not obj['tables']: - filters[key]['tables'].append('metadata_samples') - - resulting_samples = {} - - # Loop through the features - for key, feature in valid_attrs.items(): - # Get a count for each feature - table_values = {} - feature['total'] = 0 - for table in feature['tables']: - # Check if the filters make this table 0 anyway - # We do this to avoid SQL errors for columns that don't exist - should_be_queried = True - if cohort_id and sample_tables[table]['sample_ids'] is None: - should_be_queried = False - - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - # Build Filter Where Clause - filter_copy = copy.deepcopy(filters) - key_map = table_key_map[table] if table in table_key_map else False - where_clause = build_where_clause(filter_copy, alt_key_map=key_map) - col_name = feature['name'] - if key_map and key in key_map: - col_name = key_map[key] - - cursor = db.cursor() - if should_be_queried: - # Query the table for counts and values - query = ('SELECT DISTINCT %s, COUNT(1) as count FROM %s') % (col_name, table) - sample_query = ('SELECT DISTINCT %s AS sample_id FROM %s') % ('sample_barcode', table) - query_clause = '' - if where_clause['query_str']: - query_clause = ' WHERE ' + where_clause['query_str'] - if sample_tables[table]['sample_ids']: - barcode_key = 'sample_barcode' - addt_cond = sample_tables[table]['sample_ids'][barcode_key]['query_str'] - if addt_cond and where_clause['query_str']: - query_clause += ' AND ' + addt_cond - elif addt_cond: - query_clause = ' WHERE ' + addt_cond - where_clause['value_tuple'] += sample_tables[table]['sample_ids'][barcode_key]['value_tuple'] - query += query_clause + (' GROUP BY %s ' % col_name) - sample_query += query_clause - - logger.debug("In api count_metadata, executing query "+query) - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - table_values[row[0]] += int(row[1]) - feature['total'] += int(row[1]) - - cursor.execute(sample_query, where_clause['value_tuple']) - for row in cursor.fetchall(): - resulting_samples[row[0]] = 1 - else: - # Just get the values so we can have them be 0 - cursor.execute(('SELECT DISTINCT %s FROM %s') % (col_name, table)) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - - cursor.close() - - feature['values'] = table_values - - sample_set = () - for sample in resulting_samples: - sample_set += (sample,) - - counts_and_total['participants'] = get_participant_count(sample_set) if sample_set.__len__() > 0 else 0 - counts_and_total['counts'] = [] - counts_and_total['total'] = 0 - for key, feature in valid_attrs.items(): - value_list = [] - - # Special case for age ranges and BMI - if key == 'CLIN:age_at_initial_pathologic_diagnosis': - feature['values'] = normalize_ages(feature['values']) - elif key == 'CLIN:BMI': - feature['values'] = normalize_BMI(feature['values']) - - - for value, count in feature['values'].items(): - if feature['name'].startswith('has_'): - value = 'True' if value else 'False' - - value_list.append(MetaValueListCount(value=str(value), count=count)) - - counts_and_total['counts'].append( - MetadataAttributeValues(name=feature['name'], values=value_list, id=key, total=feature['total'])) - if feature['total'] > counts_and_total['total']: - counts_and_total['total'] = feature['total'] - - db.close() - - return counts_and_total - - except (Exception) as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in count_metadata.') - - -def query_samples_and_studies(parameter, bucket_by=None): - - query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - if bucket_by is not None and bucket_by not in query_str: - logging.error("Cannot group barcodes: column '" + bucket_by + - "' not found in query string '"+query_str+"'. Barcodes will not be grouped.") - bucket_by = None - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, (parameter,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs for cohort '" + parameter + "': " + (stop-start).__str__()) - - samples = () - - if bucket_by is not None: - samples = {} - - for row in cursor.fetchall(): - if bucket_by is not None: - if row[bucket_by] not in samples: - samples[row[bucket_by]] = [] - samples[row[bucket_by]].append(row['sample_barcode']) - else: - samples += ({"sample_id": row['sample_barcode'], "project_id": row['project_id']},) - cursor.close() - db.close() - - return samples - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - -Meta_Endpoints = endpoints.api(name='meta_api', version='v1', description='Metadata endpoints used by the web application.', - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - - -@Meta_Endpoints.api_class(resource_name='meta_endpoints') -class Meta_Endpoints_API(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(IncomingMetadataItem, - limit=messages.IntegerField(2), - cohort_id=messages.IntegerField(3)) - @endpoints.method(GET_RESOURCE, MetadataPlatformItemList, - path='metadata_platform_list', http_method='GET', - name='meta.metadata_platform_list') - def metadata_platform_list(self, request): - """ Used by the web application.""" - query_dict = {} - sample_ids = None - - db = sql_connection() - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - - except (TypeError, IndexError) as e: - print e - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None and len(request.__getattribute__(key)) > 0: - values = [] - for val in request.__getattribute__(key): - values.append(val) - if key.startswith('has_'): - query_dict[key] = '1' if values[0] == 'True' else '0' - else: - query_dict[key] = ','.join(values).replace('_', ' ') - # combinations: has_UNC_HiSeq_RNASeq and has_UNC_GA_RNASeq 20 rows - # has_UNC_HiSeq_RNASeq and has_BCGSC_GA_RNASeq 209 rows - # has_BCGSC_HiSeq_RNASeq and has_UNC_HiSeq_RNASeq 919 rows - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - if request.__getattribute__('limit') is not None: - query_str += ' LIMIT %s' % request.__getattribute__('limit') - - query_str += ';' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataPlatformItemList(items=data) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - - - # UNUSED ENDPOINT - GET_RESOURCE = endpoints.ResourceContainer(IncomingMetadataItem, - limit=messages.IntegerField(2), - cohort_id=messages.IntegerField(3), - selectors=messages.StringField(4, repeated=True)) - @endpoints.method(GET_RESOURCE, MetadataItemList, - path='metadata_list', http_method='GET', - name='meta.metadata_list') - def metadata_list(self, request): - """ Used by the web application.""" - select = '*' - query_dict = {} - selector_list = [] # todo: determine use or delete this - sample_ids = None - - db = sql_connection() - query_str, value_tuple, selector_list = generateSQLQuery(request) - if debug: print >> sys.stderr,query_str - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - - for row in cursor.fetchall(): - if selector_list: - item = createDataItem(row, selector_list) - else: - item = MetadataItem( - age_at_initial_pathologic_diagnosis=None if "age_at_initial_pathologic_diagnosis" not in row or row["age_at_initial_pathologic_diagnosis"] is None else int(row["age_at_initial_pathologic_diagnosis"]), - anatomic_neoplasm_subdivision=str(row["anatomic_neoplasm_subdivision"]), - avg_percent_lymphocyte_infiltration=None if "avg_percent_lymphocyte_infiltration" not in row or row["avg_percent_lymphocyte_infiltration"] is None else float(row["avg_percent_lymphocyte_infiltration"]), - avg_percent_monocyte_infiltration=None if "avg_percent_monocyte_infiltration" not in row or row["avg_percent_monocyte_infiltration"] is None else float(row["avg_percent_monocyte_infiltration"]), - avg_percent_necrosis=None if "avg_percent_necrosis" not in row or row["avg_percent_necrosis"] is None else float(row["avg_percent_necrosis"]), - avg_percent_neutrophil_infiltration=None if "avg_percent_neutrophil_infiltration" not in row or row["avg_percent_neutrophil_infiltration"] is None else float(row["avg_percent_neutrophil_infiltration"]), - avg_percent_normal_cells=None if "avg_percent_normal_cells" not in row or row["avg_percent_normal_cells"] is None else float(row["avg_percent_normal_cells"]), - avg_percent_stromal_cells=None if "avg_percent_stromal_cells" not in row or row["avg_percent_stromal_cells"] is None else float(row["avg_percent_stromal_cells"]), - avg_percent_tumor_cells=None if "avg_percent_tumor_cells" not in row or row["avg_percent_tumor_cells"] is None else float(row["avg_percent_tumor_cells"]), - avg_percent_tumor_nuclei=None if "avg_percent_tumor_nuclei" not in row or row["avg_percent_tumor_nuclei"] is None else float(row["avg_percent_tumor_nuclei"]), - batch_number=None if "batch_number" not in row or row["batch_number"] is None else int(row["batch_number"]), - bcr=str(row["bcr"]), - clinical_M=str(row["clinical_M"]), - clinical_N=str(row["clinical_N"]), - clinical_stage=str(row["clinical_stage"]), - clinical_T=str(row["clinical_T"]), - colorectal_cancer=str(row["colorectal_cancer"]), - country=str(row["country"]), - days_to_birth=None if "days_to_birth" not in row or row['days_to_birth'] is None else int(row["days_to_birth"]), - days_to_collection=None if "days_to_collection" not in row or row['days_to_collection'] is None else int(row["days_to_collection"]), - days_to_death=None if "days_to_death" not in row or row['days_to_death'] is None else int(row["days_to_death"]), - days_to_initial_pathologic_diagnosis=None if "days_to_initial_pathologic_diagnosis" not in row or row['days_to_initial_pathologic_diagnosis'] is None else int(row["days_to_initial_pathologic_diagnosis"]), - days_to_last_followup=None if "days_to_last_followup" not in row or row['days_to_last_followup'] is None else int(row["days_to_last_followup"]), - days_to_submitted_specimen_dx=None if "days_to_submitted_specimen_dx" not in row or row['days_to_submitted_specimen_dx'] is None else int(row["days_to_submitted_specimen_dx"]), - Study=str(row["Study"]), - ethnicity=str(row["ethnicity"]), - frozen_specimen_anatomic_site=str(row["frozen_specimen_anatomic_site"]), - gender=str(row["gender"]), - height=None if "height" not in row or row['height'] is None else int(row["height"]), - histological_type=str(row["histological_type"]), - history_of_colon_polyps=str(row["history_of_colon_polyps"]), - history_of_neoadjuvant_treatment=str(row["history_of_neoadjuvant_treatment"]), - history_of_prior_malignancy=str(row["history_of_prior_malignancy"]), - hpv_calls=str(row["hpv_calls"]), - hpv_status=str(row["hpv_status"]), - icd_10=str(row["icd_10"]), - icd_o_3_histology=str(row["icd_o_3_histology"]), - icd_o_3_site=str(row["icd_o_3_site"]), - lymphatic_invasion=str(row["lymphatic_invasion"]), - lymphnodes_examined=str(row["lymphnodes_examined"]), - lymphovascular_invasion_present=str(row["lymphovascular_invasion_present"]), - max_percent_lymphocyte_infiltration=None if "max_percent_lymphocyte_infiltration" not in row or row["max_percent_lymphocyte_infiltration"] is None else int(row["max_percent_lymphocyte_infiltration"]), # 46) - max_percent_monocyte_infiltration=None if "max_percent_monocyte_infiltration" not in row or row["max_percent_monocyte_infiltration"] is None else int(row["max_percent_monocyte_infiltration"]), # 47) - max_percent_necrosis=None if "max_percent_necrosis" not in row or row["max_percent_necrosis"] is None else int(row["max_percent_necrosis"]), # 48) - max_percent_neutrophil_infiltration=None if "max_percent_neutrophil_infiltration" not in row or row["max_percent_neutrophil_infiltration"] is None else int(row["max_percent_neutrophil_infiltration"]), # 49) - max_percent_normal_cells=None if "max_percent_normal_cells" not in row or row["max_percent_normal_cells"] is None else int(row["max_percent_normal_cells"]), # 50) - max_percent_stromal_cells=None if "max_percent_stromal_cells" not in row or row["max_percent_stromal_cells"] is None else int(row["max_percent_stromal_cells"]), # 51) - max_percent_tumor_cells=None if "max_percent_tumor_cells" not in row or row["max_percent_tumor_cells"] is None else int(row["max_percent_tumor_cells"]), # 52) - max_percent_tumor_nuclei=None if "max_percent_tumor_nuclei" not in row or row["max_percent_tumor_nuclei"] is None else int(row["max_percent_tumor_nuclei"]), # 53) - menopause_status=str(row["menopause_status"]), - min_percent_lymphocyte_infiltration=None if "min_percent_lymphocyte_infiltration" not in row or row["min_percent_lymphocyte_infiltration"] is None else int(row["min_percent_lymphocyte_infiltration"]), # 55) - min_percent_monocyte_infiltration=None if "min_percent_monocyte_infiltration" not in row or row["min_percent_monocyte_infiltration"] is None else int(row["min_percent_monocyte_infiltration"]), # 56) - min_percent_necrosis=None if "min_percent_necrosis" not in row or row["min_percent_necrosis"] is None else int(row["min_percent_necrosis"]), # 57) - min_percent_neutrophil_infiltration=None if "min_percent_neutrophil_infiltration" not in row or row["min_percent_neutrophil_infiltration"] is None else int(row["min_percent_neutrophil_infiltration"]), # 58) - min_percent_normal_cells=None if "min_percent_normal_cells" not in row or row["min_percent_normal_cells"] is None else int(row["min_percent_normal_cells"]), # 59) - min_percent_stromal_cells=None if "min_percent_stromal_cells" not in row or row["min_percent_stromal_cells"] is None else int(row["min_percent_stromal_cells"]), # 60) - min_percent_tumor_cells=None if "min_percent_tumor_cells" not in row or row["min_percent_tumor_cells"] is None else int(row["min_percent_tumor_cells"]), # 61) - min_percent_tumor_nuclei=None if "min_percent_tumor_nuclei" not in row or row["min_percent_tumor_nuclei"] is None else int(row["min_percent_tumor_nuclei"]), # 62) - mononucleotide_and_dinucleotide_marker_panel_analysis_status=str(row["mononucleotide_and_dinucleotide_marker_panel_analysis_status"]), - mononucleotide_marker_panel_analysis_status=str(row["mononucleotide_marker_panel_analysis_status"]), - neoplasm_histologic_grade=str(row["neoplasm_histologic_grade"]), - new_tumor_event_after_initial_treatment=str(row["new_tumor_event_after_initial_treatment"]), - number_of_lymphnodes_examined=None if "number_of_lymphnodes_examined" not in row or row['number_of_lymphnodes_examined'] is None else int(row["number_of_lymphnodes_examined"]), - number_of_lymphnodes_positive_by_he=None if "number_of_lymphnodes_positive_by_he" not in row or row['number_of_lymphnodes_positive_by_he'] is None else int(row["number_of_lymphnodes_positive_by_he"]), - case_barcode=str(row["case_barcode"]), - pathologic_M=str(row["pathologic_M"]), - pathologic_N=str(row["pathologic_N"]), - pathologic_stage=str(row["pathologic_stage"]), - pathologic_T=str(row["pathologic_T"]), - person_neoplasm_cancer_status=str(row["person_neoplasm_cancer_status"]), - pregnancies=str(row["pregnancies"]), - primary_neoplasm_melanoma_dx=str(row["primary_neoplasm_melanoma_dx"]), - primary_therapy_outcome_success=str(row["primary_therapy_outcome_success"]), - prior_dx=str(row["prior_dx"]), - Project=str(row["Project"]), - psa_value=None if "psa_value" not in row or row["psa_value"] is None else float(row["psa_value"]), - race=str(row["race"]), - residual_tumor=str(row["residual_tumor"]), - sample_barcode=str(row["sample_barcode"]), - tobacco_smoking_history=str(row["tobacco_smoking_history"]), - tumor_tissue_site=str(row["tumor_tissue_site"]), - tumor_type=str(row["tumor_type"]), - weiss_venous_invasion=str(row["weiss_venous_invasion"]), - vital_status=str(row["vital_status"]), - weight=None if "weight" not in row or row["weight"] is None else int(float(row["weight"])), - year_of_initial_pathologic_diagnosis=str(row["year_of_initial_pathologic_diagnosis"]), - SampleTypeCode=str(row["SampleTypeCode"]), - has_Illumina_DNASeq=str(bool(row["has_Illumina_DNASeq"])), - has_BCGSC_HiSeq_RNASeq=str(bool(row["has_BCGSC_HiSeq_RNASeq"])), - has_UNC_HiSeq_RNASeq=str(bool(row["has_UNC_HiSeq_RNASeq"])), - has_BCGSC_GA_RNASeq=str(bool(row["has_BCGSC_GA_RNASeq"])), - has_UNC_GA_RNASeq=str(bool(row["has_UNC_GA_RNASeq"])), - has_HiSeq_miRnaSeq=str(bool(row["has_HiSeq_miRnaSeq"])), - has_GA_miRNASeq=str(bool(row["has_GA_miRNASeq"])), - has_RPPA=str(bool(row["has_RPPA"])), - has_SNP6=str(bool(row["has_SNP6"])), - has_27k=str(bool(row["has_27k"])), - has_450k=str(bool(row["has_450k"])) - ) - - data.append(item) - - cursor.close() - db.close() - - return MetadataItemList(items=data, total=len(data)) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - # UNUSED ENDPOINT - GET_RESOURCE = endpoints.ResourceContainer(IncomingMetadataItem, - is_landing=messages.BooleanField(2), - cohort_id=messages.IntegerField(3)) - @endpoints.method(GET_RESOURCE, MetadataItemList, - path='metadata_counts', http_method='GET', - name='meta.metadata_counts') - def metadata_counts(self, request): - """ Used by the web application.""" - query_dict = {} - sample_ids = None - is_landing = False - - if request.__getattribute__('is_landing') is not None: - is_landing = request.__getattribute__('is_landing') - - if is_landing: - try: - db = sql_connection() - cursor = db.cursor() - cursor.execute('SELECT Study, COUNT(Study) as disease_count FROM metadata_samples GROUP BY Study;') - data = [] - for row in cursor.fetchall(): - value_list_count = MetaValueListCount( - value=row[0], - count=int(row[1]) - ) - data.append(value_list_count) - - attr_values_list = MetaAttrValuesList(Study=data) - - cursor.close() - db.close() - - return MetadataItemList(count=attr_values_list) - - except (IndexError, TypeError) as e: - - raise endpoints.NotFoundException('Error in getting landing data.') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - cursor.close() - db.close() - - except (TypeError, IndexError) as e: - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None: - if key.startswith('has_'): - query_dict[key] = '1' if request.__getattribute__(key) == 'True' else '0' - else: - query_dict[key] = request.__getattribute__(key).replace('_', ' ') - - value_list = {} - total = 0 - for key in METADATA_SHORTLIST: # fix metadata_shortlist - # counts[key] = {} - domain_query_str = 'SELECT %s FROM metadata_samples GROUP BY %s' % (key, key) - value_count_query_str = 'SELECT %s, COUNT(*) FROM metadata_samples' % key - value_count_tuple = () - if len(query_dict) > 0: - - where_clause = applyFilter(key, query_dict) - - if where_clause is not None: - value_count_query_str += ' WHERE' + where_clause['query_str'] - value_count_tuple += where_clause['value_tuple'] - - if sample_ids: - if value_count_query_str.rfind('WHERE') >= 0: - value_count_query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - value_count_query_str += ' where sample_barcode in %s' % (sample_ids,) - - value_count_query_str += ' GROUP BY %s;' % key - - try: - db = sql_connection() - cursor = db.cursor() - cursor.execute(value_count_query_str, value_count_tuple) - - value_list[key] = [] - count = 0 - data = [] - data_domain = [] - # Real data counts - key_total = 0 - for row in cursor.fetchall(): - - # print row - - key_total += row[1] - # note: assumes no null values for data availability fields (which start with 'has_') - if key.startswith('has_'): - value_list[key].append(MetaValueListCount(value=str(bool(row[0])), count=row[1])) - data_domain.append(str(bool(row[0]))) - elif type(row[0]) == long: - value_list[key].append(MetaValueListCount(value=str(int(row[0])), count=row[1])) - data_domain.append(int(row[0])) - else: - value_list[key].append(MetaValueListCount(value=str(row[0]), count=row[1])) - data_domain.append(str(row[0])) - # Find the total number of samples - if key_total > total: - total = key_total - - # Fill in other values for each feature with 0 - cursor.execute(domain_query_str) - for row in cursor.fetchall(): - # note: assumes no null values for data availability fields (which start with 'has_') - if key.startswith('has_'): - if str(bool(row[0])) not in data_domain: - value_list[key].append(MetaValueListCount(value=str(bool(row[0])), count=0)) - data_domain.append(bool(row[0])) - elif str(row[0]) not in data_domain: - if type(row[0]) == long: - value_list[key].append(MetaValueListCount(value=str(int(row[0])), count=0)) - data_domain.append(int(row[0])) - else: - value_list[key].append(MetaValueListCount(value=str(row[0]), count=0)) - data_domain.append(str(row[0])) - - if key == 'age_at_initial_pathologic_diagnosis': - value_list['age_at_initial_pathologic_diagnosis'] = normalize_metadata_ages(value_list['age_at_initial_pathologic_diagnosis']) - cursor.close() - db.close() - except (KeyError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in getting value counts.') - - value_list_item = MetaAttrValuesList() - for key in METADATA_SHORTLIST: - value_list_item.__setattr__(key, None if key not in value_list else value_list[key]) - - return MetadataItemList(count=value_list_item, total=total) - - # UNUSED ENDPOINT - GET_RESOURCE = endpoints.ResourceContainer( - MetadataAttr) - @endpoints.method(GET_RESOURCE, MetadataAttrList, - path='metadata_attr_list', http_method='GET', - name='meta.metadata_attr_list') - def metadata_attr_list(self, request): - """ Used by the web application.""" - query_dict = {} - value_tuple = () - for key, value in MetadataAttr.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) != None: - query_dict[key] = request.__getattribute__(key) - - if len(query_dict) == 0: - query_str = 'SELECT * FROM metadata_attr' - else: - query_str = 'SELECT * FROM metadata_attr where' - - where_clause = build_where_clause(query_dict) - query_str += where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - data.append(MetadataAttr(attribute=str(row['attribute']), - code=str(row['code']), - spec=str(row['spec']), - )) - - cursor.close() - db.close() - return MetadataAttrList(items=data, count=len(data)) - - except (IndexError, TypeError): - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample %s not found.' % (request.id,)) - - # UNUSED ENDPOINT - @endpoints.method(message_types.VoidMessage, MetaDomainsList, - path='metadata_domains', http_method='GET', - name='meta.metadata_domains') - def domains_list(self, request): - """ Used by the web application.""" - db = sql_connection() - cursor = db.cursor() - items = {} - feature_list = MetaDomainsList.__dict__.keys() - meta_categorical_attributes = [ - 'gender', - 'history_of_neoadjuvant_treatment', - 'icd_o_3_histology', - 'prior_dx', - 'vital_status', - 'country', - 'Study', - 'histological_type', - 'icd_10', - 'icd_o_3_site', - 'tumor_tissue_site', - 'tumor_type', - 'person_neoplasm_cancer_status', - 'pathologic_N', - 'pathologic_T', - 'race', - 'ethnicity', - 'SampleTypeCode', - 'has_Illumina_DNASeq', - 'has_BCGSC_HiSeq_RNASeq', - 'has_UNC_HiSeq_RNASeq', - 'has_BCGSC_GA_RNASeq', - 'has_HiSeq_miRnaSeq', - 'has_GA_miRNASeq', - 'has_RPPA', - 'has_SNP6', - 'has_27k', - 'has_450k' - ] - - try: - for feature in feature_list: - if '__' not in feature: - query_str = 'SELECT DISTINCT %s from metadata_samples;' % feature - cursor.execute(query_str) - item_list = [] - for item in cursor.fetchall(): - if feature.startswith('has_'): - item_list.append(str(bool(int(item[0])))) - else: - item_list.append(str(item[0])) - items[feature] = item_list - items['age_at_initial_pathologic_diagnosis'] = ['10 to 39', '40 to 49', '50 to 59', '60 to 69', '70 to 79', 'Over 80', 'None'] - cursor.close() - db.close() - return MetaDomainsList( - gender = items['gender'], - history_of_neoadjuvant_treatment = items['history_of_neoadjuvant_treatment'], - icd_o_3_histology = items['icd_o_3_histology'], - prior_dx = items['prior_dx'], - vital_status = items['vital_status'], - country = items['country'], - Study = items['Study'], - histological_type = items['histological_type'], - icd_10 = items['icd_10'], - icd_o_3_site = items['icd_o_3_site'], - tumor_tissue_site = items['tumor_tissue_site'], - tumor_type = items['tumor_type'], - person_neoplasm_cancer_status = items['person_neoplasm_cancer_status'], - pathologic_N = items['pathologic_N'], - pathologic_T = items['pathologic_T'], - race = items['race'], - ethnicity = items['ethnicity'], - SampleTypeCode = items['SampleTypeCode'], - has_Illumina_DNASeq = items['has_Illumina_DNASeq'], - has_BCGSC_HiSeq_RNASeq = items['has_BCGSC_HiSeq_RNASeq'], - has_UNC_HiSeq_RNASeq = items['has_UNC_HiSeq_RNASeq'], - has_BCGSC_GA_RNASeq = items['has_BCGSC_GA_RNASeq'], - has_HiSeq_miRnaSeq = items['has_HiSeq_miRnaSeq'], - has_GA_miRNASeq = items['has_GA_miRNASeq'], - has_RPPA = items['has_RPPA'], - has_SNP6 = items['has_SNP6'], - has_27k = items['has_27k'], - has_450k = items['has_450k'] - ) - - except (IndexError, TypeError): - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in meta_domains') - - GET_RESOURCE = endpoints.ResourceContainer(IncomingPlatformSelection, - cohort_id=messages.IntegerField(1, required=True), - page=messages.IntegerField(2), - limit=messages.IntegerField(3), - token=messages.StringField(4), - platform_count_only=messages.StringField(5), - offset=messages.IntegerField(6) - ) - - @endpoints.method(GET_RESOURCE, SampleFiles, - path='cohort_files', http_method='GET', - name='meta.cohort_files') - def cohort_files(self, request): - """ Used by the web application.""" - limit = 20 - page = 1 - offset = 0 - cohort_id = request.cohort_id - - platform_count_only = request.__getattribute__('platform_count_only') - is_dbGaP_authorized = False - user_email = None - user_id = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.__getattribute__('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email or user_id: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error(e) - logger.error(traceback.format_exc()) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - logger.error(traceback.format_exc()) - request_finished.send(self) - raise endpoints.UnauthorizedException("%s does not have permission to view cohort %d." % (user_email, cohort_id)) - - try: - nih_user = NIH_User.objects.get(user_id=user_id) - is_dbGaP_authorized = nih_user.dbGaP_authorized and nih_user.active - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error("%s does not have an entry in NIH_User: %s" % (user_email, str(e))) - logger.error(traceback.format_exc()) - else: - logger.error("[ERROR] Authentication required for cohort_files endpoint.") - raise endpoints.UnauthorizedException("No user email found.") - - if request.__getattribute__('page') is not None: - page = request.page - offset = (page - 1) * 20 - elif request.__getattribute__('offset') is not None: - offset = request.offset - - if request.__getattribute__('limit') is not None: - limit = request.limit - - sample_query = 'select sample_barcode from cohorts_samples where cohort_id=%s;' - sample_list = () - db = None - cursor = None - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query, (cohort_id,)) - in_clause = '(' - first = True - for row in cursor.fetchall(): - sample_list += (row['sample_barcode'],) - if first: - in_clause += '%s' - first = False - else: - in_clause += ',%s' - in_clause += ')' - except (IndexError, TypeError): - logger.error("Error obtaining list of samples in cohort file list") - logger.error(traceback.format_exc()) - raise endpoints.ServiceException('Error obtaining list of samples in cohort file list') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - platform_count_query = 'select Platform, count(Platform) as platform_count from metadata_data where sample_barcode in {0} and DatafileUploaded="true" group by Platform order by sample_barcode;'.format(in_clause) - query = 'select sample_barcode, DatafileName, DatafileNameKey, SecurityProtocol, Pipeline, Platform, DataLevel, Datatype, GG_readgroupset_id, Repository, SecurityProtocol from metadata_data where sample_barcode in {0} and DatafileUploaded="true" '.format(in_clause) - - # Check for incoming platform selectors - platform_selector_list = [] - for key, value in IncomingPlatformSelection.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None and request.__getattribute__(key) == 'True': - platform_selector_list.append(key) - - if len(platform_selector_list): - query += ' and Platform in ("' + '","'.join(platform_selector_list) + '")' - - query_tuple = sample_list - - if limit > 0: - query += ' limit %s' - query_tuple += (limit,) - # Offset is only valid when there is a limit - if offset > 0: - query += ' offset %s' - query_tuple += (offset,) - - query += ';' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(platform_count_query, sample_list) - platform_count_list = [] - count = 0 - if cursor.rowcount > 0: - for row in cursor.fetchall(): - if len(platform_selector_list): - if row['Platform'] in platform_selector_list: - count += int(row['platform_count']) - else: - count += int(row['platform_count']) - platform_count_list.append(PlatformCount(platform=row['Platform'], count=row['platform_count'])) - else: - platform_count_list.append(PlatformCount(platform='None', count=0)) - - file_list = [] - if not platform_count_only: - cursor.execute(query, query_tuple) - if cursor.rowcount > 0: - for item in cursor.fetchall(): - # If there's a datafilenamekey - if 'DatafileNameKey' in item and item['DatafileNameKey'] != '': - # Find protected bucket it should belong to - bucket_name = '' - if item['Repository'] and item['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif item['Repository'] and item['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: - bucket_name = settings.OPEN_DATA_BUCKET - - item['DatafileNameKey'] = "gs://{}{}".format(bucket_name, item['DatafileNameKey']) - - file_list.append(FileDetails(sample=item['sample_barcode'], cloudstorage_location=item['DatafileNameKey'], access=(item['SecurityProtocol'] or 'N/A'), filename=item['DatafileName'], pipeline=item['Pipeline'], platform=item['Platform'], datalevel=item['DataLevel'], datatype=(item['Datatype'] or " "), gg_readgroupset_id=item['GG_readgroupset_id'])) - else: - file_list.append(FileDetails(sample='None', filename='', pipeline='', platform='', datalevel='')) - return SampleFiles(total_file_count=count, page=page, platform_count_list=platform_count_list, file_list=file_list) - - except Exception as e: - logger.error("Error obtaining platform counts") - logger.error(traceback.format_exc()) - raise endpoints.ServiceException('Error getting counts') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - GET_RESOURCE = endpoints.ResourceContainer(sample_id=messages.StringField(1, required=True)) - @endpoints.method(GET_RESOURCE, SampleFiles, - path='sample_files', http_method='GET', - name='meta.sample_files') - def sample_files(self, request): - ''' - Takes a sample_barcode and returns a list of filenames and their associated cloudstorage locations, pipelines, and platforms. - Also returns counts for the number of files derived from each platform. - Checks user for dbGaP authorization and returns "Restricted" if a filename is controlled access and the user is not dbGaP authorized - :param sample_id: sample_barcode - :return SampleFiles: a list of filenames and their associated cloudstorage locations, pipelines, and platforms - as well as counts for the number of files for each platform. - ''' - - global cloudstorage_location - sample_id = request.sample_id - dbGaP_authorized = False - - query = "select sample_barcode, " \ - "DatafileName, " \ - "Pipeline, " \ - "Platform, " \ - "IF(SecurityProtocol LIKE '%%open%%', DatafileNameKey, 'Restricted') as DatafileNameKey, " \ - "SecurityProtocol " \ - "from metadata_data " \ - "where sample_barcode=%s;" - - if endpoints.get_current_user(): - user_email = endpoints.get_current_user().email() - try: - user_id = Django_User.objects.get(email=user_email) - nih_user = NIH_User.objects.get(user_id=user_id) - dbGaP_authorized = nih_user.dbGaP_authorized and nih_user.active - if dbGaP_authorized: - query = "select sample_barcode, " \ - "DatafileName, " \ - "Pipeline, " \ - "Platform, " \ - "DatafileNameKey, " \ - "SecurityProtocol, " \ - "Repository " \ - "from metadata_data " \ - "where sample_barcode=%s;" - except (ObjectDoesNotExist, MultipleObjectsReturned) as e: - if type(e) is MultipleObjectsReturned: - logger.error("Meta.sample_files endpoint with user {} gave error: {}".format(user_email, str(e))) - - platform_query = "select Platform, " \ - "count(Platform) as platform_count " \ - "from metadata_data " \ - "where sample_barcode=%s " \ - "group by Platform;" - - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - try: - cursor.execute(query, (sample_id,)) - file_list = [] - platform_list = [] - for item in cursor.fetchall(): - if len(item.get('DatafileNameKey', '')) == 0: - cloudstorage_location = 'File location not found.' - elif 'open' in item['SecurityProtocol']: - cloudstorage_location = 'gs://{}{}'.format(OPEN_DATA_BUCKET, item['DatafileNameKey']) - elif dbGaP_authorized: - # hard-coding mock bucket names for now --testing purposes only - if item['Repository'].lower() == 'dcc': - cloudstorage_location = 'gs://{}{}'.format( - 'gs://62f2c827-mock-mock-mock-1cde698a4f77', item['DatafileNameKey']) - elif item['Repository'].lower() == 'cghub': - cloudstorage_location = 'gs://{}{}'.format( - 'gs://360ee3ad-mock-mock-mock-52f9a5e7f99a', item['DatafileNameKey']) - else: - cloudstorage_location = item.get('DatafileNameKey') # this will return "Restricted" - - file_list.append( - FileDetails( - filename=item['DatafileName'], - pipeline=item['Pipeline'], - platform=item['Platform'], - cloudstorage_location=cloudstorage_location - ) - ) - cursor.execute(platform_query, (sample_id,)) - for item in cursor.fetchall(): - platform_list.append(PlatformCount(platform=item['Platform'], count=item['platform_count'])) - cursor.close() - db.close() - return SampleFiles(total_file_count=len(file_list), page=1, platform_count_list=platform_list, file_list=file_list) - except Exception as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error getting file details: {}'.format(str(e))) - -""" -Metadata Endpoints v2 - -Includes User Uploaded Data -""" -Meta_Endpoints_v2 = endpoints.api(name='meta_api', version='v2', - description='Retrieve metadata information relating to projects, cohorts, and other data', - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - -@Meta_Endpoints_v2.api_class(resource_name='meta_endpoints') -class Meta_Endpoints_API_v2(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer( - MetadataAttr, - token=messages.StringField(4), - ) - @endpoints.method(GET_RESOURCE, MetadataAttrList, - path='attributes', http_method='GET', - name='meta.attr_list') - def metadata_attr_list(self, request): - - cursor = None - db = None - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - query_dict = {} - value_tuple = () - for key, value in MetadataAttr.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) != None: - query_dict[key] = request.__getattribute__(key) - - if len(query_dict) == 0: - query_str = 'SELECT * FROM metadata_attr' - else: - query_str = 'SELECT * FROM metadata_attr where' - where_clause = build_where_clause(query_dict) - query_str += where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - data.append(MetadataAttr(attribute=str(row['attribute']), - code=str(row['code']), - spec=str(row['spec']), - key=str(row['spec']) + ':' + str(row['attribute']) - )) - - if user: - studies = Project.get_user_studies(user) - feature_defs = User_Feature_Definitions.objects.filter(project__in=studies) - for feature in feature_defs: - data_table = User_Data_Tables.objects.get(project=feature.project).metadata_samples_table - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - - data.append(MetadataAttr(attribute=name, - code='N' if feature.is_numeric else 'C', - spec='USER', - key=key - )) - - cursor.close() - db.close() - request_finished.send(self) - return MetadataAttrList(items=data, count=len(data)) - - except (IndexError, TypeError): - raise endpoints.InternalServerErrorException('Error retrieving attribute list') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsItem, - path='metadata_counts', http_method='POST', - name='meta.metadata_counts') - def metadata_counts(self, request): - filters = {} - sample_ids = None - cohort_id = None - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for reqFilter in tmp: - key = reqFilter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(reqFilter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - start = time.time() - counts_and_totals = count_metadata(user, cohort_id, sample_ids, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts" - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - request_finished.send(self) - return MetadataCountsItem(count=counts_and_totals['counts'], total=counts_and_totals['total']) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - @endpoints.method(POST_RESOURCE, SampleBarcodeList, - path='metadata_sample_list', http_method='POST', - name='meta.metadata_sample_list') - def metadata_list(self, request): - filters = {} - valid_attrs = {} - sample_tables = {} - table_key_map = {} - sample_ids = None - project_ids = () - cohort_id = None - cursor = None - db = None - mutation_filters = None - mutation_results = {} - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if 'MUT:' in key: - if not mutation_filters: - mutation_filters = {} - if not key in mutation_filters: - mutation_filters[key] = [] - mutation_filters[key].append(filter['value']) - else: - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - django.setup() - - # TODO enable filtering based off of this - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - if mutation_filters: - mutation_where_clause = build_where_clause(mutation_filters) - print >> sys.stdout, mutation_where_clause - cohort_join_str = '' - cohort_where_str = '' - bq_cohort_table = '' - bq_cohort_dataset = '' - cohort = '' - query_template = None - - if cohort_id is not None: - query_template = \ - ("SELECT ct.sample_barcode" - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] ct" - " JOIN (SELECT Tumor_SampleBarcode AS barcode " - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY barcode) mt" - " ON mt.barcode = ct.sample_barcode" - " WHERE ct.cohort_id = {cohort};") - bq_cohort_table = settings.BIGQUERY_COHORT_TABLE_ID - bq_cohort_dataset = settings.COHORT_DATASET_ID - cohort = cohort_id - else: - query_template = \ - ("SELECT Tumor_SampleBarcode" - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY Tumor_SampleBarcode; ") - - params = mutation_where_clause['value_tuple'][0] - - query = query_template.format(dataset_name=settings.BIGQUERY_DATASET, - project_name=settings.BIGQUERY_PROJECT_NAME, - table_name="Somatic_Mutation_calls", hugo_symbol=str(params['gene']), - var_class=params['var_class'], cohort_dataset=bq_cohort_dataset, - cohort_table=bq_cohort_table, cohort=cohort) - - bq_service = authorize_credentials_with_Google() - query_job = submit_bigquery_job(bq_service, settings.BQ_PROJECT_ID, query) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - retries = 0 - - while not job_is_done and retries < 10: - retries += 1 - sleep(1) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - results = get_bq_job_results(bq_service, query_job['jobReference']) - - # for-each result, add to list - - if results.__len__() > 0: - for barcode in results: - mutation_results[str(barcode['f'][0]['v'])] = 1 - - else: - print >> sys.stdout, "Mutation filter result was empty!" - # Put in one 'not found' entry to zero out the rest of the queries - barcodes = ['NONE_FOUND', ] - - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'features': {}, 'barcode': 'sample_barcode', 'project_id': None} - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - key = row['spec'] + ':' + row['attribute'] - valid_attrs[key] = {'name': row['attribute']} - sample_tables['metadata_samples']['features'][key] = row['attribute'] - if key in filters: - filters[key]['tables'] += ('metadata_samples',) - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - # Add all tables from each project - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = { - 'features':{}, - 'barcode':'sample_barcode', - 'project_id': project.id - } - - # Record features that should be in each sample table so we can know how and when we need to query - for feature in User_Feature_Definitions.objects.filter(project=project): - name = feature.feature_name - key = 'study:' + str(project.id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name} - - for tables in User_Data_Tables.objects.filter(project=feature.project_id): - sample_tables[tables.metadata_samples_table]['features'][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - results = [] - # Loop through the sample tables - for table, table_settings in sample_tables.items(): - # Make sure we should run the query here, or if we have filters that won't return anything, skip - should_be_queried = True - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - if not should_be_queried: - continue - - filter_copy = copy.deepcopy(filters) - where_clause = build_where_clause(filter_copy, table_settings['features']) - query = 'SELECT DISTINCT %s FROM %s' % (table_settings['barcode'], table) - if where_clause['query_str']: - query += ' WHERE ' + where_clause['query_str'] - cursor = db.cursor() - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - project_id = table_settings['project_id'] - if cohort_id and (project_id not in sample_ids or row[0] not in sample_ids[project_id]): - # This barcode was not in our cohort's list of barcodes, skip it - continue - if mutation_filters: - if row[0] in mutation_results: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id'])) - else: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id']) ) - cursor.close() - - db.close() - request_finished.send(self) - return SampleBarcodeList( items=results, count=len(results) ) - - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - ) - @endpoints.method(GET_RESOURCE, SampleBarcodeList, - path='metadata_participant_list', http_method='GET', - name='meta.metadata_participant_list') - def metadata_participant_list(self, request): - cursor = None - db = None - - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = [] - - for row in cursor.fetchall(): - sample_ids.append(row['sample_barcode']) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - cursor.close() - db.close() - return SampleBarcodeList(items=results, count=len(results)) - - except (TypeError, IndexError) as e: - raise endpoints.NotFoundException('Error in retrieving barcodes.') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataPlatformItemList, - path='metadata_platform_list', http_method='POST', - name='meta.metadata_platform_list') - def metadata_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - cursor = None - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(sample_query_str, (cohort_id,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs in metadata_platform_list for cohort '" + cohort_id + "': " + (stop - start).__str__()) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - - except (TypeError, IndexError) as e: - print e - cursor.close() - db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_platform_list for cohort '" + str(request.cohort_id) + "': " + (stop - start).__str__()) - data = [] - for row in cursor.fetchall(): - - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataPlatformItemList(items=data) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsPlatformItem, - path='metadata_counts_platform_list', http_method='POST', - name='meta.metadata_counts_platform_list') - def metadata_counts_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - samples_by_project = None - cohort_id = None - participants = 0 - user = get_current_user(request) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - samples = query_samples_and_studies(cohort_id, ) - - sample_ids = () - samples_by_project = {} - - for sample in samples: - sample_ids += (sample['sample_id'],) - if sample['project_id'] not in samples_by_project: - samples_by_project[sample['project_id']] = [] - samples_by_project[sample['project_id']].append(sample['sample_id']) - - participants = get_participant_count(sample_ids) - - start = time.time() - counts_and_total = count_metadata(user, cohort_id, samples_by_project, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts " - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - db = sql_connection() - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - data = [] - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_counts_platform_list for cohort '" + str( - request.cohort_id) + "': " + (stop - start).__str__()) - for row in cursor.fetchall(): - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataCountsPlatformItem(items=data, count=counts_and_total['counts'], - participants=counts_and_total['participants'], - total=counts_and_total['total']) - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Exception in metadata_counts_platforms_list.') diff --git a/api/pairwise.py b/api/pairwise.py deleted file mode 100755 index acb7f8ba..00000000 --- a/api/pairwise.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import json -import base64 -import logging -import urllib -import traceback - -from google.appengine.api import urlfetch -from django.conf import settings - -from bq_data_access.data_access import get_feature_vector -from bq_data_access.feature_value_types import ValueType -from bq_data_access.utils import VectorMergeSupport - -logger = logging.getLogger(__name__) - -class PairwiseInputVector(object): - def __init__(self, feature_id, value_type, data): - self.feature_id = feature_id - self.value_type = value_type - self.data = data - - -class Pairwise(object): - def __init__(self): - pass - - @classmethod - def prepare_features(self, cohort_id, features): - # Get the feature data - feature_vector_mapping = {} - vectors = [] - for feature in features: - value_type, vector = get_feature_vector(feature, cohort_id) - - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - vms = VectorMergeSupport('NA', 'sample_id', row_ids=features) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def prepare_feature_vector(self, input_vectors): - feature_vector_mapping = {} - vectors = [] - for item in input_vectors: - feature_id, value_type, vector = item.feature_id, item.value_type, item.data - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature_id] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - feature_ids = [v.feature_id for v in input_vectors] - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', row_ids=feature_ids) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def run_pairwise(self, feature_rows): - url = settings.PAIRWISE_SERVICE_URL - - data_dict = {} - row_count = 1 - for row in feature_rows: - label = "row_{count}".format(count=row_count) - data_dict[label] = row - row_count += 1 - - # Encode the data to be sent to the service - data = urllib.urlencode(data_dict) - decoded_response = None - - try: - pairwise_response = urlfetch.fetch(url=url, payload=data, method=urlfetch.POST) - response = pairwise_response.content - decoded_response = json.loads(base64.b64decode(response)) - except Exception as e: - decoded_response = None - logger.error(traceback.format_exc()) - - return decoded_response diff --git a/api/pairwise_api.py b/api/pairwise_api.py deleted file mode 100644 index d089916c..00000000 --- a/api/pairwise_api.py +++ /dev/null @@ -1,166 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -import endpoints -from protorpc import messages, remote, message_types -from django.conf import settings - -from api.api_helpers import authorize_credentials_with_Google -from api.pairwise import Pairwise - -package = 'pairwise' - - -class PairwiseJobRequest(messages.Message): - cohort_id = messages.StringField(1, required=True) - feature = messages.StringField(2, repeated=True) - - -class PairwiseResultVector(messages.Message): - feature_1 = messages.StringField(1, required=True) - feature_2 = messages.StringField(2, required=True) - comparison_type = messages.StringField(3, required=True) - correlation_coefficient = messages.StringField(4, required=True) - n = messages.IntegerField(5, required=True) - _logp = messages.FloatField(6, required=True) - n_A = messages.IntegerField(7, required=True) - p_A = messages.FloatField(8, required=True) - n_B = messages.IntegerField(9, required=True) - p_B = messages.FloatField(10, required=True) - exclusion_rules = messages.StringField(11, required=True) - - -class PairwiseFilterMessage(messages.Message): - filter_message = messages.StringField(1, required=True) - - -class PairwiseResults(messages.Message): - result_vectors = messages.MessageField(PairwiseResultVector, 1, repeated=True) - filter_messages = messages.MessageField(PairwiseFilterMessage, 2, repeated=True) - - -class Feature(messages.Message): - annotated_type = messages.StringField(1) - chr = messages.StringField(2) - start = messages.IntegerField(3) - end = messages.IntegerField(4) - label = messages.StringField(5) - mutation_count = messages.IntegerField(6) - source = messages.StringField(7) - - -class Association(messages.Message): - node1 = messages.MessageField(Feature, 1) - node2 = messages.MessageField(Feature, 2) - logged_pvalue = messages.FloatField(3) - - -class CircvizOutput(messages.Message): - items = messages.MessageField(Association, 1, repeated=True) - - -Pairwise_Endpoints = endpoints.api(name='pairwise', version='v1') - - -@Pairwise_Endpoints.api_class(resource_name='pairwise_api') -class PairwiseApi(remote.Service): - """Pairwise API v1""" - - @endpoints.method(PairwiseJobRequest, PairwiseResults, name="run", http_method="POST") - def run_job(self, request): - """ Used by the web application.""" - features = [] - count = len(request.feature) - 1 - while count >= 0: - features.append(str(request.feature[count])) - count -= 1 - - prepped_features = Pairwise.prepare_features(request.cohort_id, features) - outputs = Pairwise.run_pairwise(prepped_features) - - results = PairwiseResults(result_vectors=[], filter_messages=[]) - logging.info(results) - - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - - return results - - @endpoints.method(message_types.VoidMessage, CircvizOutput, - path='precomp', http_method='GET', name='precomputed') - def precomputed_results(self, request): - """ Used by the web application.""" - bq_table = 'brca_pwpv' - query = 'SELECT A_valueType, A_chr, A_startPos, A_endPos, A_featureName, A_N, A_dataType,' \ - 'B_valueType, B_chr, B_startPos, B_endPos, B_featureName, B_N, B_dataType,' \ - 'logP FROM [isb-cgc:test.brca_pwpv] ' \ - 'where B_chr != "null" ' \ - 'and A_chr != "null"' \ - 'and A_startPos != "null" and A_endPos != "null"' \ - 'and B_startPos != "null" and B_endPos != "null"' \ - 'LIMIT 50;' - query_body = { - 'query': query - } - bigquery_service = authorize_credentials_with_Google() - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=settings.BQ_PROJECT_ID, body=query_body).execute() - association_list = [] - feature_list = [] - for row in query_response['rows']: - node1 = Feature( - annotated_type=row['f'][0]['v'].encode('utf-8') if row['f'][0]['v'] else None, - chr=row['f'][1]['v'].encode('utf-8').replace('chr','') if row['f'][1]['v'] else None, - start=int(row['f'][2]['v']) if row['f'][2]['v'] else None, - end=int(row['f'][3]['v']) if row['f'][3]['v'] else None, - label=row['f'][4]['v'].encode('utf-8') if row['f'][4]['v'] else '', - mutation_count=int(row['f'][5]['v']) if row['f'][5]['v'] else None, - source=row['f'][6]['v'].encode('utf-8') if row['f'][6]['v'] else None - ) - node2 = Feature( - annotated_type=row['f'][7]['v'].encode('utf-8') if row['f'][7]['v'] else None, - chr=row['f'][8]['v'].encode('utf-8').replace('chr','') if row['f'][8]['v'] else None, - start=int(row['f'][9]['v']) if row['f'][9]['v'] else None, - end=int(row['f'][10]['v']) if row['f'][10]['v'] else None, - label=row['f'][11]['v'].encode('utf-8') if row['f'][11]['v'] else '', - mutation_count=int(row['f'][12]['v']) if row['f'][12]['v'] else None, - source=row['f'][13]['v'].encode('utf-8') if row['f'][13]['v'] else None - ) - logP = float(row['f'][14]['v']) - association_list.append(Association(node1=node1, node2=node2, logged_pvalue=logP)) - feature_list.append(node1) - feature_list.append(node2) - return CircvizOutput(items=association_list) - - -APPLICATION = endpoints.api_server([PairwiseApi]) diff --git a/api/schema/__init__.py b/api/schema/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api/schema/tcga_clinical.py b/api/schema/tcga_clinical.py deleted file mode 100755 index 0ec1191b..00000000 --- a/api/schema/tcga_clinical.py +++ /dev/null @@ -1,283 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -# Updated from -# https://github.com/isb-cgc/data-prototyping/blob/537c5c24646f87bda804ca95dee6cf479f0b1fb9/tcga_etl_pipeline/schemas/clinical.json - -schema = [ - { - "type": "STRING", - "name": "ParticipantBarcode" - }, - { - "type": "STRING", - "name": "Study" - }, - { - "type": "STRING", - "name": "Project" - }, - { - "type": "STRING", - "name": "ParticipantUUID" - }, - { - "type": "STRING", - "name": "TSSCode" - }, - { - "type": "INTEGER", - "name": "age_at_initial_pathologic_diagnosis" - }, - { - "type": "STRING", - "name": "anatomic_neoplasm_subdivision" - }, - { - "type": "INTEGER", - "name": "batch_number" - }, - { - "type": "STRING", - "name": "bcr" - }, - { - "type": "STRING", - "name": "clinical_M" - }, - { - "type": "STRING", - "name": "clinical_N" - }, - { - "type": "STRING", - "name": "clinical_T" - }, - { - "type": "STRING", - "name": "clinical_stage" - }, - { - "type": "STRING", - "name": "colorectal_cancer" - }, - { - "type": "STRING", - "name": "country" - }, - { - "type": "STRING", - "name": "vital_status" - }, - { - "type": "INTEGER", - "name": "days_to_birth" - }, - { - "type": "INTEGER", - "name": "days_to_death" - }, - { - "type": "INTEGER", - "name": "days_to_last_known_alive" - }, - { - "type": "INTEGER", - "name": "days_to_last_followup" - }, - { - "type": "INTEGER", - "name": "days_to_initial_pathologic_diagnosis" - }, - { - "type": "INTEGER", - "name": "days_to_submitted_specimen_dx" - }, - { - "type": "STRING", - "name": "ethnicity" - }, - { - "type": "STRING", - "name": "frozen_specimen_anatomic_site" - }, - { - "type": "STRING", - "name": "gender" - }, - { - "type": "FLOAT", - "name": "gleason_score_combined" - }, - { - "type": "STRING", - "name": "histological_type" - }, - { - "type": "STRING", - "name": "history_of_colon_polyps" - }, - { - "type": "STRING", - "name": "history_of_neoadjuvant_treatment" - }, - { - "type": "STRING", - "name": "hpv_calls" - }, - { - "type": "STRING", - "name": "hpv_status" - }, - { - "type": "STRING", - "name": "icd_10" - }, - { - "type": "STRING", - "name": "icd_o_3_histology" - }, - { - "type": "STRING", - "name": "icd_o_3_site" - }, - { - "type": "STRING", - "name": "lymphatic_invasion" - }, - { - "type": "STRING", - "name": "lymphnodes_examined" - }, - { - "type": "STRING", - "name": "lymphovascular_invasion_present" - }, - { - "type": "STRING", - "name": "menopause_status" - }, - { - "type": "STRING", - "name": "mononucleotide_and_dinucleotide_marker_panel_analysis_status" - }, - { - "type": "FLOAT", - "name": "mononucleotide_marker_panel_analysis_status" - }, - { - "type": "STRING", - "name": "neoplasm_histologic_grade" - }, - { - "type": "STRING", - "name": "new_tumor_event_after_initial_treatment" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_examined" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_positive_by_he" - }, - { - "type": "FLOAT", - "name": "number_pack_years_smoked" - }, - { - "type": "INTEGER", - "name": "year_of_initial_pathologic_diagnosis" - }, - { - "type": "STRING", - "name": "pathologic_M" - }, - { - "type": "STRING", - "name": "pathologic_N" - }, - { - "type": "STRING", - "name": "pathologic_T" - }, - { - "type": "STRING", - "name": "pathologic_stage" - }, - { - "type": "STRING", - "name": "person_neoplasm_cancer_status" - }, - { - "type": "STRING", - "name": "pregnancies" - }, - { - "type": "STRING", - "name": "primary_neoplasm_melanoma_dx" - }, - { - "type": "STRING", - "name": "primary_therapy_outcome_success" - }, - { - "type": "STRING", - "name": "prior_dx" - }, - { - "type": "FLOAT", - "name": "psa_value" - }, - { - "type": "STRING", - "name": "race" - }, - { - "type": "STRING", - "name": "residual_tumor" - }, - { - "type": "STRING", - "name": "tobacco_smoking_history" - }, - { - "type": "STRING", - "name": "tumor_tissue_site" - }, - { - "type": "STRING", - "name": "tumor_type" - }, - { - "type": "STRING", - "name": "venous_invasion" - }, - { - "type": "FLOAT", - "name": "weight" - }, - { - "type": "FLOAT", - "name": "height" - }, - { - "type": "FLOAT", - "name": "BMI" - } -] \ No newline at end of file diff --git a/api/seqpeek_api.py b/api/seqpeek_api.py deleted file mode 100644 index 2c8e2c4a..00000000 --- a/api/seqpeek_api.py +++ /dev/null @@ -1,94 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from api.data_access import PlotDataCohortInfo - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class MAFRecord(Message): - sample_id = StringField(1) - patient_id = StringField(2) - aliquot_id = StringField(3) - hugo_symbol = StringField(4) - uniprot_aapos = IntegerField(5, variant=Variant.INT32) - uniprot_id = StringField(6) - variant_classification = StringField(7) - cohort = IntegerField(8, repeated=True) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 2, repeated=True) - -SeqPeekDataEndpointsAPI = endpoints_api(name='seqpeek_data_api', version='v1', - description='Endpoints used by the seqpeek visualization in the web application.') - - -def maf_array_to_record(maf_array): - data_points = [] - for item in maf_array: - data_points.append(MAFRecord(**item)) - - return data_points - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekDataAccessAPI(remote.Service): - - def create_response(self, maf_with_cohorts): - - data_points = maf_array_to_record(maf_with_cohorts.maf_vector) - - cohort_info_obj_array = [] - for item in maf_with_cohorts.cohort_info: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return MAFRecordList(items=data_points, cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, MAFRecordList, - path='by_gnab_feature', http_method='GET', name='seqpeek.getMAFDataWithCohorts') - def data_access_by_gnab_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - - maf_with_cohorts = SeqPeekMAFDataFormatter().format_maf_vector_for_view(feature_id, cohort_id_array) - response = self.create_response(maf_with_cohorts) - return response - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api/seqpeek_view_api.py b/api/seqpeek_view_api.py deleted file mode 100644 index b7dbabe2..00000000 --- a/api/seqpeek_view_api.py +++ /dev/null @@ -1,238 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from endpoints import method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant - -from bq_data_access.seqpeek.seqpeek_view import SeqPeekViewDataBuilder -from bq_data_access.data_access import get_feature_vectors_tcga_only -from api.seqpeek_api import SeqPeekDataEndpointsAPI, MAFRecord, maf_array_to_record -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter -from bq_data_access.seqpeek_maf_data import SeqPeekDataProvider -from bq_data_access.data_access import ProviderClassQueryDescription -from api.data_access import fetch_isbcgc_project_set -from api.api_helpers import sql_connection - -from projects.models import Project - -class SeqPeekViewDataRequest(Message): - hugo_symbol = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class InterproMatchLocation(Message): - # TODO this should likely be a float - score = IntegerField(1, variant=Variant.INT32) - start = IntegerField(2, variant=Variant.INT32) - end = IntegerField(3, variant=Variant.INT32) - - -class InterproMatch(Message): - status = StringField(1) - name = StringField(2) - evd = StringField(3) - locations = MessageField(InterproMatchLocation, 4, repeated=True) - dbname = StringField(5) - id = StringField(6) - - -class InterproJson(Message): - matches = MessageField(InterproMatch, 1, repeated=True) - uniprot_id = StringField(2) - length = IntegerField(3, variant=Variant.INT32) - name = StringField(4) - - -class InterproItem(Message): - uniprot_id = StringField(1) - interpro_json = MessageField(InterproJson, 2, repeated=False) - - -class SeqPeekRegionRecord(Message): - type = StringField(1, required=True) - start = IntegerField(2, required=True, variant=Variant.INT32) - end = IntegerField(3, required=True, variant=Variant.INT32) - - -class SeqPeekTrackRecord(Message): - mutations = MessageField(MAFRecord, 1, repeated=True) - type = StringField(2, required=True) - label = StringField(3, required=True) - number_of_samples = IntegerField(4, required=True) - mutated_positions = IntegerField(5, required=True) - cohort_size = IntegerField(6, required=False) - row_id = StringField(7, required=True) - - -class SeqPeekViewPlotDataRecord(Message): - tracks = MessageField(SeqPeekTrackRecord, 1, repeated=True) - protein = MessageField(InterproItem, 2, required=False) - regions = MessageField(SeqPeekRegionRecord, 3, repeated=True) - - -class SeqPeekRemovedRow(Message): - name = StringField(1, required=True) - num = IntegerField(2, required=True) - - -class SeqPeekViewRecord(Message): - cohort_id_list = StringField(1, repeated=True) - hugo_symbol = StringField(2, required=True) - plot_data = MessageField(SeqPeekViewPlotDataRecord, 3, required=True) - removed_row_statistics = MessageField(SeqPeekRemovedRow, 4, repeated=True) - - -def create_interpro_record(interpro_literal): - match_data = [] - for match in interpro_literal['matches']: - - match_location_data = [] - for location in match['locations']: - match_location_data.append(InterproMatchLocation( - score=int(location['score']), - start=int(location['start']), - end=int(location['end']) - )) - - match_data.append(InterproMatch( - status=str(match['status']), - name=str(match['name']), - evd=str(match['evd']), - locations=match_location_data, - dbname=str(match['dbname']), - id=str(match['id']), - )) - - interpro_json = InterproJson( - matches=match_data, - uniprot_id=str(interpro_literal['uniprot_id']), - length=int(interpro_literal['length']), - name=str(interpro_literal['name']) - ) - - return InterproItem( - interpro_json=interpro_json - ) - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekViewDataAccessAPI(remote.Service): - def build_gnab_feature_id(self, gene_label): - return "GNAB:{gene_label}:variant_classification".format(gene_label=gene_label) - - def create_response(self, seqpeek_view_data): - plot_data = seqpeek_view_data['plot_data'] - tracks = [] - for track in plot_data['tracks']: - mutations = maf_array_to_record(track['mutations']) - tracks.append(SeqPeekTrackRecord(mutations=mutations, label=track['label'], type=track["type"], - row_id=track['render_info']['row_id'], - number_of_samples=track['statistics']['samples']['numberOf'], - mutated_positions=track['statistics']['samples']['mutated_positions'], - cohort_size=track['statistics']['cohort_size'])) - - region_records = [] - for region in plot_data['regions']: - region_records.append(SeqPeekRegionRecord(**region)) - - protein = create_interpro_record(plot_data['protein']) - plot_data_record = SeqPeekViewPlotDataRecord(tracks=tracks, protein=protein, regions=region_records) - - removed_row_statistics = [] - for item in seqpeek_view_data['removed_row_statistics']: - removed_row_statistics.append(SeqPeekRemovedRow(**item)) - - return SeqPeekViewRecord(plot_data=plot_data_record, hugo_symbol=seqpeek_view_data['hugo_symbol'], - cohort_id_list=seqpeek_view_data['cohort_id_list'], - removed_row_statistics=removed_row_statistics) - - @endpoints_method(SeqPeekViewDataRequest, SeqPeekViewRecord, - path='view_data', http_method='GET', name='seqpeek.getViewData') - def seqpeek_view_data(self, request): - try: - hugo_symbol = request.hugo_symbol - cohort_id_array = request.cohort_id - - gnab_feature_id = self.build_gnab_feature_id(hugo_symbol) - logging.debug("GNAB feature ID for SeqPeke: {0}".format(gnab_feature_id)) - - # Get the project IDs these cohorts' samples come from - cohort_vals = tuple(int(i) for i in cohort_id_array) - cohort_params = ('%s,' * len(cohort_id_array))[:-1] - - db = sql_connection() - cursor = db.cursor() - - isbcgc_projects = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN (%s);" % cohort_params, - cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_project_ids = [] - unconfirmed_project_ids = [] - - for row in cursor.fetchall(): - if row[0] in isbcgc_projects: - if row[0] not in confirmed_project_ids: - confirmed_project_ids.append(row[0]) - elif row[0] not in unconfirmed_project_ids: - unconfirmed_project_ids.append(row[0]) - - if len(unconfirmed_project_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_project_ids) - - for proj in projects: - if proj.get_my_root_and_depth()['root'] in isbcgc_projects: - confirmed_project_ids.append(proj.id) - - async_params = [ProviderClassQueryDescription(SeqPeekDataProvider, gnab_feature_id, cohort_id_array, confirmed_project_ids)] - maf_data_result = get_feature_vectors_tcga_only(async_params, skip_formatting_for_plot=True) - - maf_data_vector = maf_data_result[gnab_feature_id]['data'] - - if len(maf_data_vector) > 0: - # Since the gene (hugo_symbol) parameter is part of the GNAB feature ID, - # it will be sanity-checked in the SeqPeekMAFDataAccess instance. - seqpeek_data = SeqPeekMAFDataFormatter().format_maf_vector_for_view(maf_data_vector, cohort_id_array) - - seqpeek_maf_vector = seqpeek_data.maf_vector - seqpeek_cohort_info = seqpeek_data.cohort_info - removed_row_statistics_dict = seqpeek_data.removed_row_statistics - - seqpeek_view_data = SeqPeekViewDataBuilder().build_view_data(hugo_symbol, - seqpeek_maf_vector, - seqpeek_cohort_info, - cohort_id_array, - removed_row_statistics_dict) - - response = self.create_response(seqpeek_view_data) - return response - else: - # No data found - return SeqPeekViewRecord(plot_data=SeqPeekViewPlotDataRecord(tracks=[], protein=None, regions=[]), - hugo_symbol=hugo_symbol, cohort_id_list=[str(i) for i in cohort_id_array], - removed_row_statistics=[]) - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - diff --git a/api/single_feature_access.py b/api/single_feature_access.py deleted file mode 100644 index e270610f..00000000 --- a/api/single_feature_access.py +++ /dev/null @@ -1,131 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import time - -from api.data_access import FeatureDataEndpointsAPI, PlotDataCohortInfo - -from endpoints import method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class DataPointList(Message): - type = EnumField(ValueType, 1, required=True) - items = MessageField(DataPoint, 2, repeated=True) - label = StringField(3, required=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class SingleFeatureDataAccess(remote.Service): - def get_feature_vector(self, feature_id, cohort_id_array): - start = time.time() - - async_params = [(feature_id, cohort_id_array)] - async_result = get_feature_vectors_with_user_data(async_params) - - feature_type, feature_vec = async_result[feature_id]['type'], async_result[feature_id]['data'] - - end = time.time() - time_elapsed = end-start - logging.info('Time elapsed: ' + str(time_elapsed)) - - vms = VectorMergeSupport('NA', 'sample_id', [feature_id]) - vms.add_dict_array(feature_vec, feature_id, 'value') - - merged = vms.get_merged_dict() - - return feature_type, merged - - def annotate_vector_with_cohorts(self, cohort_id_array, merged): - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - value_bundle['cohort'] = cohort_set - - def get_cohort_information(self, cohort_id_array): - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - - return cohort_info_array - - def create_response(self, feature_id, vector_type, vector, cohort_info_array): - data_points = [] - for item in vector: - data_points.append(DataPoint( - sample_id=item['sample_id'], - value=item[feature_id], - cohort=item['cohort'] - )) - - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return DataPointList(type=vector_type, items=data_points, label=feature_id, - cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, DataPointList, - path='feature_data', http_method='GET', name='feature_access.getFeatureData') - def data_access_by_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - vector_type, vector = self.get_feature_vector(feature_id, cohort_id_array) - - self.annotate_vector_with_cohorts(cohort_id_array, vector) - - cohort_info = self.get_cohort_information(cohort_id_array) - response = self.create_response(feature_id, vector_type, vector, cohort_info) - return response - except FeatureNotFoundException as fnf: - logging.error("Invalid internal feature ID '{}'".format(str(fnf))) - raise NotFoundException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/README.md b/api_3/README.md deleted file mode 100755 index d1213d73..00000000 --- a/api_3/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# ISB-CGC-API/api_3 -Google Endpoints API Used to Drive CGC Webapp. -The endpoints defined here are intended for the switch over to the GDC -metadata schema and to migrate to the Cloud Endpoints Frameworks for App Engine -The original `api` directory was copied and named `api_3` and all the endpoint versions -set to '3' diff --git a/api_3/__init__.py b/api_3/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/api_helpers.py b/api_3/api_helpers.py deleted file mode 100755 index 6ad2483d..00000000 --- a/api_3/api_helpers.py +++ /dev/null @@ -1,483 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import os -import MySQLdb -import httplib2 -from oauth2client.client import GoogleCredentials, AccessTokenCredentials -from django.conf import settings -from googleapiclient.discovery import build - -from cohorts.metadata_helpers import get_sql_connection - -debug = settings.DEBUG - -WHITELIST_RE = ur'([^\\\_\|\"\+~@:#\$%\^&\*=\-\.,\(\)0-9a-zA-Z\s\xc7\xfc\xe9\xe2\xe4\xe0\xe5\xe7\xea\xeb\xe8\xef\xee\xed\xec\xc4\xc5\xc9\xe6\xc6\xf4\xf6\xf2\xfb\xf9\xd6\xdc\xe1\xf3\xfa\xf1\xd1\xc0\xc1\xc2\xc3\xc8\xca\xcb\xcc\xcd\xce\xcf\xd0\xd2\xd3\xd4\xd5\xd8\xd9\xda\xdb\xdd\xdf\xe3\xf0\xf5\xf8\xfd\xfe\xff])' - -MOLECULAR_CATEGORIES = { - 'nonsilent': [ - 'Missense_Mutation', - 'Nonsense_Mutation', - 'Nonstop_Mutation', - 'Frame_Shift_Del', - 'Frame_Shift_Ins', - 'De_novo_Start_OutOfFrame', - 'In_Frame_Del', - 'In_Frame_Ins', - 'Start_Codon_SNP', - 'Start_Codon_Del', - ] -} - -# Database connection -def sql_connection(): - return get_sql_connection() - -def sql_bmi_by_ranges(value): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - if 'None' in value: - result += 'BMI is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == 'underweight': - result += ' (BMI < 18.5)' - elif str(val) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(val) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(val) == 'obese': - result += ' (BMI > 29.9)' - - else: - # value is a single range - if str(value) == 'underweight': - result += ' (BMI < 18.5)' - elif str(value) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(value) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(value) == 'obese': - result += ' (BMI > 29.9)' - - return result - - -def sql_age_by_ranges(value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - #value is a list of ranges - first = True - if 'None' in value: - result += 'age_at_diagnosis is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == '10 to 39': - result += ' (age_at_diagnosis >= 10 and age_at_diagnosis < 40)' - elif str(val) == '40 to 49': - result += ' (age_at_diagnosis >= 40 and age_at_diagnosis < 50)' - elif str(val) == '50 to 59': - result += ' (age_at_diagnosis >= 50 and age_at_diagnosis < 60)' - elif str(val) == '60 to 69': - result += ' (age_at_diagnosis >= 60 and age_at_diagnosis < 70)' - elif str(val) == '70 to 79': - result += ' (age_at_diagnosis >= 70 and age_at_diagnosis < 80)' - elif str(val).lower() == 'over 80': - result += ' (age_at_diagnosis >= 80)' - else: - #value is a single range - if str(value) == '10 to 39': - result += ' (age_at_diagnosis >= 10 and age_at_diagnosis < 40)' - elif str(value) == '40 to 49': - result += ' (age_at_diagnosis >= 40 and age_at_diagnosis < 50)' - elif str(value) == '50 to 59': - result += ' (age_at_diagnosis >= 50 and age_at_diagnosis < 60)' - elif str(value) == '60 to 69': - result += ' (age_at_diagnosis >= 60 and age_at_diagnosis < 70)' - elif str(value) == '70 to 79': - result += ' (age_at_diagnosis >= 70 and age_at_diagnosis < 80)' - elif str(value).lower() == 'over 80': - result += ' (age_at_diagnosis >= 80)' - elif str(value) == 'None': - result += ' age_at_diagnosis is null' - - return result - -def gql_age_by_ranges(q, key, value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - for val in value: - if first: - first = False - else: - result += ' or' - if str(val) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(val) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(val) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(val) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(val) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(val).lower() == 'over80': - result += ' (%s >= 80)' % key - else: - # value is a single range - if str(value) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(value) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(value) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(value) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(value) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(value).lower() == 'over80': - result += ' (%s >= 80)' % key - return result - - -def normalize_bmi(bmis): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - bmi_list = {'underweight': 0, 'normal weight': 0, 'overweight': 0, 'obese': 0, 'None': 0} - for bmi, count in bmis.items(): - if type(bmi) != dict: - if bmi and bmi != 'None': - fl_bmi = float(bmi) - if fl_bmi < 18.5: - bmi_list['underweight'] += int(count) - elif 18.5 <= fl_bmi <= 24.9: - bmi_list['normal weight'] += int(count) - elif 25 <= fl_bmi <= 29.9: - bmi_list['overweight'] += int(count) - elif fl_bmi >= 30: - bmi_list['obese'] += int(count) - else: - bmi_list['None'] += int(count) - - return bmi_list - - -def normalize_ages(ages): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age, count in ages.items(): - if type(age) != dict: - if age and age != 'None': - int_age = float(age) - if int_age < 40: - new_age_list['10 to 39'] += int(count) - elif int_age < 50: - new_age_list['40 to 49'] += int(count) - elif int_age < 60: - new_age_list['50 to 59'] += int(count) - elif int_age < 70: - new_age_list['60 to 69'] += int(count) - elif int_age < 80: - new_age_list['70 to 79'] += int(count) - else: - new_age_list['Over 80'] += int(count) - else: - new_age_list['None'] += int(count) - else: - print age - - return new_age_list - -def applyFilter(field, dict): -# this one gets called a lot... -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - query_dict = dict.copy() - if field in dict: - query_dict.pop(field, None) - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - else: - where_clause = None - else: - where_clause = build_where_clause(dict) - - return where_clause - - -def build_where_clause(filters, alt_key_map=False): -# this one gets called a lot -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - first = True - query_str = '' - big_query_str = '' # todo: make this work for non-string values -- use {}.format - value_tuple = () - key_order = [] - keyType = None - gene = None - - grouped_filters = None - - for key, value in filters.items(): - if isinstance(value, dict) and 'values' in value: - value = value['values'] - - if isinstance(value, list) and len(value) == 1: - value = value[0] - # Check if we need to map to a different column name for a given key - if alt_key_map and key in alt_key_map: - key = alt_key_map[key] - - # Multitable where's will come in with : in the name. Only grab the column piece for now - # TODO: Shouldn't throw away the entire key - elif ':' in key: - keyType = key.split(':')[0] - if keyType == 'MUT': - gene = key.split(':')[1] - key = key.split(':')[-1] - - # Multitable filter lists don't come in as string as they can contain arbitrary text in values - elif isinstance(value, basestring): - # If it's a list of values, split it into an array - if ',' in value: - value = value.split(',') - - key_order.append(key) - - # Bucket the grouped filter types (currently just certain has_ values, could be more) - if 'has_' in key and not key == 'has_Illumina_DNASeq' and not key == 'has_SNP6' and not key == 'has_RPPA': - if grouped_filters is None: - grouped_filters = {} - - if key == 'has_27k' or key == 'has_450k': - if 'DNA_methylation' not in grouped_filters: - grouped_filters['DNA_methylation'] = [] - grouped_filters['DNA_methylation'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_HiSeq_miRnaSeq' or key == 'has_GA_miRNASeq': - if 'miRNA_sequencing' not in grouped_filters: - grouped_filters['miRNA_sequencing'] = [] - grouped_filters['miRNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_UNC_HiSeq_RNASeq' or key == 'has_UNC_GA_RNASeq' or key == 'has_BCGSC_HiSeq_RNASeq' or key == 'has_BCGSC_GA_RNASeq': - if 'RNA_sequencing' not in grouped_filters: - grouped_filters['RNA_sequencing'] = [] - grouped_filters['RNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - # BQ-only format - elif keyType == 'MUT': - # If it's first in the list, don't append an "and" - params = {} - value_tuple += (params,) - - if first: - first = False - else: - big_query_str += ' AND' - - big_query_str += " %s = '{hugo_symbol}' AND " % 'Hugo_Symbol' - params['gene'] = gene - - if(key == 'category'): - if value == 'any': - big_query_str += '%s IS NOT NULL' % 'Variant_Classification' - params['var_class'] = '' - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = MOLECULAR_CATEGORIES[value] - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = value - - if value != 'any': - if isinstance(values, list): - j = 0 - for vclass in values: - if j == 0: - params['var_class'] = "'%s'" % vclass.replace("'", "\\'") - j = 1 - else: - params['var_class'] += ",'%s'" % vclass.replace("'", "\\'") - else: - params['var_class'] = "'%s'" % values.replace("'", "\\'") - - else: - # If it's first in the list, don't append an "and" - if first: - first = False - else: - query_str += ' and' - big_query_str += ' and' - - # If it's age ranges, give it special treament due to normalizations - if key == 'age_at_diagnosis': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_age_by_ranges(value) + ') ' - # If it's age ranges, give it special treament due to normalizations - elif key == 'BMI': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_bmi_by_ranges(value) + ') ' - # If it's a list of items for this key, create an or subclause - elif isinstance(value, list): - has_null = False - if 'None' in value: - has_null = True - query_str += ' (%s is null or' % key - big_query_str += ' (%s is null or' % key - value.remove('None') - query_str += ' %s in (' % key - big_query_str += ' %s in (' % key - i = 0 - for val in value: - value_tuple += (val.strip(),) if type(val) is unicode else (val,) - if i == 0: - query_str += '%s' - big_query_str += '"' + str(val) + '"' - i += 1 - else: - query_str += ',%s' - big_query_str += ',' + '"' + str(val) + '"' - query_str += ')' - big_query_str += ')' - if has_null: - query_str += ')' - big_query_str += ')' - - # If it's looking for None values - elif value == 'None': - query_str += ' %s is null' % key - big_query_str += ' %s is null' % key - - # For the general case - else: - if key == 'fl_archive_name': - big_query_str += ' %s like' % key - big_query_str += ' "%' + value + '%"' - elif key == 'fl_data_level': - big_query_str += ' %s=%s' % (key, value) - elif type(value) == bool: - big_query_str += ' %s=%r' % (key, value) - else: - query_str += ' %s=' % key - big_query_str += ' %s=' % key - query_str += '%s' - big_query_str += '"%s"' % value - value_tuple += (value.strip(),) if type(value) is unicode else (value,) - - # Handle our data buckets - if grouped_filters: - for bucket in grouped_filters: - if not query_str == '': - query_str += ' and ' - big_query_str += ' and ' - - query_str += '( ' - big_query_str += '( ' - - first = True - for filter in grouped_filters[bucket]: - if first: - first = False - else: - query_str += ' or ' - big_query_str += ' or ' - - query_str += ' %s=' % filter['filter'] - big_query_str += ' %s=' % filter['filter'] - query_str += '%s' - big_query_str += '"%s"' % filter['value'] - value_tuple += (filter['value'].strip(),) if type(filter['value']) is unicode else (filter['value'],) - - query_str += ' )' - big_query_str += ' )' - - return {'query_str': query_str, 'value_tuple': value_tuple, 'key_order': key_order, 'big_query_str': big_query_str} - - -def possible_future_authorization_function(): - # will put a decorator on this to ensure user has correct authorization before running - # such as if they are dbgap authorized - from oauth2client.client import flow_from_clientsecrets - from oauth2client.file import Storage - from oauth2client import tools - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - flow = flow_from_clientsecrets(settings.CLIENT_SECRETS, scope='https://www.googleapis.com/auth/bigquery') - ## in future, make storage file temporary somehow? - storage = Storage('bigquery_credentials.dat') - credentials = storage.get() - - if credentials is None or credentials.invalid: - credentials = tools.run_flow(flow, storage, tools.argparser.parse_args([])) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - return service - - -def authorize_credentials_with_Google(): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - # credentials = GoogleCredentials.get_application_default().create_scoped(SCOPES) - credentials = GoogleCredentials.from_stream(settings.GOOGLE_APPLICATION_CREDENTIALS).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - if debug: print >> sys.stderr,' big query authorization '+sys._getframe().f_code.co_name - return service - -# TODO refactor to remove duplicate code -def authorize_credentials_with_google_from_file(credentials_path): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - credentials = GoogleCredentials.from_stream(credentials_path).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - - return service - - -def get_user_email_from_token(access_token): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - user_email = None - credentials = AccessTokenCredentials(access_token, 'test-user') - http = credentials.authorize(httplib2.Http()) - user_info_service = build('oauth2', 'v2', http=http) - user_info = user_info_service.userinfo().get().execute() - if 'email' in user_info: - user_email = user_info['email'] - return user_email diff --git a/api_3/cloudstoragefilepaths_helper.py b/api_3/cloudstoragefilepaths_helper.py deleted file mode 100755 index f6d47139..00000000 --- a/api_3/cloudstoragefilepaths_helper.py +++ /dev/null @@ -1,212 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from collections import OrderedDict -import logging -import MySQLdb - -try: - import endpoints -except Exception as e: - print 'couldn\'t import google endpoints, using mock for testing: %s' % (e) -import django - -from django.conf import settings -from django.core.signals import request_finished -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from sharing.models import User -# from django.contrib.auth.models import User as Django_User -try: - from protorpc import remote, messages -except Exception as e: - print 'couldn\'t import google protorpc, using mock for testing: %s' % (e) - -from api_3.api_helpers import sql_connection -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - -class GCSFilePathList(messages.Message): - cloud_storage_file_paths = messages.StringField(1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -class CloudStorageFilePathsAPI(remote.Service): - def setup_param_map(self, request, param_names): - param_map = OrderedDict() - for param_name in param_names: - param_map[param_name] = request.get_assigned_value(param_name) - return param_map - - - def get_genomic_builds(self, param_map, program): - if 'CCLE' == program: - builds = ['HG19'] - else: - builds = ['HG19', 'HG38'] - - if 'genomic_build' in param_map and param_map['genomic_build']: - if 'HG19' == param_map['genomic_build'].upper(): - builds = ['HG19'] - elif 'HG38' == param_map['genomic_build'].upper(): - builds = ['HG38'] - else: - msg = 'Unknown genomic build: {}. Acceptable genomic builds are HG19 and HG38.'.format(param_map['genomic_build']) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for cohort. {}".format(msg)) - return builds - - def get_cloud_storage_file_paths(self, param_map, program): - """ - Uses the param_map to pass to the query builder then executes the query to obtain the cloud - storage file paths. if cohort_id is on the parameter list, verifies that the calling user has - permission for the cohort. - """ - cursor = None - db = None - - query_str, query_tuple = self.build_query(param_map, program) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - cursor_rows = cursor.fetchall() - cloud_storage_path_list = [row['file_name_key'] for row in cursor_rows] - return GCSFilePathList(cloud_storage_file_paths=cloud_storage_path_list, count=len(cloud_storage_path_list)) - except (IndexError, TypeError), e: - logger.warn('problem getting file paths: {}\nSQL: {}\nparams: {}'.format(e, query_str, query_tuple)) - raise endpoints.NotFoundException("File paths for {} {} not found." \ - .format('cohort', param_map['cohort_id'] if 'cohort_id' in param_map else 'sample', param_map['sample_barcode'])) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving file paths. {}\nSQL: {}\nparams: {}".format(e, query_str, query_tuple)) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(e)) - finally: - if cursor: - cursor.close() - if db and db.open: - db.close() - -class SamplesCloudStorageFilePathsHelper(CloudStorageFilePathsAPI): - - GET_RESOURCE = endpoints.ResourceContainer( - sample_barcode=messages.StringField(1, required=True), - data_type=messages.StringField(2), - data_category=messages.StringField(3), - experimental_strategy=messages.StringField(4), - data_format=messages.StringField(5), - platform=messages.StringField(6), - genomic_build=messages.StringField(7), - analysis_workflow_type=messages.StringField(8) - ) - - def build_query(self, param_map, program): - builds = self.get_genomic_builds(param_map, program) - final_query_str = '' - query_tuple = [] - for build in builds: - query_str = 'SELECT md.file_name_key, md.access ' \ - 'FROM {}_metadata_data_{} md '.format(program, build) - - query_str += 'WHERE sample_barcode=%s ' - query_tuple += [param_map['sample_barcode']] - query_str += 'AND file_name_key != "" AND file_name_key is not null ' - for field, value in param_map.iteritems(): - if field not in ['limit', 'cohort_id', 'sample_barcode', 'genomic_build'] and value: - query_str += ' and md.{}=%s '.format(field) - query_tuple += [value] - query_str += ' GROUP BY md.file_name_key, md.access ' - if 0 < len(final_query_str): - final_query_str += ' UNION ' - final_query_str += query_str - - if 'limit' in param_map and param_map['limit']: - final_query_str += ' LIMIT %s' - query_tuple += [param_map['limit']] - else: - final_query_str += ' LIMIT 10000' - return final_query_str, query_tuple - - def cloud_storage_file_paths(self, request, program): - param_map = self.setup_param_map(request, [ - 'data_type', - 'data_category', - 'experimental_strategy', - 'data_format', - 'platform', - 'genomic_build', - 'analysis_workflow_type', - 'sample_barcode' - ] - ) - return self.get_cloud_storage_file_paths(param_map, program) - -class CohortsCloudStorageFilePathsHelper(CloudStorageFilePathsAPI): - - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - limit=messages.IntegerField(2), - data_type=messages.StringField(3), - data_category=messages.StringField(4), - experimental_strategy=messages.StringField(5), - data_format=messages.StringField(6), - platform=messages.StringField(7), - genomic_build=messages.StringField(8), - analysis_workflow_type=messages.StringField(9) - ) - - def validate_user(self, cohort_id): - user_email = None - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = User.objects.get(email=user_email).id - Django_Cohort.objects.get(id=cohort_id) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}.".format(user_email, cohort_id) - raise endpoints.UnauthorizedException(err_msg) - finally: - request_finished.send(self) - - def cloud_storage_file_paths(self, request): - param_map = self.setup_param_map(request, [ - 'data_type', - 'data_category', - 'experimental_strategy', - 'data_format', - 'platform', - 'genomic_build', - 'analysis_workflow_type', - 'limit', - 'cohort_id' - ] - ) - self.validate_user(param_map['cohort_id']) - return self.get_cloud_storage_file_paths(param_map, None) diff --git a/api_3/cohort_create_preview_helper.py b/api_3/cohort_create_preview_helper.py deleted file mode 100755 index 86b0e252..00000000 --- a/api_3/cohort_create_preview_helper.py +++ /dev/null @@ -1,396 +0,0 @@ -''' -Created on Mar 30, 2017 - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import django -import re -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages -from datetime import datetime -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from api_3.cohort_endpoint_helpers import are_there_bad_keys, are_there_no_acceptable_keys, construct_parameter_error_message - -from api_3.api_helpers import sql_connection, WHITELIST_RE -from bq_data_access.cohort_bigquery import BigQueryCohortSupport -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms, Samples, Filters -from projects.models import Program, Project - -logger = logging.getLogger(__name__) - -class CohortsCreatePreviewAPI(remote.Service): - def build_query_dictionaries(self, request): - """ - Builds the query dictionaries for create and preview cohort endpoints. - Returns query_dict, gte_query_dict, lte_query_dict. - """ - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - and k.name is not 'name' - and not k.name.endswith('_gte') - and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - return query_dict, gte_query_dict, lte_query_dict - - def build_query(self, program, table, query_dict, gte_query_dict, lte_query_dict): - """ - Builds the queries that selects the case and sample barcodes - that meet the criteria specified in the request body. - Returns case query string, sample query string, value tuple. - """ - if (0 == len(query_dict) and 0 == len(gte_query_dict) and 0 == len(lte_query_dict)): - return None, None - - if 'Clinical' == table: - query_str = 'SELECT sample_barcode, c.case_barcode, c.project_short_name ' \ - 'FROM {0}_metadata_clinical c join {0}_metadata_biospecimen b on c.case_barcode = b.case_barcode ' \ - 'WHERE '.format(program) - else: - query_str = 'SELECT sample_barcode, case_barcode, project_short_name ' \ - 'FROM {0}_metadata_{1} ' \ - 'WHERE '.format(program, table[:1].lower() + table[1:]) - value_tuple = () - - alias = 'c.' if 'Clinical' == table else '' - for key, value_list in query_dict.iteritems(): - query_str += ' AND ' if not query_str.endswith('WHERE ') else '' - if "None" in value_list: - value_list.remove("None") - query_str += ' ( {alias}{key} is null '.format(alias=alias, key=key) - if len(value_list) > 0: - query_str += ' OR {alias}{key} IN ({vals}) '.format( - alias=alias, key=key, vals=', '.join(['%s'] * len(value_list))) - query_str += ') ' - else: - query_str += ' {alias}{key} IN ({vals}) '.format(alias=alias, key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - query_str += ' AND ' if not query_str.endswith('WHERE ') else '' - query_str += ' {}{} >=%s '.format(alias, key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - query_str += ' AND ' if not query_str.endswith('WHERE ') else '' - query_str += ' {}{} <=%s '.format(alias, key) - value_tuple += (value,) - - query_str += ' GROUP BY sample_barcode, {0}case_barcode, {0}project_short_name'.format(alias) - - return query_str, value_tuple - - def query(self, request): - try: - db = None - try: - db = sql_connection() - except Exception as e: - logger.exception(e) - raise endpoints.NotFoundException("Error connecting to database: {}".format(e)) - - fields = request.get_assigned_value('Common') - common_query_dict = {} - - if fields: - common_query_dict, _, _ = self.build_query_dictionaries(fields) - - # Our return values - ret_query_dict = {} - ret_lte_query_dict = {} - ret_gte_query_dict = {} - ret_rows = None - - for table in ('Clinical', 'Biospecimen', 'Data_HG19', 'Data_HG38'): - if 'CCLE' == self.program and 'Data_HG38' == table: - continue - - fields = request.get_assigned_value(table) - - if fields and (are_there_bad_keys(fields) or are_there_no_acceptable_keys(fields)): - err_msg = construct_parameter_error_message(request, True) - raise endpoints.BadRequestException(err_msg) - - query_dict = {} - gte_query_dict = {} - lte_query_dict = {} - - if fields: - query_dict, gte_query_dict, lte_query_dict = self.build_query_dictionaries(fields) - - logger.info("query_dict: {}".format(str(query_dict))) - logger.info("common_query_dict: {}".format(str(common_query_dict))) - - query_dict.update(common_query_dict) - - ret_query_dict.update(query_dict) - ret_lte_query_dict.update(lte_query_dict) - ret_gte_query_dict.update(gte_query_dict) - - query_str, value_tuple = self.build_query(self.program, table, query_dict, gte_query_dict, lte_query_dict) - - if not query_str: - continue - - cursor = None - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - rows = list(cursor.fetchall()) - if 0 == len(rows): - # if any query returns no rows, then, overall, no sample_barcode will match - return [], '', '', '' - - if ret_rows is None: - ret_rows = rows - else: - cur_samples = set() - for row in rows: - cur_samples.add(row['sample_barcode']) - not_in_sample = [] - for row in ret_rows: - if row['sample_barcode'] not in cur_samples: - not_in_sample += [row] - for row in not_in_sample: - ret_rows.remove(row) - except (IndexError, TypeError) as e: - logger.exception(e) - raise endpoints.NotFoundException("Error retrieving samples and cases: {}\n{} {}".format(e, query_str, value_tuple)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tsample query: {} {}'.format(e, query_str, value_tuple) - logger.exception(msg) - raise endpoints.BadRequestException("Error creating/previewing cohort. {}".format(e)) - finally: - if cursor: - cursor.close() - finally: - if db and db.open: - db.close() - request_finished.send(self) - - return ret_rows, ret_query_dict, ret_lte_query_dict, ret_gte_query_dict - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - -class CreatedCohort(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - filters = messages.MessageField(FilterDetails, 4, repeated=True) - case_count = messages.IntegerField(5, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(6, variant=messages.Variant.INT32) - -class CohortsCreateHelper(CohortsCreatePreviewAPI): - BASE_URL = settings.BASE_URL - - def get_django_program(self, program_name): - try: - # get the ISB superuser - isb_superuser = Django_User.objects.get(username='isb', is_staff=True, is_superuser=True, is_active=True) - # get the program - program = Program.objects.get(name=program_name, is_public=True, active=True, owner=isb_superuser) - finally: - request_finished.send(self) - return program - - def get_django_project(self, project_short_name): - try: - # get the ISB superuser - isb_superuser = Django_User.objects.get(username='isb', is_staff=True, is_superuser=True, is_active=True) - # get the program - program = self.get_django_program(project_short_name.split('-')[0]) - # get the project - project = Project.objects.get(name=project_short_name[project_short_name.find('-') + 1:], active=True, owner=isb_superuser, program=program) - finally: - request_finished.send(self) - return project - - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - user = endpoints.get_current_user() - user_email = user.email() if user else None - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(self.BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - finally: - request_finished.send(self) - - # get the sample barcode information for use in creating the sample list for the cohort - rows, query_dict, lte_query_dict, gte_query_dict = self.query(request) - project2django = {} - if rows: - for row in rows: - if row['project_short_name'] not in project2django: - project2django[row['project_short_name']] = self.get_django_project(row['project_short_name']) - else: - raise endpoints.BadRequestException("No samples meet the specified parameters.") - - sample_barcodes = [{'sample_barcode': row['sample_barcode'], 'case_barcode': row['case_barcode'], 'project': project2django[row['project_short_name']]} for row in rows] - cohort_name = request.get_assigned_value('name') - - # Validate the cohort name against a whitelist - whitelist = re.compile(WHITELIST_RE, re.UNICODE) - match = whitelist.search(unicode(cohort_name)) - if match: - # XSS risk, log and fail this cohort save - match = whitelist.findall(unicode(cohort_name)) - logger.error( - '[ERROR] While saving a cohort, saw a malformed name: ' + cohort_name + ', characters: ' + match.__str__()) - raise endpoints.BadRequestException( - "Your cohort's name contains invalid characters (" + match.__str__() + "); please choose another name.") - - if len(sample_barcodes) == 0: - raise endpoints.BadRequestException( - "The cohort could not be saved because no samples meet the specified parameters.") - - # todo: maybe create all objects first, then save them all at the end? - # 1. create new cohorts_cohort with name, active=True, last_date_saved=now - try: - created_cohort = Django_Cohort.objects.create(name=cohort_name, active=True, last_date_saved=datetime.utcnow()) - created_cohort.save() - finally: - request_finished.send(self) - - # 2. insert samples into cohort_samples - try: - sample_list = [Samples(cohort=created_cohort, sample_barcode=sample['sample_barcode'], case_barcode=sample['case_barcode'], project=sample['project']) for sample in sample_barcodes] - Samples.objects.bulk_create(sample_list) - finally: - request_finished.send(self) - - # 3. Set permission for user to be owner - try: - perm = Cohort_Perms(cohort=created_cohort, user=django_user, perm=Cohort_Perms.OWNER) - perm.save() - finally: - request_finished.send(self) - - # 4. Create filters applied - filter_data = [] - django_program = self.get_django_program(self.program) - try: - # special case sample barcode since the list can be ALL the sample barcodes in the program - edit_barcodes = set() - for key, value_list in query_dict.items(): - if 'sample_barcode' == key: - edit_barcodes |= set(value_list) - continue - for val in value_list: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val, program=django_program).save() - if 0 < len(edit_barcodes): - if len(edit_barcodes) < 6: - val = 'barcodes: {}'.format(', '.join(sorted(list(edit_barcodes)))) - else: - val = '{} barcodes beginning with {}'.format(len(edit_barcodes), ', '.join(sorted(list(edit_barcodes))[:5])) - filter_data.append(FilterDetails(name='sample_barcode', value=val)) - Filters.objects.create(resulting_cohort=created_cohort, name='sample_barcode', value=val, program=django_program).save() - - for key, val in [(k + '_lte', v) for k, v in lte_query_dict.items()] + [(k + '_gte', v) for k, v in gte_query_dict.items()]: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val, program=django_program).save() - finally: - request_finished.send(self) - - # 5. Store cohort to BigQuery - project_id = settings.BQ_PROJECT_ID - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - bcs = BigQueryCohortSupport(project_id, cohort_settings.dataset_id, cohort_settings.table_id) - batch = 5000 - start = 0 - while start < len(sample_barcodes): - bcs.add_cohort_to_bq(created_cohort.id, sample_barcodes[start:start + batch]) - start += batch - - request_finished.send(self) - - return CreatedCohort( - id=str(created_cohort.id), - name=cohort_name, - last_date_saved=str(datetime.utcnow()), - filters=filter_data, - case_count=created_cohort.case_size(), - sample_count=len(sample_barcodes) - ) - -class CohortsPreviewHelper(CohortsCreatePreviewAPI): - class CohortCasesSamplesList(messages.Message): - cases = messages.StringField(1, repeated=True) - case_count = messages.IntegerField(2, variant=messages.Variant.INT32) - samples = messages.StringField(3, repeated=True) - sample_count = messages.IntegerField(4, variant=messages.Variant.INT32) - - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - rows, _, _, _ = self.query(request) - - case_barcodes = set() - sample_barcodes = [] - - if rows: - for row in rows: - case_barcodes.add(row['case_barcode']) - sample_barcodes.append(row['sample_barcode']) - case_barcodes = list(case_barcodes) - - if len(sample_barcodes) == 0: - raise endpoints.BadRequestException("No samples meet the specified parameters.") - - return self.CohortCasesSamplesList(cases=case_barcodes, - case_count=len(case_barcodes), - samples=sample_barcodes, - sample_count=len(sample_barcodes)) diff --git a/api_3/cohort_endpoint_helpers.py b/api_3/cohort_endpoint_helpers.py deleted file mode 100755 index 2406d732..00000000 --- a/api_3/cohort_endpoint_helpers.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from django.conf import settings -from protorpc import messages -import logging - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -BUILTIN_ENDPOINTS_PARAMETERS = [ - 'alt', - 'fields', - 'enum', - 'enumDescriptions', - 'key', - 'oauth_token', - 'prettyPrint', - 'quotaUser', - 'userIp' -] - - -def are_there_bad_keys(request): - ''' - Checks for unrecognized fields in an endpoint request - :param request: the request object from the endpoint - :return: boolean indicating True if bad (unrecognized) fields are present in the request - ''' - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - return unrecognized_param_dict != {} - - -def are_there_no_acceptable_keys(request): - """ - Checks for a lack of recognized fields in an endpoints request. Used in save_cohort and preview_cohort endpoints. - :param request: the request object from the endpoint - :return: boolean indicating True if there are no recognized fields in the request. - """ - param_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - } - return param_dict == {} - - -def construct_parameter_error_message(request, filter_required): - err_msg = '' - sorted_acceptable_keys = sorted([k.name for k in request.all_fields()], key=lambda s: s.lower()) - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - if unrecognized_param_dict: - bad_key_str = "'" + "', '".join(unrecognized_param_dict.keys()) + "'" - err_msg += "The following filters were not recognized: {}. ".format(bad_key_str) - if filter_required: - err_msg += "You must specify at least one of the following " \ - "case-sensitive filters: {}".format(sorted_acceptable_keys) - else: - err_msg += "Acceptable filters are: {}".format(sorted_acceptable_keys) - - return err_msg - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -def build_constructor_dict_for_message(message_class, row): - """ - Takes an instance of a message class and a dictionary of values from a database query - and first validates the values in the dictionary against the message class fields - and then returns a dictionary of all the validated key-value pairs in the database query. - This will only work if the headers in the database query have the same name as the names of - fields in the message class. - """ - constructor_dict = {} - metadata_item_dict = {field.name: field for field in message_class.all_fields()} - for name, field in metadata_item_dict.iteritems(): - if row.get(name) is not None: - try: - field.validate(row[name]) - constructor_dict[name] = row[name] - except messages.ValidationError, e: - constructor_dict[name] = None - logger.warn('{name}: {value} was not validated while constructing kwargs for {message_class}. Error: {e}' - .format(name=name, value=str(row[name]), message_class=str(message_class), e=e)) - else: - constructor_dict[name] = None - - return constructor_dict - - -def build_constructor_dicts_for_message(message_class, rows): - """ - Takes an instance of a message class and a dictionary of values from a database query - and first validates the values in the dictionary against the message class fields - and then returns a dictionary of all the validated key-value pairs in the database query. - This will only work if the headers in the database query have the same name as the names of - fields in the message class. - """ - constructor_dicts = [] - metadata_item_dict = {field.name: field for field in message_class.all_fields()} - - for row in rows: - constructor_dict = {} - for name, field in metadata_item_dict.iteritems(): - if row.get(name) is not None: - try: - field.validate(row[name]) - constructor_dict[name] = row[name] - except messages.ValidationError, e: - constructor_dict[name] = None - logger.warn('{name}: {value} was not validated while constructing kwargs for {message_class}. Error: {e}' - .format(name=name, value=str(row[name]), message_class=str(message_class), e=e)) - else: - constructor_dict[name] = None - - if len(constructor_dict): - constructor_dicts.append(constructor_dict) - - return constructor_dicts \ No newline at end of file diff --git a/api_3/data_access.py b/api_3/data_access.py deleted file mode 100755 index b9abc7b9..00000000 --- a/api_3/data_access.py +++ /dev/null @@ -1,542 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import json -import math -import traceback - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import BooleanField, EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.feature_value_types import ValueType, is_log_transformable -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_tcga_only, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess -from bq_data_access.utils import DurationLogged -from bq_data_access.data_access import FeatureIdQueryDescription - -from api_3.pairwise import PairwiseInputVector, Pairwise -from api_3.pairwise_api import PairwiseResults, PairwiseResultVector, PairwiseFilterMessage -from api_3.api_helpers import sql_connection - -from projects.models import Project - -import sys - -logger = logging.getLogger(__name__) - -VIZ_UNIT_DATADICTIONARY = { - 'BMI': 'kg/m^2', -} - -ISB_CGC_PROJECTS = { - 'list': [] -} - -# DUPLICATE METHOD -# Due to the way sql connections are done, it's easiest to duplicate this method and the static variable -# it creates. The original is in Cohorts/views, and all changes will happen there first. -# -# Generate the ISB_CGC_PROJECTS['list'] value set based on the get_isbcgc_project_set sproc -def fetch_isbcgc_project_set(): - try: - cursor = None - db = sql_connection() - if not ISB_CGC_PROJECTS['list'] or len(ISB_CGC_PROJECTS['list']) <= 0: - cursor = db.cursor() - cursor.execute("SELECT COUNT(SPECIFIC_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE SPECIFIC_NAME = 'get_isbcgc_project_set';") - # Only try to fetch the study set if the sproc exists - if cursor.fetchall()[0][0] > 0: - cursor.execute("CALL get_isbcgc_project_set();") - ISB_CGC_PROJECTS['list'] = [] - for row in cursor.fetchall(): - ISB_CGC_PROJECTS['list'].append(row[0]) - else: - # Otherwise just warn - logger.warn("[WARNING] Stored procedure get_isbcgc_project_set was not found!") - - return ISB_CGC_PROJECTS['list'] - except Exception as e: - logger.error(e) - logger.error(traceback.format_exc()) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - -def get_axis_units(xAttr, yAttr): - units = {'x': '', 'y': ''} - - checkUnits = {} - if xAttr is not None: - checkUnits[xAttr] = 'x' - if yAttr is not None: - checkUnits[yAttr] = 'y' - - for attr in checkUnits: - - if '_age' in attr or 'age_' in attr or 'year_' in attr: - units[checkUnits[attr]] = 'years' - elif '_days' in attr or 'days_' in attr: - units[checkUnits[attr]] = 'days' - elif 'percent' in attr: - units[checkUnits[attr]] = 'percent' - elif 'CNVR:' in attr: - units[checkUnits[attr]] = 'log(CN/2)' - elif 'RPPA:' in attr: - units[checkUnits[attr]] = 'protein expression' - elif 'METH:' in attr: - units[checkUnits[attr]] = 'beta value' - elif 'GEXP:' in attr or 'MIRN:' in attr or ('GNAB:' in attr and "num_mutations" in attr): - units[checkUnits[attr]] = 'count' - elif attr.split(':')[1] in VIZ_UNIT_DATADICTIONARY: - units[checkUnits[attr]] = VIZ_UNIT_DATADICTIONARY[attr.split(':')[1]] - - return units - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, required=True) - - -class DataPoint(Message): - patient_id = StringField(1) - sample_id = StringField(2) - aliquot_id = StringField(3) - value = StringField(4) - - -class DataPointList(Message): - type = EnumField(ValueType, 1) - items = MessageField(DataPoint, 2, repeated=True) - - -class PlotDataRequest(Message): - x_id = StringField(1, required=True) - y_id = StringField(2, required=False) - c_id = StringField(3, required=False) - log_transform = StringField(4, required=False) - cohort_id = IntegerField(5, repeated=True) - pairwise = BooleanField(6, required=False) - - -class PlotDataPointCohortMemberships(Message): - ids = IntegerField(1, repeated=True) - - -class PlotDataCohortInfo(Message): - id = IntegerField(1, required=True) - name = StringField(2, required=True) - -DATAPOINT_COHORT_THRESHOLD = 1 - - -class PlotDataPoint(Message): - sample_id = StringField(1) - case_id = StringField(2) - x = StringField(3) - y = StringField(4) - c = StringField(5) - cohort = IntegerField(6, repeated=True) - - -class PlotDataTypes(Message): - x = EnumField(ValueType, 1) - y = EnumField(ValueType, 2) - c = EnumField(ValueType, 3) - - -class PlotDataFeatureLabels(Message): - x = StringField(1) - y = StringField(2) - c = StringField(3) - - -class PlotDatapointCohortSet(Message): - datapoint_id = StringField(1, required=True) - - -class PlotDatapointCount(Message): - total_num_patients = IntegerField(1, required=True) - total_num_samples = IntegerField(2, required=True) - num_patients_w_xy = IntegerField(3, required=True) - num_samples_w_xy = IntegerField(4, required=True) - num_patients_wo_x = IntegerField(5, required=True) - num_samples_wo_x = IntegerField(6, required=True) - num_patients_wo_y = IntegerField(7, required=True) - num_samples_wo_y = IntegerField(8, required=True) - - -class PlotDataResponse(Message): - types = MessageField(PlotDataTypes, 1, required=True) - labels = MessageField(PlotDataFeatureLabels, 2, required=True) - items = MessageField(PlotDataPoint, 3, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - counts = MessageField(PlotDatapointCount, 5) - pairwise_result = MessageField(PairwiseResults, 6, required=False) - xUnits = StringField(7, required=False) - yUnits = StringField(8, required=False) - - -FeatureDataEndpointsAPI = endpoints_api(name='feature_data_api', version='v1', - description='Endpoints for feature data used by the web application.') - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class FeatureDataEndpoints(remote.Service): - def get_counts(self, data): - total_num_patients = [] - total_num_samples = [] - num_samples_w_xy = [] - num_patients_w_xy = [] - num_samples_wo_x = [] - num_samples_wo_y = [] - num_patients_wo_x = [] - num_patients_wo_y = [] - - result = {} - - for item in data: - total_num_samples.append(item['sample_id']) - total_num_patients.append(item['sample_id'][:12]) - - if item['x'] != 'NA' and item['y'] != 'NA': - num_samples_w_xy.append(item['sample_id']) - num_patients_w_xy.append(item['sample_id'][:12]) - else: - if item['x'] == 'NA': - num_samples_wo_x.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_x.append(item['sample_id'][:12]) - elif item['y'] == 'NA': - num_samples_wo_y.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_y.append(item['sample_id'][:12]) - - result['total_num_patients'] = len(set(total_num_patients)) - result['total_num_samples'] = len(set(total_num_samples)) - result['num_patients_w_xy'] = len(set(num_patients_w_xy)) - result['num_samples_w_xy'] = len(set(num_samples_w_xy)) - result['num_patients_wo_x'] = len(set(num_patients_wo_x)) - result['num_samples_wo_x'] = len(set(num_samples_wo_x)) - result['num_patients_wo_y'] = len(set(num_patients_wo_y)) - result['num_samples_wo_y'] = len(set(num_samples_wo_y)) - - return result - - # TODO refactor to separate module - @DurationLogged('PAIRWISE', 'GET') - def get_pairwise_result(self, feature_array): - # Format the feature vectors for pairwise - input_vectors = Pairwise.prepare_feature_vector(feature_array) - outputs = None - results = None - - try: - outputs = Pairwise.run_pairwise(input_vectors) - - if outputs is not None: - results = PairwiseResults(result_vectors=[], filter_messages=[]) - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - except Exception as e: - outputs = None - results = None - logger.error(traceback.format_exc()) - - return results - - @DurationLogged('FEATURE', 'VECTOR_MERGE') - def get_merged_dict_timed(self, vms): - return vms.get_merged_dict() - - # TODO refactor missing value logic out of this module - @DurationLogged('FEATURE', 'GET_VECTORS') - def get_merged_feature_vectors(self, x_id, y_id, c_id, cohort_id_array, logTransform, study_id_array): - """ - Fetches and merges data for two or three feature vectors (see parameter documentation below). - The vectors have to be an array of dictionaries, with each dictionary containing a 'value' field - (other fields are ignored): - [ - { - 'value': 0.5 - }, - { - 'value': 1.0 - } - ] - The merged result: - [ - { - 'patient_id': - 'x': - 'y': - 'c': - }, - { - 'patient_id': - 'x': - 'y': - 'c': - } - ... - ] - - :param x_id: Feature identifier for x-axis e.g. 'CLIN:age_at_diagnosis' - :param y_id: Feature identifier for y-axis. If None, values for 'y' in the response will be marked as missing. - :param c_id: Feature identifier for color-by. If None, values for 'c' in the response will be marked as missing. - :param cohort_id_array: Cohort identifier array. - - :return: PlotDataResponse - """ - - async_params = [FeatureIdQueryDescription(x_id, cohort_id_array, study_id_array)] - - c_type, c_vec = ValueType.STRING, [] - y_type, y_vec = ValueType.STRING, [] - - units = get_axis_units(x_id, y_id) - - if c_id is not None: - async_params.append(FeatureIdQueryDescription(c_id, cohort_id_array, study_id_array)) - if y_id is not None: - async_params.append(FeatureIdQueryDescription(y_id, cohort_id_array, study_id_array)) - - async_result = get_feature_vectors_tcga_only(async_params) - - if c_id is not None: - c_type, c_vec = async_result[c_id]['type'], async_result[c_id]['data'] - if y_id is not None: - y_type, y_vec = async_result[y_id]['type'], async_result[y_id]['data'] - if logTransform is not None and logTransform['y'] and y_vec and is_log_transformable(y_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum y-value - ''' - yvals = [] - for yd in y_vec: - if 'value' in yd and yd['value'] is not None and yd['value'] != "NA" and yd['value'] != "None": - yvals.append(float(yd['value'])) - y_min = min(yvals) - ''' - for ydata in y_vec: - if 'value' in ydata and ydata['value'] is not None and ydata['value'] != "NA" and ydata['value'] != "None": - if float(ydata['value']) < 0: - ydata['value'] = "NA" - elif logTransform['yBase'] == 10: - ydata['value'] = str(math.log10((float(ydata['value']) + 1))) - elif logTransform['yBase'] == 'e': - ydata['value'] = str(math.log((float(ydata['value']) + 1))) - elif type(logTransform['yBase']) is int: - ydata['value'] = str(math.log((float(ydata['value']) + 1), logTransform['yBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - x_type, x_vec = async_result[x_id]['type'], async_result[x_id]['data'] - - if logTransform is not None and logTransform['x'] and x_vec and is_log_transformable(x_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum x-value - ''' - xvals = [] - for xd in x_vec: - if 'value' in xd and xd['value'] is not None and xd['value'] != "NA" and xd['value'] != "None": - xvals.append(float(xd['value'])) - x_min = min(xvals) - ''' - - for xdata in x_vec: - if 'value' in xdata and xdata['value'] is not None and xdata['value'] != "NA" and xdata['value'] != "None": - if float(xdata['value']) < 0: - xdata['value'] = "NA" - elif logTransform['xBase'] == 10: - xdata['value'] = str(math.log10((float(xdata['value']) + 1))) - elif logTransform['xBase'] == 'e': - xdata['value'] = str(math.log((float(xdata['value']) + 1))) - elif type(logTransform['xBase']) is int: - xdata['value'] = str(math.log((float(xdata['value']) + 1), logTransform['xBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', ['x', 'y', 'c']) # changed so that it plots per sample not patient - vms.add_dict_array(x_vec, 'x', 'value') - vms.add_dict_array(y_vec, 'y', 'value') - vms.add_dict_array(c_vec, 'c', 'value') - merged = self.get_merged_dict_timed(vms) - - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(id=item['id'], name=item['name'])) - - items = [] - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - - if len(cohort_set) >= DATAPOINT_COHORT_THRESHOLD: - value_bundle['cohort'] = cohort_set - - items.append(PlotDataPoint(**value_bundle)) - - counts = self.get_counts(merged) - count_message = PlotDatapointCount(**counts) - - type_message = PlotDataTypes(x=x_type, y=y_type, c=c_type) - - # TODO assign label for y if y_id is None, as in that case the y-field will be missing from the response - label_message = PlotDataFeatureLabels(x=x_id, y=y_id, c=c_id) - - # TODO Refactor pairwise call to separate function - # Include pairwise results - input_vectors = [PairwiseInputVector(x_id, x_type, x_vec)] - if c_id is not None: - input_vectors.append(PairwiseInputVector(c_id, c_type, c_vec)) - if y_id is not None: - input_vectors.append(PairwiseInputVector(y_id, y_type, y_vec)) - - - pairwise_result = None - - if len(input_vectors) > 1: - pairwise_result = self.get_pairwise_result(input_vectors) - - if pairwise_result is None: - logger.warn("[WARNING] Pairwise results not included in returned object") - - return PlotDataResponse(types=type_message, labels=label_message, items=items, - cohort_set=cohort_info_obj_array, - counts=count_message, pairwise_result=pairwise_result, xUnits=units['x'], yUnits=units['y']) - - def get_feature_id_validity_for_array(self, feature_id_array): - """ - For each feature identifier in an array, check whether or not the identifier is - valid. - - Args: - feature_id_array: - - Returns: - Array of tuples - (feature identifier, ) - """ - result = [] - for feature_id in feature_id_array: - result.append((feature_id, is_valid_feature_identifier(feature_id))) - - return result - - @endpoints_method(PlotDataRequest, PlotDataResponse, - path='feature_data_plot', http_method='GET', name='feature_access.getFeatureDataForPlot') - def data_access_for_plot(self, request): - """ Used by the web application.""" - try: - x_id = request.x_id - y_id = request.y_id - c_id = request.c_id - logTransform = json.loads(request.log_transform) - cohort_id_array = request.cohort_id - - # Check that all requested feature identifiers are valid. Do not check for y_id if it is not - # supplied in the request. - feature_ids_to_check = [x_id] - if c_id is not None: - feature_ids_to_check.append(c_id) - if y_id is not None: - feature_ids_to_check.append(y_id) - - valid_features = self.get_feature_id_validity_for_array(feature_ids_to_check) - - for feature_id, is_valid in valid_features: - logging.info((feature_id, is_valid)) - if not is_valid: - logging.error("Invalid internal feature ID '{}'".format(feature_id)) - raise NotFoundException() - - # Get the project IDs these cohorts' samples come from - cohort_vals = () - cohort_params = "" - - for cohort in cohort_id_array: - cohort_params += "%s," - cohort_vals += (cohort,) - - cohort_params = cohort_params[:-1] - - db = sql_connection() - cursor = db.cursor() - - tcga_studies = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN ("+cohort_params+");",cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_study_ids = [] - unconfirmed_study_ids = [] - - for row in cursor.fetchall(): - if row[0] in tcga_studies: - if row[0] not in confirmed_study_ids: - confirmed_study_ids.append(row[0]) - elif row[0] not in unconfirmed_study_ids: - unconfirmed_study_ids.append(row[0]) - - if len(unconfirmed_study_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_study_ids) - - for project in projects: - if project.get_my_root_and_depth()['root'] in tcga_studies: - confirmed_study_ids.append(project.id) - - return self.get_merged_feature_vectors(x_id, y_id, c_id, cohort_id_array, logTransform, confirmed_study_ids) - except NotFoundException as nfe: - # Pass through NotFoundException so that it is not handled as Exception below. - raise nfe - except Exception as e: - logger.exception(e) - raise InternalServerErrorException() - diff --git a/api_3/feature_access.py b/api_3/feature_access.py deleted file mode 100755 index 1e987916..00000000 --- a/api_3/feature_access.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import BadRequestException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import Message, MessageField, StringField - -from bq_data_access.feature_search.common import BackendException, EmptyQueryException, InvalidFieldException, InvalidDataTypeException -from bq_data_access.gexp_data import GEXP_FEATURE_TYPE -from bq_data_access.clinical_data import CLINICAL_FEATURE_TYPE -from bq_data_access.methylation_data import METH_FEATURE_TYPE -from bq_data_access.copynumber_data import CNVR_FEATURE_TYPE -from bq_data_access.protein_data import RPPA_FEATURE_TYPE -from bq_data_access.mirna_data import MIRN_FEATURE_TYPE -from bq_data_access.gnab_data import GNAB_FEATURE_TYPE -from bq_data_access.feature_search.gexp_searcher import GEXPSearcher -from bq_data_access.feature_search.clinical_searcher import ClinicalSearcher -from bq_data_access.feature_search.methylation_searcher import METHSearcher -from bq_data_access.feature_search.copynumber_search import CNVRSearcher -from bq_data_access.feature_search.protein import RPPASearcher -from bq_data_access.feature_search.microrna_searcher import MIRNSearcher -from bq_data_access.feature_search.gnab_searcher import GNABSearcher - -class ClinicalFeatureType(Message): - feature_type = StringField(1) - gene = StringField(2) - label = StringField(3) - internal_id = StringField(4) - -class FeatureTypesRequest(Message): - keyword = StringField(1, required=True) - -class FeatureDataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = StringField(2, required=True) - -class FeatureTypeSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=False) - gene_name = StringField(3, required=False) - platform = StringField(4, required=False) - center = StringField(5, required=False) - protein_name = StringField(6, required=False) - value_field = StringField(7, required=False) - probe_name = StringField(8, required=False) - relation_to_gene = StringField(9, required=False) - relation_to_island = StringField(10, required=False) - mirna_name = StringField(11, required=False) - -class FeatureSearchResult(Message): - feature_type = StringField(1) - internal_feature_id = StringField(2) - label = StringField(3) - type = StringField(4) - -class FeatureTypeList(Message): - items = MessageField(FeatureSearchResult, 1, repeated=True) - -class FeatureTypeFieldSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=True) - field = StringField(3, required=True) - -class FeatureFieldSearchResult(Message): - values = StringField(1, repeated=True) - - -class FeatureDefinitionSearcherFactory(object): - @classmethod - def build_from_datatype(cls, datatype): - if datatype == CLINICAL_FEATURE_TYPE: - return ClinicalSearcher() - elif datatype == GEXP_FEATURE_TYPE: - return GEXPSearcher() - elif datatype == METH_FEATURE_TYPE: - return METHSearcher() - elif datatype == CNVR_FEATURE_TYPE: - return CNVRSearcher() - elif datatype == RPPA_FEATURE_TYPE: - return RPPASearcher() - elif datatype == MIRN_FEATURE_TYPE: - return MIRNSearcher() - elif datatype == GNAB_FEATURE_TYPE: - return GNABSearcher() - #TODO build a full search on all features - #elif datatype == ALL: - # return FullSearcher() - raise InvalidDataTypeException("Invalid datatype '{datatype}'".format(datatype=datatype)) - -FeatureAccessEndpointsAPI = endpoints_api(name='feature_type_api', version='v1', - description='Endpoints used by the web application to return features.') -@FeatureAccessEndpointsAPI.api_class(resource_name='feature_type_endpoints') -class FeatureAccessEndpoints(remote.Service): - @endpoints_method(FeatureTypeSearchRequest, FeatureTypeList, - path='feature_search', http_method='GET', name='feature_access.FeatureSearch') - def feature_search(self, request): - """ Used by the web application.""" - try: - datatype = request.datatype - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - parameters = {} - for message in request.all_fields(): - field_name = message.name - if field_name != 'datatype': - value = request.get_assigned_value(field_name) - if value is not None: - parameters[field_name] = value - - result = searcher.search(parameters) - items = [] - fields = ['label', 'internal_feature_id', 'feature_type'] - for row in result: - obj = {key: row[key] for key in fields} - if obj['feature_type'] == 'CLIN': - obj['type'] = row['type'] - items.append(obj) - - return FeatureTypeList(items=items) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except EmptyQueryException as e: - logging.error("Empty query: %s", str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error("Invalid field: %s", str(e)) - raise BadRequestException(str(e)) - except BackendException: - logging.exception("feature_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - - @endpoints_method(FeatureTypeFieldSearchRequest, FeatureFieldSearchResult, - path='feature_field_search', http_method='GET', name='feature_access.getFeatureFieldSearch') - def feature_field_search(self, request): - """ Used by the web application.""" - try: - datatype, keyword, field = request.datatype, request.keyword, request.field - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - result = searcher.field_value_search(keyword, field) - return FeatureFieldSearchResult(values=result) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error(str(e)) - raise BadRequestException() - except BackendException: - logging.exception("feature_field_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/isb_cgc_api/__init__.py b/api_3/isb_cgc_api/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api/cohort_file_manifest.py b/api_3/isb_cgc_api/cohort_file_manifest.py deleted file mode 100644 index 4f2244c4..00000000 --- a/api_3/isb_cgc_api/cohort_file_manifest.py +++ /dev/null @@ -1,198 +0,0 @@ -""" - -Copyright 2018, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -import logging -import django -from protorpc import remote, messages - -from django.core.signals import request_finished -from django.contrib.auth.models import User as Django_User -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.conf import settings - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints - -from cohorts.file_helpers import cohort_files -from cohorts.models import Cohort_Perms -from accounts.sa_utils import auth_dataset_whitelists_for_user - -BASE_URL = settings.BASE_URL - -logger = logging.getLogger(__name__) - - -class FileDetail(messages.Message): - program = messages.StringField(1, required=True) - case_barcode = messages.StringField(2, required=True) - file_path = messages.StringField(3, required=True) - file_gdc_uuid = messages.StringField(4) - disease_code = messages.StringField(5, required=True) - experimental_strategy = messages.StringField(6) - platform = messages.StringField(7) - data_category = messages.StringField(8) - data_type = messages.StringField(9) - data_format = messages.StringField(10) - access = messages.StringField(11) - case_gdc_uuid = messages.StringField(12) - project_short_name = messages.StringField(13) - - -class FileManifest(messages.Message): - files = messages.MessageField(FileDetail, 1, repeated=True) - total_file_count = messages.IntegerField(2, variant=messages.Variant.INT32) - files_retrieved = messages.IntegerField(3, variant=messages.Variant.INT32) - - -class FileManifestFilters(messages.Message): - program = messages.StringField(1, repeated=True) - disease_code = messages.StringField(2, repeated=True) - experimental_strategy = messages.StringField(3, repeated=True) - platform = messages.StringField(4, repeated=True) - data_category = messages.StringField(5, repeated=True) - data_type = messages.StringField(6, repeated=True) - data_format = messages.StringField(7, repeated=True) - project_short_name = messages.StringField(8, repeated=True) - - -class CohortFileManifest(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - fetch_count=messages.IntegerField(2), - offset=messages.IntegerField(3), - genomic_build=messages.StringField(4), - do_filter_count=messages.BooleanField(6) - ) - - POST_RESOURCE = endpoints.ResourceContainer( - FileManifestFilters, - cohort_id=messages.IntegerField(1, required=True), - fetch_count=messages.IntegerField(2), - offset=messages.IntegerField(3), - genomic_build=messages.StringField(4), - do_filter_count=messages.BooleanField(6) - ) - - def validate_user(self, cohort_id): - user_email = None - user = None - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.". - format(BASE_URL)) - django.setup() - try: - user = Django_User.objects.get(email=user_email) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) - except ObjectDoesNotExist as e: - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}.".format(user_email, cohort_id) - logger.exception(e) - raise endpoints.UnauthorizedException(err_msg) - finally: - request_finished.send(self) - return user - - def file_manifest(self, request): - cohort_id = request.get_assigned_value('cohort_id') - file_manifest = None - - try: - user = self.validate_user(cohort_id) - has_access = auth_dataset_whitelists_for_user(user.id) - - params = {} - - if request.get_assigned_value('offset'): - params['offset'] = request.get_assigned_value('offset') - - if request.get_assigned_value('fetch_count'): - params['limit'] = request.get_assigned_value('fetch_count') - else: - params['limit'] = settings.MAX_FILE_LIST_REQUEST - - if request.get_assigned_value('genomic_build'): - params['build'] = request.get_assigned_value('genomic_build').upper() - else: - params['build'] = 'HG19' - - params['access'] = has_access - - inc_filters = { - filter.name: request.get_assigned_value(filter.name) - for filter in request.all_fields() - if request.get_assigned_value(filter.name) and filter.name not in ['cohort_id','fetch_count','offset','genomic_build'] - } - - response = cohort_files(cohort_id, user=user, inc_filters=inc_filters, **params) - - file_manifest = FileManifest() - files = [] - - for file in response['file_list']: - files.append(FileDetail( - program=file['program'], - case_barcode=file['case'], - case_gdc_uuid=file['case_gdc_id'], - file_path=file['cloudstorage_location'], - file_gdc_uuid=file['file_gdc_id'], - disease_code=file['disease_code'], - project_short_name=file['project_short_name'], - experimental_strategy=file['exp_strat'], - platform=file['platform'], - data_category=file['datacat'], - data_type=file['datatype'], - data_format=file['dataformat'], - access=file['access'] - )) - - file_manifest.files = files - file_manifest.total_file_count = response['total_file_count'] - file_manifest.files_retrieved = len(files) - - except Exception as e: - logger.exception(e) - raise endpoints.InternalServerErrorException("There was an error while processing your request.") - - return file_manifest - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortFileManifestAPI(CohortFileManifest): - - @endpoints.method(CohortFileManifest.GET_RESOURCE, FileManifest, http_method='GET', path='cohorts/{cohort_id}/file_manifest') - def file_manifest(self, request): - """ - Takes a cohort id as a required parameter and returns a manifest of all files associated - with all the samples in that cohort, as well as their metadata, up to a default limit of - 50,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - return super(CohortFileManifestAPI, self).file_manifest(request) - - @endpoints.method(CohortFileManifest.POST_RESOURCE, FileManifest, http_method='POST', path='cohorts/{cohort_id}/file_manifest') - def file_manifest_filtered(self, request): - """ - Takes a cohort id as a required parameter and a filter set as a required data payload, - and returns a manifest of all files associated with all the samples in that cohort, as - well as their metadata, up to a default limit of 50,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - return super(CohortFileManifestAPI, self).file_manifest(request) diff --git a/api_3/isb_cgc_api/cohort_get_list_helper.py b/api_3/isb_cgc_api/cohort_get_list_helper.py deleted file mode 100755 index 5a0e4f66..00000000 --- a/api_3/isb_cgc_api/cohort_get_list_helper.py +++ /dev/null @@ -1,408 +0,0 @@ -''' -Created on Apr 5, 2017 - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import django -import endpoints -import logging -import MySQLdb -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.cohort_endpoint_helpers import FilterDetails -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - -class CohortDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - case_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - cases = messages.StringField(13, repeated=True) - samples = messages.StringField(14, repeated=True) - -class CohortListDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - case_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - -class CohortDetailsList(messages.Message): - items = messages.MessageField(CohortListDetails, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - -class CohortsGetListQueryBuilder(object): - - def build_cohort_query(self, query_dict): - """ - Builds the query that will select cohort id, name, last_date_saved, - perms, comments, source type, and source notes - :param query_dict: should contain {'cohorts_cohort_perms.user_id': user_id, 'cohorts_cohort.active': unicode('1')} - :return: query_str, query_tuple - """ - query_str = 'SELECT cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'cohorts_cohort_comments.content AS comments, ' \ - 'cohorts_source.type AS source_type, ' \ - 'cohorts_source.notes AS source_notes ' \ - 'FROM cohorts_cohort_perms ' \ - 'JOIN cohorts_cohort ' \ - 'ON cohorts_cohort.id=cohorts_cohort_perms.cohort_id ' \ - 'JOIN auth_user ' \ - 'ON auth_user.id=cohorts_cohort_perms.user_id ' \ - 'LEFT JOIN cohorts_cohort_comments ' \ - 'ON cohorts_cohort_comments.user_id=cohorts_cohort_perms.user_id ' \ - 'AND cohorts_cohort_comments.cohort_id=cohorts_cohort.id ' \ - 'LEFT JOIN cohorts_source ' \ - 'ON cohorts_source.cohort_id=cohorts_cohort_perms.cohort_id ' - - query_tuple = () - if query_dict: - query_str += ' WHERE ' + '=%s and '.join(key for key in query_dict.keys()) + '=%s ' - query_tuple = tuple(value for value in query_dict.values()) - - query_str += 'GROUP BY ' \ - 'cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'comments, ' \ - 'source_type, ' \ - 'source_notes ' - - return query_str, query_tuple - - def build_filter_query(self, filter_query_dict): - """ - Builds the query that selects the filter name and value for a particular cohort - :param filter_query_dict: should be {'cohorts_filters.resulting_cohort_id:': id} - :return: filter_query_str, filter_query_tuple - """ - filter_query_str = 'SELECT name, value ' \ - 'FROM cohorts_filters ' - - filter_query_str += ' WHERE ' + '=%s AND '.join(key for key in filter_query_dict.keys()) + '=%s ' - filter_query_tuple = tuple(value for value in filter_query_dict.values()) - - return filter_query_str, filter_query_tuple - - def build_parent_query(self, parent_query_dict): - """ - Builds the query that selects parent_ids for a particular cohort - :param parent_query_dict: should be {'cohort_id': str(row['id'])} - :return: parent_query_str, parent_query_tuple - """ - parent_query_str = 'SELECT parent_id ' \ - 'FROM cohorts_source ' - parent_query_str += ' WHERE ' + '=%s AND '.join(key for key in parent_query_dict.keys()) + '=%s ' - parent_query_tuple = tuple(value for value in parent_query_dict.values()) - - return parent_query_str, parent_query_tuple - - def build_cases_query(self, case_query_dict): - """ - Builds the query that selects the case count for a particular cohort - :param case_query_dict: should be {'cohort_id': str(row['id])} - :return: case_query_str, case_query_tuple - """ - cases_query_str = 'SELECT cohort_id id, case_barcode ' \ - 'FROM cohorts_samples ' - - cases_query_str += ' WHERE ' + '=%s AND '.join(key for key in case_query_dict.keys()) + '=%s ' - case_query_tuple = tuple(value for value in case_query_dict.values()) - - return cases_query_str, case_query_tuple - - def build_samples_query(self, sample_query_dict): - """ - Builds the query that selects the sample count for a particular cohort - :param sample_query_dict: should be {'cohort_id': str(row['id])} - :return: sample_query_str, sample_query_tuple - """ - samples_query_str = 'SELECT sample_barcode, case_barcode ' \ - 'FROM cohorts_samples ' - - samples_query_str += ' WHERE ' + '=%s AND '.join(key for key in sample_query_dict.keys()) + '=%s ' - sample_query_tuple = tuple(value for value in sample_query_dict.values()) - - return samples_query_str, sample_query_tuple - -class CohortsGetListMessageBuilder(object): - def make_filter_details_from_cursor(self, filter_cursor_dict): - """ - Returns list of FilterDetails from a dictionary of results - from a filter query. - """ - filter_data = [] - for filter_row in filter_cursor_dict: - filter_data.append(FilterDetails( - name=str(filter_row['name']), - value=str(filter_row['value']) - )) - - if len(filter_data) == 0: - filter_data.append(FilterDetails( - name="None", - value="None" - )) - - return filter_data - - def make_parent_id_list_from_cursor(self, parent_cursor_dict, row): - """ - Returns list of parent_id's from a dictionary of results - from a parent id query. - """ - parent_id_data = [str(p_row['parent_id']) for p_row in parent_cursor_dict if row.get('parent_id')] - if len(parent_id_data) == 0: - parent_id_data.append("None") - - return parent_id_data - -class CohortsGetListAPI(remote.Service): - def validate_user(self): - user_email = None - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.". - format(BASE_URL)) - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned) as e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - return user_id,user_email - - def execute_getlist_query(self, param_map): - query_str, query_tuple = CohortsGetListQueryBuilder().build_cohort_query(param_map) - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - return cursor, db - - def get_cohort_details(self, cursor, row): - # get filters for each cohort - filter_query_str, filter_query_tuple = CohortsGetListQueryBuilder().build_filter_query( - { - 'cohorts_filters.resulting_cohort_id':str(row['id']) - } - ) - cursor.execute(filter_query_str, filter_query_tuple) - filter_data = CohortsGetListMessageBuilder().make_filter_details_from_cursor(cursor.fetchall()) - - # getting the parent_id's for each cohort is a separate query - # since a single cohort may have multiple parent cohorts - parent_query_str, parent_query_tuple = CohortsGetListQueryBuilder().build_parent_query( - { - 'cohort_id':str(row['id']) - } - ) - cursor.execute(parent_query_str, parent_query_tuple) - parent_id_data = CohortsGetListMessageBuilder().make_parent_id_list_from_cursor( - cursor.fetchall(), row) - - # get number of cases for each cohort - case_query_str, case_query_tuple = CohortsGetListQueryBuilder().build_cases_query( - { - 'cohort_id':str(row['id']) - } - ) - cursor.execute(case_query_str, case_query_tuple) - case_list = set() - for row in cursor.fetchall(): - case_list.add(row['case_barcode']) - case_count = len(case_list) - - # get number of samples for each cohort - sample_query_str, sample_query_tuple = CohortsGetListQueryBuilder().build_samples_query( - { - 'cohort_id':str(row['id']) - } - ) - cursor.execute(sample_query_str, sample_query_tuple) - sample_list = [] - for row in cursor.fetchall(): - sample_list.append(row['sample_barcode']) - sample_count = len(sample_list) - - return parent_id_data, filter_data, case_list, case_count, sample_list, sample_count - -class CohortsGetHelper(CohortsGetListAPI): - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - def get(self, request): - """ - Returns information about a specific cohort the user has READER or OWNER permission on - when given a cohort ID. Authentication is required. - """ - user_id, user_email = self.validate_user() - - cursor = None - db = None - cohort_id = request.get_assigned_value('cohort_id') - param_map = { - 'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1'), - 'cohorts_cohort.id': cohort_id - } - try: - cursor, db = self.execute_getlist_query(param_map) - row = cursor.fetchone() - - if row is None: - raise endpoints.NotFoundException( - "Cohort {id} not found. Either it never existed, it was deleted, " - "or {user_email} does not have permission to view it.".format( - id=cohort_id, user_email=user_email)) - - parent_id_data, filter_data, case_list, _, sample_list, _ = self.get_cohort_details(cursor, row) - - temp_list = [] - if len(sample_list) == 0: - sample_list = ["None"] - else: - temp_list = [] - for sample in sample_list: - if sample is not None: - temp_list += [sample] - sample_list = temp_list - - if len(case_list) == 0: - case_list = ["None"] - else: - temp_list = [] - for case in case_list: - if case is not None: - temp_list += [case] - case_list = temp_list - - return CohortDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - case_count=len(case_list), - sample_count=len(sample_list), - cases=case_list, - samples=sample_list - ) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "Cohort {} for user {} not found. {}: {}".format(cohort_id, user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - -class CohortsListHelper(CohortsGetListAPI): - def list(self, unused_request): - """ - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. - """ - user_id, user_email = self.validate_user() - - param_map = { - 'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1') - } - db = None - cursor = None - try: - cursor, db = self.execute_getlist_query(param_map) - data = [] - for row in cursor.fetchall(): - parent_id_data, filter_data, _, case_count, _, sample_count = self.get_cohort_details(cursor, row) - - data.append( - CohortListDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - case_count=case_count, - sample_count=sample_count - ) - ) - - if len(data) == 0: - raise endpoints.NotFoundException("{} has no active cohorts.".format(user_email)) - - return CohortDetailsList(items=data, count=len(data)) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "User {}'s cohorts not found. {}: {}".format(user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py b/api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py deleted file mode 100755 index 26fc5df1..00000000 --- a/api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py +++ /dev/null @@ -1,68 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, CohortsCloudStorageFilePathsHelper - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsCloudStorageFilePathsAPI(CohortsCloudStorageFilePathsHelper): - - def build_program_query(self, final_query_str, query_tuple, program, param_map, build): - query_str = 'SELECT md.file_name_key, md.access '\ - 'FROM {}_metadata_data_{} md '.format(program, build) - query_str += 'JOIN cohorts_samples cs ON md.sample_barcode=cs.sample_barcode WHERE cs.cohort_id=%s ' - query_tuple += [param_map['cohort_id']] - query_str += 'AND file_name_key != "" AND file_name_key is not null ' - for field, value in param_map.iteritems(): - if field not in ['limit', 'cohort_id', 'sample_barcode', 'genomic_build'] and value: - query_str += ' and md.{}=%s '.format(field) - query_tuple += [value] - - query_str += ' GROUP BY md.file_name_key, md.access ' - if 0 < len(final_query_str): - final_query_str += ' UNION ' - final_query_str += query_str - return final_query_str, query_tuple - - def build_query(self, param_map, program): - builds = self.get_genomic_builds(param_map, '') - final_query_str = '' - query_tuple = [] - for program in ('CCLE', 'TARGET', 'TCGA'): - for build in builds: - if program == 'CCLE' and build != 'HG19': - continue - final_query_str, query_tuple = self.build_program_query(final_query_str, query_tuple, program, param_map, build) - - if 'limit' in param_map and param_map['limit']: - final_query_str += ' LIMIT %s' - query_tuple += [param_map['limit']] - else: - final_query_str += ' LIMIT 10000' - return final_query_str, query_tuple - - @endpoints.method(CohortsCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, http_method='GET', - path='cohorts/{cohort_id}/cloud_storage_file_paths') - def cloud_storage_file_paths(self, request): - """ - Takes a cohort id as a required parameter and returns cloud storage paths to files - associated with all the samples in that cohort, up to a default limit of 10,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - return super(CohortsCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request) diff --git a/api_3/isb_cgc_api/cohorts_delete.py b/api_3/isb_cgc_api/cohorts_delete.py deleted file mode 100755 index 3cc3a2c3..00000000 --- a/api_3/isb_cgc_api/cohorts_delete.py +++ /dev/null @@ -1,85 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import django -import endpoints -import logging - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - -class ReturnJSON(messages.Message): - message = messages.StringField(1) - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsDeleteAPI(remote.Service): - DELETE_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - @endpoints.method(DELETE_RESOURCE, ReturnJSON, http_method='DELETE', path='cohorts/{cohort_id}') - def delete(self, request): - """ - Deletes a cohort. User must have owner permissions on the cohort. - """ - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application.".format(BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_to_deactivate = Django_Cohort.objects.get(id=cohort_id) - if cohort_to_deactivate.active is True: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - if cohort_perm.perm == 'OWNER': - cohort_to_deactivate.active = False - cohort_to_deactivate.save() - return_message = 'Cohort %d successfully deleted.' % cohort_id - else: - return_message = 'You do not have owner permission on cohort %d.' % cohort_id - else: - return_message = "Cohort %d was already deleted." % cohort_id - return ReturnJSON(message=return_message) - - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Either cohort %d does not have an entry in the database " - "or you do not have owner or reader permissions on this cohort." % cohort_id) - finally: - request_finished.send(self) diff --git a/api_3/isb_cgc_api/cohorts_get.py b/api_3/isb_cgc_api/cohorts_get.py deleted file mode 100755 index 0c3464df..00000000 --- a/api_3/isb_cgc_api/cohorts_get.py +++ /dev/null @@ -1,30 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints -from api_3.isb_cgc_api.cohort_get_list_helper import CohortDetails, CohortsGetHelper - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsGetAPI(CohortsGetHelper): - @endpoints.method(CohortsGetHelper.GET_RESOURCE, CohortDetails, http_method='GET', path='cohorts/{cohort_id}') - def get(self, request): - """ - Returns information about a specific cohort the user has READER or OWNER permission on - when given a cohort ID. Authentication is required. - """ - return super(CohortsGetAPI, self).get(request) diff --git a/api_3/isb_cgc_api/cohorts_list.py b/api_3/isb_cgc_api/cohorts_list.py deleted file mode 100755 index 04078bd2..00000000 --- a/api_3/isb_cgc_api/cohorts_list.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from protorpc import message_types - -from api_3.isb_cgc_api.cohort_get_list_helper import CohortDetailsList, CohortsListHelper -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsListAPI(CohortsListHelper): - @endpoints.method(message_types.VoidMessage, CohortDetailsList, http_method='GET', path='cohorts') - def list(self, unused_request): - """ - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. - """ - return super(CohortsListAPI, self).list(unused_request) diff --git a/api_3/isb_cgc_api/files_get_file_paths.py b/api_3/isb_cgc_api/files_get_file_paths.py deleted file mode 100755 index b6d94d1f..00000000 --- a/api_3/isb_cgc_api/files_get_file_paths.py +++ /dev/null @@ -1,68 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import remote, messages - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints - -from api_3.api_helpers import sql_connection - - -@ISB_CGC_Endpoints.api_class(resource_name='files') -class FilesGetPath(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer(file_uuids=messages.StringField(1, repeated=True)) - - class FilePaths(messages.Message): - paths = messages.StringField(1, repeated=True) - - @endpoints.method(GET_RESOURCE, FilePaths, http_method='GET', path='file_paths') - def get(self, request): - """ - from the list of file gdc UUIDs, returns the google cloud storage file paths for those UUIDs. the returned - filepaths will be in the same order as the passed in UUIDs - """ - uuids = request.get_assigned_value('file_uuids') - in_clause = '' - params = [] - for uuid in uuids: - in_clause += '%s, ' - params += ['{}'.format(uuid)] - in_clause = in_clause[:-2] - - uuid2paths = {} - sql = 'select file_gdc_id, file_name_key from {}_metadata_data_{} where file_gdc_id in ({}) order by 1, 2' - programs = ['CCLE', 'TARGET', 'TCGA'] - db = sql_connection() - for program in programs: - if 'CCLE' == program: - builds = ['HG19'] - else: - builds = ['HG19', 'HG38'] - - for build in builds: - try: - cursor = db.cursor() - cursor.execute(sql.format(program, build, in_clause), params) - for row in cursor: - paths = uuid2paths.setdefault(row[0], []) - paths += [row[1]] - except Exception as e: - print 'problem executing sql({}):\n\t{}\n\t{}'.format(e, sql, params) - raise - filepaths = FilesGetPath.FilePaths() - filepaths.paths = [item for sublist in uuid2paths.values() for item in sublist] - return filepaths diff --git a/api_3/isb_cgc_api/isb_cgc_api_helpers.py b/api_3/isb_cgc_api/isb_cgc_api_helpers.py deleted file mode 100755 index 6ce7a4af..00000000 --- a/api_3/isb_cgc_api/isb_cgc_api_helpers.py +++ /dev/null @@ -1,27 +0,0 @@ -""" - -Copyright 2018, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_Endpoints = endpoints.api(name='isb_cgc_api', version='v3', - description="Get information about cohorts for ISB-CGC. List, get, delete cohorts, and retrieve or export their file manifests.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC API") diff --git a/api_3/isb_cgc_api_CCLE/__init__.py b/api_3/isb_cgc_api_CCLE/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json b/api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json deleted file mode 100755 index 1c8cef23..00000000 --- a/api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "Common": { - "disease_code": [ - "BLCA", - "BRCA", - "CESC", - "COAD", - "DLBC", - "ESCA", - "HNSC", - "KIRC", - "LCLL", - "LGG", - "LIHC", - "LUSC", - "MESO", - "MM", - "OV", - "PAAD", - "PRAD", - "SARC", - "SKCM", - "STAD", - "THCA", - "UCEC" - ], - "endpoint_type": [ - "legacy" - ], - "program_name": [ - "CCLE" - ], - "project_short_name": [ - "CCLE-BLCA", - "CCLE-BRCA", - "CCLE-CESC", - "CCLE-COAD", - "CCLE-DLBC", - "CCLE-ESCA", - "CCLE-HNSC", - "CCLE-KIRC", - "CCLE-LCLL", - "CCLE-LGG", - "CCLE-LIHC", - "CCLE-LUSC", - "CCLE-MESO", - "CCLE-MM", - "CCLE-OV", - "CCLE-PAAD", - "CCLE-PRAD", - "CCLE-SARC", - "CCLE-SKCM", - "CCLE-STAD", - "CCLE-THCA", - "CCLE-UCEC" - ] - }, - "Clinical": { - "gender": [ - "Female", - "Male", - "Undefined" - ], - "histology": [ - "carcinoid-endocrine_tumour", - "carcinoma", - "chondrosarcoma", - "Ewings_sarcoma-peripheral_primitive_neuroectodermal_tumour", - "fibrosarcoma", - "giant_cell_tumour", - "glioma", - "haematopoietic_neoplasm", - "leiomyosarcoma", - "lymphoid_neoplasm", - "malignant_fibrous_histiocytoma-pleomorphic_sarcoma", - "malignant_melanoma", - "mesothelioma", - "neuroblastoma", - "osteosarcoma", - "other", - "primitive_neuroectodermal_tumour-medulloblastoma", - "rhabdoid_tumour", - "rhabdomyosarcoma", - "sarcoma", - "sex_cord-stromal_tumour" - ], - "hist_subtype": [ - "acute_lymphoblastic_B_cell_leukaemia", - "acute_lymphoblastic_T_cell_leukaemia", - "acute_myeloid_leukaemia", - "adenocarcinoma", - "adult_T_cell_lymphoma-leukaemia", - "alveolar", - "anaplastic_carcinoma", - "anaplastic_large_cell_lymphoma", - "astrocytoma", - "astrocytoma_Grade_III", - "astrocytoma_Grade_IV", - "blast_phase_chronic_myeloid_leukaemia", - "Brenner_tumour", - "bronchioloalveolar_adenocarcinoma", - "Burkitt_lymphoma", - "B_cell_lymphoma_unspecified", - "carcinosarcoma-malignant_mesodermal_mixed_tumour", - "chronic_lymphocytic_leukaemia-small_lymphocytic_lymphoma", - "chronic_myeloid_leukaemia", - "clear_cell_carcinoma", - "clear_cell_renal_cell_carcinoma", - "dedifferentiated", - "diffuse_adenocarcinoma", - "diffuse_large_B_cell_lymphoma", - "ductal_carcinoma", - "embryonal", - "endometrioid_carcinoma", - "essential_thrombocythaemia", - "follicular_carcinoma", - "giant_cell_tumour", - "gliosarcoma", - "granulosa_cell_tumour", - "hepatoblastoma", - "hepatocellular_carcinoma", - "Hodgkin_lymphoma", - "intestinal_adenocarcinoma", - "large_cell_carcinoma", - "mantle_cell_lymphoma", - "medullary_carcinoma", - "metaplasia", - "metaplastic_carcinoma", - "mixed_adenosquamous_carcinoma", - "mixed_carcinoma", - "mucinous_carcinoma", - "mucoepidermoid_carcinoma", - "mycosis_fungoides-Sezary_syndrome", - "non_small_cell_carcinoma", - "NS", - "oligodendroglioma", - "papillary_carcinoma", - "papilloma", - "peripheral_T_cell_lymphoma_unspecified", - "plasma_cell_myeloma", - "renal_cell_carcinoma", - "serous_carcinoma", - "signet_ring_adenocarcinoma", - "small_cell_adenocarcinoma", - "small_cell_carcinoma", - "squamous_cell_carcinoma", - "transitional_cell_carcinoma", - "tubular_adenocarcinoma", - "undifferentiated_adenocarcinoma", - "undifferentiated_carcinoma" - ], - "site_primary": [ - "autonomic_ganglia", - "biliary_tract", - "bone", - "breast", - "central_nervous_system", - "endometrium", - "haematopoietic_and_lymphoid_tissue", - "kidney", - "large_intestine", - "liver", - "lung", - "oesophagus", - "ovary", - "pancreas", - "pleura", - "prostate", - "salivary_gland", - "skin", - "small_intestine", - "soft_tissue", - "stomach", - "thyroid", - "upper_aerodigestive_tract", - "urinary_tract" - ], - "source": [ - "Academic Lab", - "ATCC", - "DSMZ", - "ECACC", - "HSRRB", - "ICLC", - "KCLB", - "NCI/DCTD", - "RIKEN" - ] - }, - "Biospecimen": { - "sample_type": [ - "13", - "50" - ] - }, - "Data_HG19": { - "access": [ - "open" - ], - "analysis_workflow_type": [ - - ], - "center_code": [ - - ], - "center_name": [ - - ], - "center_type": [ - - ], - "data_category": [ - "Raw sequencing data" - ], - "data_format": [ - "BAM" - ], - "data_type": [ - "Aligned reads" - ], - "experimental_strategy": [ - "RNA-Seq", - "WGS", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "true" - ], - "platform": [ - "Illumina HiSeq" - ], - "sample_type": [ - "13", - "50" - ], - "species": [ - "Homo sapiens" - ] - } -} diff --git a/api_3/isb_cgc_api_CCLE/cohorts_create.py b/api_3/isb_cgc_api_CCLE/cohorts_create.py deleted file mode 100755 index 2d284f50..00000000 --- a/api_3/isb_cgc_api_CCLE/cohorts_create.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsCreateHelper, CreatedCohort -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.isb_cgc_api_CCLE.message_classes import MetadataRangesItem - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='cohorts') -class CCLECohortsCreateAPI(CohortsCreateHelper): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='ccle/cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - self.program = 'CCLE' - return super(CCLECohortsCreateAPI, self).create(request) diff --git a/api_3/isb_cgc_api_CCLE/cohorts_preview.py b/api_3/isb_cgc_api_CCLE/cohorts_preview.py deleted file mode 100755 index 3166782f..00000000 --- a/api_3/isb_cgc_api_CCLE/cohorts_preview.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from api_3.cohort_create_preview_helper import CohortsPreviewHelper -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from message_classes import MetadataRangesItem - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='cohorts') -class CCLECohortsPreviewAPI(CohortsPreviewHelper): - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem) - - @endpoints.method(POST_RESOURCE, CohortsPreviewHelper.CohortCasesSamplesList, path='tcga/cohorts/preview', http_method='POST') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - self.program = 'CCLE' - return super(CCLECohortsPreviewAPI, self).preview(request) diff --git a/api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py b/api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py deleted file mode 100755 index 57e67f8c..00000000 --- a/api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_CCLE_Endpoints = endpoints.api(name='isb_cgc_ccle_api', version='v3', - description="Get information about cohorts, cases, and samples for CCLE. Create cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - package_path='ccle', - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC CCLE API") diff --git a/api_3/isb_cgc_api_CCLE/message_classes.py b/api_3/isb_cgc_api_CCLE/message_classes.py deleted file mode 100755 index f73dbd6d..00000000 --- a/api_3/isb_cgc_api_CCLE/message_classes.py +++ /dev/null @@ -1,111 +0,0 @@ -from protorpc import messages - -class CommonMetadataRangesItem(messages.Message): - disease_code = messages.StringField(1, repeated=True) - endpoint_type = messages.StringField(2, repeated=True) - program_name = messages.StringField(3, repeated=True) - project_short_name = messages.StringField(4, repeated=True) - -class CommonMetadataItem(messages.Message): - disease_code = messages.StringField(1) - endpoint_type = messages.StringField(2) - program_name = messages.StringField(3) - project_short_name = messages.StringField(4) - -class ClinicalMetadataRangesItem(messages.Message): - case_barcode = messages.StringField(1, repeated=True) - case_gdc_id = messages.StringField(2, repeated=True) - gender = messages.StringField(3, repeated=True) - histology = messages.StringField(4, repeated=True) - hist_subtype = messages.StringField(5, repeated=True) - site_primary = messages.StringField(6, repeated=True) - source = messages.StringField(7, repeated=True) - summary_file_count = messages.IntegerField(8, repeated=True, variant=messages.Variant.INT32) - summary_file_count_lte = messages.IntegerField(9, variant=messages.Variant.INT32) - summary_file_count_gte = messages.IntegerField(10, variant=messages.Variant.INT32) - - -class ClinicalMetadataItem(messages.Message): - case_barcode = messages.StringField(1) - case_gdc_id = messages.StringField(2) - gender = messages.StringField(3) - histology = messages.StringField(4) - hist_subtype = messages.StringField(5) - site_primary = messages.StringField(6) - source = messages.StringField(7) - summary_file_count = messages.IntegerField(8, variant=messages.Variant.INT32) - -class BiospecimenMetadataRangesItem(messages.Message): - case_barcode = messages.StringField(1, repeated=True) - case_gdc_id = messages.StringField(2, repeated=True) - sample_barcode = messages.StringField(3, repeated=True) - sample_gdc_id = messages.StringField(4, repeated=True) - sample_type = messages.StringField(5, repeated=True) - -class BiospecimenMetadataItem(messages.Message): - case_barcode = messages.StringField(1) - case_gdc_id = messages.StringField(2) - sample_barcode = messages.StringField(3) - sample_gdc_id = messages.StringField(4) - sample_type = messages.StringField(5) - -class Data_HG19MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - aliquot_barcode = messages.StringField(2, repeated=True) - aliquot_gdc_id = messages.StringField(3, repeated=True) - analysis_workflow_type = messages.StringField(4, repeated=True) - case_barcode = messages.StringField(5, repeated=True) - case_gdc_id = messages.StringField(6, repeated=True) - center_code = messages.StringField(7, repeated=True) - center_name = messages.StringField(8, repeated=True) - center_type = messages.StringField(9, repeated=True) - data_category = messages.StringField(10, repeated=True) - data_format = messages.StringField(11, repeated=True) - data_type = messages.StringField(12, repeated=True) - experimental_strategy = messages.StringField(13, repeated=True) - file_name = messages.StringField(14, repeated=True) - file_state = messages.StringField(15, repeated=True) - file_uploaded = messages.StringField(16, repeated=True) - index_file_name = messages.StringField(17, repeated=True) - platform = messages.StringField(18, repeated=True) - sample_barcode = messages.StringField(19, repeated=True) - sample_gdc_id = messages.StringField(20, repeated=True) - sample_type = messages.StringField(21, repeated=True) - species = messages.StringField(22, repeated=True) - -class Data_HG19MetadataItem(messages.Message): - access = messages.StringField(1) - aliquot_barcode = messages.StringField(2) - aliquot_gdc_id = messages.StringField(3) - analysis_workflow_type = messages.StringField(4) - case_barcode = messages.StringField(5) - case_gdc_id = messages.StringField(6) - center_code = messages.StringField(7) - center_name = messages.StringField(8) - center_type = messages.StringField(9) - data_category = messages.StringField(10) - data_format = messages.StringField(11) - data_type = messages.StringField(12) - experimental_strategy = messages.StringField(13) - file_name = messages.StringField(14) - file_state = messages.StringField(15) - file_uploaded = messages.StringField(16) - index_file_name = messages.StringField(17) - platform = messages.StringField(18) - sample_barcode = messages.StringField(19) - sample_gdc_id = messages.StringField(20) - sample_type = messages.StringField(21) - species = messages.StringField(22) - -class MetadataRangesItem(messages.Message): - Common = messages.MessageField(CommonMetadataRangesItem, 1) - Clinical = messages.MessageField(ClinicalMetadataRangesItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataRangesItem, 3) - Data_HG19 = messages.MessageField(Data_HG19MetadataRangesItem, 4) - -class MetadataItem(messages.Message): - Common = messages.MessageField(CommonMetadataItem, 1) - Clinical = messages.MessageField(ClinicalMetadataItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataItem, 3) - Data_HG19 = messages.MessageField(Data_HG19MetadataItem, 4) - diff --git a/api_3/isb_cgc_api_CCLE/patients_get.py b/api_3/isb_cgc_api_CCLE/patients_get.py deleted file mode 100755 index 84d8aea1..00000000 --- a/api_3/isb_cgc_api_CCLE/patients_get.py +++ /dev/null @@ -1,55 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.patients_get_helper import CasesGetHelper -from api_3.isb_cgc_api_CCLE.message_classes import ClinicalMetadataItem as MetadataItem -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints - -class CaseDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4) - -class CaseSetDetails(messages.Message): - cases = messages.MessageField(CaseDetails, 1, repeated=True) - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='cases') -class CCLECasesGetAPI(CasesGetHelper): - @endpoints.method(CasesGetHelper.GET_RESOURCE, CaseDetails, path='ccle/cases/{case_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case barcode (*eg* ACC-MESO-1) as a required parameter. - User does not need to be authenticated. - """ - return super(CCLECasesGetAPI, self).get(request, CaseDetails, MetadataItem, 'CCLE') - - - @endpoints.method(CasesGetHelper.POST_RESOURCE, CaseSetDetails, path='ccle/cases', http_method='POST') - def get_list(self, request): - """ - Given a list of case barcodes (*eg* ACC-MESO-1), returns information about them, including a - list of samples and aliquots derived from this case. - Takes a list of case barcodes (*eg* ACC-MESO-1) as a required data payload. - User does not need to be authenticated. - """ - return super(CCLECasesGetAPI, self).get_list(request, CaseSetDetails, CaseDetails, MetadataItem, 'CCLE') diff --git a/api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py b/api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py deleted file mode 100755 index 62b2afd8..00000000 --- a/api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, SamplesCloudStorageFilePathsHelper - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='samples') -class CCLESamplesCloudStorageFilePathsAPI(SamplesCloudStorageFilePathsHelper): - - @endpoints.method(SamplesCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, - path='ccle/samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - return super(CCLESamplesCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request, 'CCLE') diff --git a/api_3/isb_cgc_api_CCLE/samples_get.py b/api_3/isb_cgc_api_CCLE/samples_get.py deleted file mode 100755 index bc59dd43..00000000 --- a/api_3/isb_cgc_api_CCLE/samples_get.py +++ /dev/null @@ -1,59 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.isb_cgc_api_CCLE.message_classes import MetadataItem -from api_3.samples_get_helper import SamplesGetAPI, DataDetails - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(MetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - case_barcode = messages.StringField(3) - case_gdc_id = messages.StringField(4) - data_details = messages.MessageField(DataDetails, 5, repeated=True) - data_details_count = messages.IntegerField(6, variant=messages.Variant.INT32) - sample_barcode = messages.StringField(7) - - -class SampleSetDetails(messages.Message): - samples = messages.MessageField(SampleDetails, 1, repeated=True) - - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='samples') -class CCLESamplesGetAPI(SamplesGetAPI): - @endpoints.method(SamplesGetAPI.GET_RESOURCE, SampleDetails, path='ccle/samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (*eg* CCLE-ACC-MESO-1), this endpoint returns - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(CCLESamplesGetAPI, self).get(request, 'CCLE', SampleDetails, MetadataItem) - - - @endpoints.method(SamplesGetAPI.POST_RESOURCE, SampleSetDetails, path='ccle/samples', http_method='POST') - def get_list(self, request): - """ - Given a list of sample barcodes (of length 16, *eg* CCLE-ACC-MESO-1), this endpoint returns - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(CCLESamplesGetAPI, self).get_list(request, 'CCLE', SampleSetDetails, SampleDetails, MetadataItem) diff --git a/api_3/isb_cgc_api_CCLE/samples_googlegenomics.py b/api_3/isb_cgc_api_CCLE/samples_googlegenomics.py deleted file mode 100755 index 2b6e1921..00000000 --- a/api_3/isb_cgc_api_CCLE/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_CCLE_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api_3/isb_cgc_api_TARGET/__init__.py b/api_3/isb_cgc_api_TARGET/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json b/api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json deleted file mode 100755 index 0f61d013..00000000 --- a/api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "Common": { - "disease_code": [ - "ALL", - "AML", - "CCSK", - "NBL", - "OS", - "RT", - "WT" - ], - "endpoint_type": [ - "current", - "legacy" - ], - "program_name": [ - "TARGET" - ], - "project_short_name": [ - "TARGET-ALL-P1", - "TARGET-ALL-P2", - "TARGET-AML", - "TARGET-CCSK", - "TARGET-NBL", - "TARGET-OS", - "TARGET-RT", - "TARGET-WT" - ] - }, - "Clinical": { - "ethnicity": [ - "Hispanic or Latino", - "Not Hispanic or Latino" - ], - "first_event": [ - "Censored", - "Death", - "Death without remission", - "Event", - "Induction failure", - "Progression", - "Relapse", - "Second Malignant Neoplasm" - ], - "gender": [ - "Female", - "Male" - ], - "race": [ - "American Indian or Alaska Native", - "Asian", - "Black or African American", - "Native Hawaiian or other Pacific Islander", - "Other", - "White" - ], - "vital_status": [ - "alive", - "dead" - ] - }, - "Biospecimen": { - "sample_type": [ - "01", - "02", - "03", - "04", - "06", - "08", - "09", - "10", - "11", - "13", - "14", - "15", - "20", - "40", - "41", - "42", - "50", - "60" - ], - "tumor_code": [ - "00", - "10", - "20", - "21", - "30", - "40", - "50", - "51", - "52" - ] - }, - "Data_HG19": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - - ], - "center_code": [ - "01", - "12", - "13", - "37" - ], - "center_name": [ - "Baylor College of Medicine", - "Broad Institute of MIT and Harvard", - "Canada's Michael Smith Genome Sciences Centre", - "Complete Genomics Inc." - ], - "center_type": [ - "CGCC", - "COM" - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw sequencing data" - ], - "data_format": [ - "BAM", - "XLSX" - ], - "data_type": [ - "Aligned reads", - "Biospecimen Supplement", - "Clinical Supplement" - ], - "experimental_strategy": [ - "Bisulfite-Seq", - "miRNA-Seq", - "RNA-Seq", - "VALIDATION", - "WGS", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "false", - "true" - ], - "platform": [ - "Complete Genomics", - "Illumina GA", - "Illumina HiSeq", - "Ion Torrent PGM" - ], - "sample_type": [ - "01", - "02", - "03", - "04", - "06", - "09", - "10", - "11", - "14", - "15", - "20", - "40", - "41", - "42", - "50" - ], - "species": [ - "Homo sapiens" - ] - }, - "Data_HG38": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - "BWA with Mark Duplicates and Cocleaning", - "BWA-aln", - "STAR 2-Pass" - ], - "center_code": [ - - ], - "center_name": [ - - ], - "center_type": [ - - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw Sequencing Data" - ], - "data_format": [ - "BAM", - "XLSX" - ], - "data_type": [ - "Aligned Reads", - "Biospecimen Supplement", - "Clinical Supplement" - ], - "experimental_strategy": [ - "miRNA-Seq", - "RNA-Seq", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "false", - "true" - ], - "platform": [ - "Illumina" - ], - "sample_type": [ - "01", - "02", - "03", - "04", - "06", - "09", - "10", - "11", - "14", - "20", - "40", - "41", - "42", - "50" - ], - "species": [ - "Homo sapiens" - ] - } -} diff --git a/api_3/isb_cgc_api_TARGET/cohorts_create.py b/api_3/isb_cgc_api_TARGET/cohorts_create.py deleted file mode 100755 index 486654f1..00000000 --- a/api_3/isb_cgc_api_TARGET/cohorts_create.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsCreateHelper, CreatedCohort -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.isb_cgc_api_TARGET.message_classes import MetadataRangesItem - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='cohorts') -class TARGETCohortsCreateAPI(CohortsCreateHelper): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='target/cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - self.program = 'TARGET' - return super(TARGETCohortsCreateAPI, self).create(request) diff --git a/api_3/isb_cgc_api_TARGET/cohorts_preview.py b/api_3/isb_cgc_api_TARGET/cohorts_preview.py deleted file mode 100755 index 09cb1e74..00000000 --- a/api_3/isb_cgc_api_TARGET/cohorts_preview.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from api_3.cohort_create_preview_helper import CohortsPreviewHelper -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from message_classes import MetadataRangesItem - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='cohorts') -class TARGETCohortsPreviewAPI(CohortsPreviewHelper): - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem) - - @endpoints.method(POST_RESOURCE, CohortsPreviewHelper.CohortCasesSamplesList, path='tcga/cohorts/preview', http_method='POST') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - self.program = 'TARGET' - return super(TARGETCohortsPreviewAPI, self).preview(request) diff --git a/api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py b/api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py deleted file mode 100755 index cda91463..00000000 --- a/api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_TARGET_Endpoints = endpoints.api(name='isb_cgc_target_api', version='v3', - description="Get information about cohorts, cases, and samples for TARGET. Create cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - package_path='target', - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC TARGET API") diff --git a/api_3/isb_cgc_api_TARGET/message_classes.py b/api_3/isb_cgc_api_TARGET/message_classes.py deleted file mode 100755 index 9c895379..00000000 --- a/api_3/isb_cgc_api_TARGET/message_classes.py +++ /dev/null @@ -1,210 +0,0 @@ -from protorpc import messages - -class CommonMetadataRangesItem(messages.Message): - disease_code = messages.StringField(1, repeated=True) - endpoint_type = messages.StringField(2, repeated=True) - program_name = messages.StringField(3, repeated=True) - project_short_name = messages.StringField(4, repeated=True) - -class CommonMetadataItem(messages.Message): - disease_code = messages.StringField(1) - endpoint_type = messages.StringField(2) - program_name = messages.StringField(3) - project_short_name = messages.StringField(4) - -class ClinicalMetadataRangesItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, repeated=True, variant=messages.Variant.INT32) - age_at_diagnosis_lte = messages.IntegerField(2, variant=messages.Variant.INT32) - age_at_diagnosis_gte = messages.IntegerField(3, variant=messages.Variant.INT32) - - case_barcode = messages.StringField(4, repeated=True) - case_gdc_id = messages.StringField(5, repeated=True) - days_to_birth = messages.IntegerField(6, repeated=True, variant=messages.Variant.INT32) - days_to_birth_lte = messages.IntegerField(7, variant=messages.Variant.INT32) - days_to_birth_gte = messages.IntegerField(8, variant=messages.Variant.INT32) - - days_to_death = messages.IntegerField(9, repeated=True, variant=messages.Variant.INT32) - days_to_death_lte = messages.IntegerField(10, variant=messages.Variant.INT32) - days_to_death_gte = messages.IntegerField(11, variant=messages.Variant.INT32) - - days_to_last_followup = messages.IntegerField(12, repeated=True, variant=messages.Variant.INT32) - days_to_last_followup_lte = messages.IntegerField(13, variant=messages.Variant.INT32) - days_to_last_followup_gte = messages.IntegerField(14, variant=messages.Variant.INT32) - - days_to_last_known_alive = messages.IntegerField(15, repeated=True, variant=messages.Variant.INT32) - days_to_last_known_alive_lte = messages.IntegerField(16, variant=messages.Variant.INT32) - days_to_last_known_alive_gte = messages.IntegerField(17, variant=messages.Variant.INT32) - - ethnicity = messages.StringField(18, repeated=True) - event_free_survival = messages.IntegerField(19, repeated=True, variant=messages.Variant.INT32) - event_free_survival_lte = messages.IntegerField(20, variant=messages.Variant.INT32) - event_free_survival_gte = messages.IntegerField(21, variant=messages.Variant.INT32) - - first_event = messages.StringField(22, repeated=True) - gender = messages.StringField(23, repeated=True) - protocol = messages.StringField(24, repeated=True) - race = messages.StringField(25, repeated=True) - summary_file_count = messages.IntegerField(26, repeated=True, variant=messages.Variant.INT32) - summary_file_count_lte = messages.IntegerField(27, variant=messages.Variant.INT32) - summary_file_count_gte = messages.IntegerField(28, variant=messages.Variant.INT32) - - vital_status = messages.StringField(29, repeated=True) - wbc_at_diagnosis = messages.FloatField(30, repeated=True) - wbc_at_diagnosis_lte = messages.FloatField(31) - wbc_at_diagnosis_gte = messages.FloatField(32) - - year_of_diagnosis = messages.IntegerField(33, repeated=True, variant=messages.Variant.INT32) - year_of_diagnosis_lte = messages.IntegerField(34, variant=messages.Variant.INT32) - year_of_diagnosis_gte = messages.IntegerField(35, variant=messages.Variant.INT32) - - year_of_last_follow_up = messages.IntegerField(36, repeated=True, variant=messages.Variant.INT32) - year_of_last_follow_up_lte = messages.IntegerField(37, variant=messages.Variant.INT32) - year_of_last_follow_up_gte = messages.IntegerField(38, variant=messages.Variant.INT32) - - -class ClinicalMetadataItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, variant=messages.Variant.INT32) - case_barcode = messages.StringField(2) - case_gdc_id = messages.StringField(3) - days_to_birth = messages.IntegerField(4, variant=messages.Variant.INT32) - days_to_death = messages.IntegerField(5, variant=messages.Variant.INT32) - days_to_last_followup = messages.IntegerField(6, variant=messages.Variant.INT32) - days_to_last_known_alive = messages.IntegerField(7, variant=messages.Variant.INT32) - ethnicity = messages.StringField(8) - event_free_survival = messages.IntegerField(9, variant=messages.Variant.INT32) - first_event = messages.StringField(10) - gender = messages.StringField(11) - protocol = messages.StringField(12) - race = messages.StringField(13) - summary_file_count = messages.IntegerField(14, variant=messages.Variant.INT32) - vital_status = messages.StringField(15) - wbc_at_diagnosis = messages.FloatField(16) - year_of_diagnosis = messages.IntegerField(17, variant=messages.Variant.INT32) - year_of_last_follow_up = messages.IntegerField(18, variant=messages.Variant.INT32) - -class BiospecimenMetadataRangesItem(messages.Message): - case_barcode = messages.StringField(1, repeated=True) - case_gdc_id = messages.StringField(2, repeated=True) - sample_barcode = messages.StringField(3, repeated=True) - sample_gdc_id = messages.StringField(4, repeated=True) - sample_type = messages.StringField(5, repeated=True) - tumor_code = messages.StringField(6, repeated=True) - -class BiospecimenMetadataItem(messages.Message): - case_barcode = messages.StringField(1) - case_gdc_id = messages.StringField(2) - sample_barcode = messages.StringField(3) - sample_gdc_id = messages.StringField(4) - sample_type = messages.StringField(5) - tumor_code = messages.StringField(6) - -class Data_HG19MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - aliquot_barcode = messages.StringField(2, repeated=True) - aliquot_gdc_id = messages.StringField(3, repeated=True) - analysis_workflow_type = messages.StringField(4, repeated=True) - case_barcode = messages.StringField(5, repeated=True) - case_gdc_id = messages.StringField(6, repeated=True) - center_code = messages.StringField(7, repeated=True) - center_name = messages.StringField(8, repeated=True) - center_type = messages.StringField(9, repeated=True) - data_category = messages.StringField(10, repeated=True) - data_format = messages.StringField(11, repeated=True) - data_type = messages.StringField(12, repeated=True) - experimental_strategy = messages.StringField(13, repeated=True) - file_name = messages.StringField(14, repeated=True) - file_state = messages.StringField(15, repeated=True) - file_uploaded = messages.StringField(16, repeated=True) - index_file_name = messages.StringField(17, repeated=True) - platform = messages.StringField(18, repeated=True) - sample_barcode = messages.StringField(19, repeated=True) - sample_gdc_id = messages.StringField(20, repeated=True) - sample_type = messages.StringField(21, repeated=True) - species = messages.StringField(22, repeated=True) - -class Data_HG19MetadataItem(messages.Message): - access = messages.StringField(1) - aliquot_barcode = messages.StringField(2) - aliquot_gdc_id = messages.StringField(3) - analysis_workflow_type = messages.StringField(4) - case_barcode = messages.StringField(5) - case_gdc_id = messages.StringField(6) - center_code = messages.StringField(7) - center_name = messages.StringField(8) - center_type = messages.StringField(9) - data_category = messages.StringField(10) - data_format = messages.StringField(11) - data_type = messages.StringField(12) - experimental_strategy = messages.StringField(13) - file_name = messages.StringField(14) - file_state = messages.StringField(15) - file_uploaded = messages.StringField(16) - index_file_name = messages.StringField(17) - platform = messages.StringField(18) - sample_barcode = messages.StringField(19) - sample_gdc_id = messages.StringField(20) - sample_type = messages.StringField(21) - species = messages.StringField(22) - -class Data_HG38MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - aliquot_barcode = messages.StringField(2, repeated=True) - aliquot_gdc_id = messages.StringField(3, repeated=True) - analysis_workflow_type = messages.StringField(4, repeated=True) - case_barcode = messages.StringField(5, repeated=True) - case_gdc_id = messages.StringField(6, repeated=True) - center_code = messages.StringField(7, repeated=True) - center_name = messages.StringField(8, repeated=True) - center_type = messages.StringField(9, repeated=True) - data_category = messages.StringField(10, repeated=True) - data_format = messages.StringField(11, repeated=True) - data_type = messages.StringField(12, repeated=True) - experimental_strategy = messages.StringField(13, repeated=True) - file_name = messages.StringField(14, repeated=True) - file_state = messages.StringField(15, repeated=True) - file_uploaded = messages.StringField(16, repeated=True) - index_file_name = messages.StringField(17, repeated=True) - platform = messages.StringField(18, repeated=True) - sample_barcode = messages.StringField(19, repeated=True) - sample_gdc_id = messages.StringField(20, repeated=True) - sample_type = messages.StringField(21, repeated=True) - species = messages.StringField(22, repeated=True) - -class Data_HG38MetadataItem(messages.Message): - access = messages.StringField(1) - aliquot_barcode = messages.StringField(2) - aliquot_gdc_id = messages.StringField(3) - analysis_workflow_type = messages.StringField(4) - case_barcode = messages.StringField(5) - case_gdc_id = messages.StringField(6) - center_code = messages.StringField(7) - center_name = messages.StringField(8) - center_type = messages.StringField(9) - data_category = messages.StringField(10) - data_format = messages.StringField(11) - data_type = messages.StringField(12) - experimental_strategy = messages.StringField(13) - file_name = messages.StringField(14) - file_state = messages.StringField(15) - file_uploaded = messages.StringField(16) - index_file_name = messages.StringField(17) - platform = messages.StringField(18) - sample_barcode = messages.StringField(19) - sample_gdc_id = messages.StringField(20) - sample_type = messages.StringField(21) - species = messages.StringField(22) - -class MetadataRangesItem(messages.Message): - Common = messages.MessageField(CommonMetadataRangesItem, 1) - Clinical = messages.MessageField(ClinicalMetadataRangesItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataRangesItem, 3) - Data_HG19 = messages.MessageField(Data_HG19MetadataRangesItem, 4) - Data_HG38 = messages.MessageField(Data_HG38MetadataRangesItem, 5) - -class MetadataItem(messages.Message): - Common = messages.MessageField(CommonMetadataItem, 1) - Clinical = messages.MessageField(ClinicalMetadataItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataItem, 3) - Data_HG19 = messages.MessageField(Data_HG19MetadataItem, 4) - Data_HG38 = messages.MessageField(Data_HG38MetadataItem, 5) - diff --git a/api_3/isb_cgc_api_TARGET/patients_get.py b/api_3/isb_cgc_api_TARGET/patients_get.py deleted file mode 100755 index 7e40dbfb..00000000 --- a/api_3/isb_cgc_api_TARGET/patients_get.py +++ /dev/null @@ -1,55 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.patients_get_helper import CasesGetHelper -from api_3.isb_cgc_api_TARGET.message_classes import ClinicalMetadataItem as MetadataItem -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints - -class CaseDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4) - -class CaseSetDetails(messages.Message): - cases = messages.MessageField(CaseDetails, 1, repeated=True) - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='cases') -class TARGETCasesGetAPI(CasesGetHelper): - @endpoints.method(CasesGetHelper.GET_RESOURCE, CaseDetails, path='target/cases/{case_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case barcode (of length 16, *eg* TARGET-51-PALFYG) as a required parameter. - User does not need to be authenticated. - """ - return super(TARGETCasesGetAPI, self).get(request, CaseDetails, MetadataItem, 'TARGET') - - @endpoints.method(CasesGetHelper.POST_RESOURCE, CaseSetDetails, path='target/cases', http_method='POST') - def get_list(self, request): - """ - Given a list of case barcodes (of length 11, *eg* TARGET-51-PALFYG), this endpoint returns - all available "biospecimen" information about them, including a list of samples and aliquots - derived from them. - Takes a list of case barcodes (of length 12, *eg* TARGET-51-PALFYG) as a required data payload. - User does not need to be authenticated. - """ - return super(TARGETCasesGetAPI, self).get_list(request, CaseSetDetails, CaseDetails, MetadataItem, 'TARGET') diff --git a/api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py b/api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py deleted file mode 100755 index 5612988e..00000000 --- a/api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, SamplesCloudStorageFilePathsHelper - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='samples') -class TARGETSamplesCloudStorageFilePathsAPI(SamplesCloudStorageFilePathsHelper): - - @endpoints.method(SamplesCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, - path='target/samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - return super(TARGETSamplesCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request, 'TARGET') diff --git a/api_3/isb_cgc_api_TARGET/samples_get.py b/api_3/isb_cgc_api_TARGET/samples_get.py deleted file mode 100755 index d31b2fa2..00000000 --- a/api_3/isb_cgc_api_TARGET/samples_get.py +++ /dev/null @@ -1,60 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.isb_cgc_api_TARGET.message_classes import BiospecimenMetadataItem -from api_3.samples_get_helper import SamplesGetAPI, DataDetails - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(BiospecimenMetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - case_barcode = messages.StringField(3) - case_gdc_id = messages.StringField(4) - data_details = messages.MessageField(DataDetails, 5, repeated=True) - data_details_count = messages.IntegerField(6, variant=messages.Variant.INT32) - sample_barcode = messages.StringField(7) - - -class SampleSetDetails(messages.Message): - samples = messages.MessageField(SampleDetails, 1, repeated=True) - - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='samples') -class TARGETSamplesGetAPI(SamplesGetAPI): - @endpoints.method(SamplesGetAPI.GET_RESOURCE, SampleDetails, path='target/samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (of length 20-22, *eg* TARGET-51-PALFYG-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TARGETSamplesGetAPI, self).get(request, 'TARGET', SampleDetails, BiospecimenMetadataItem) - - @endpoints.method(SamplesGetAPI.POST_RESOURCE, SampleSetDetails, path='target/samples', http_method='POST') - def get_list(self, request): - """ - Given a list of sample barcodes (of length 16, *eg* TARGET-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TARGETSamplesGetAPI, self).get_list(request, 'TARGET', SampleSetDetails, SampleDetails, BiospecimenMetadataItem) diff --git a/api_3/isb_cgc_api_TARGET/samples_googlegenomics.py b/api_3/isb_cgc_api_TARGET/samples_googlegenomics.py deleted file mode 100755 index b31fcc8d..00000000 --- a/api_3/isb_cgc_api_TARGET/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_TARGET_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api_3/isb_cgc_api_TARGET/users_get.py b/api_3/isb_cgc_api_TARGET/users_get.py deleted file mode 100755 index 767066e1..00000000 --- a/api_3/isb_cgc_api_TARGET/users_get.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import message_types - -from api_3.users_get_common import UserGetAPICommon, UserGetAPIReturnJSON -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='users') -class TARGETUserGetAPI(UserGetAPICommon): - - @endpoints.method(message_types.VoidMessage, UserGetAPIReturnJSON, http_method='GET', path='target/users') - def get(self, _): - ''' - Returns the dbGaP authorization status of the user. - ''' - return super(TARGETUserGetAPI, self).get('TARGET') diff --git a/api_3/isb_cgc_api_TCGA/__init__.py b/api_3/isb_cgc_api_TCGA/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api_TCGA/aliquots_annotations.py b/api_3/isb_cgc_api_TCGA/aliquots_annotations.py deleted file mode 100644 index e4b64fea..00000000 --- a/api_3/isb_cgc_api_TCGA/aliquots_annotations.py +++ /dev/null @@ -1,72 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import logging - -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TCGA.annotations_api import AnnotationAPI, MetadataAnnotationList -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -logger = logging.getLogger(__name__) - -class AliquotsAnnotationsQueryBuilder(object): - - @staticmethod - def build_query(entity_types=None): - query_str = 'select * ' \ - 'from TCGA_metadata_annotation ' \ - 'where aliquot_barcode=%s and status = "Approved" ' - if len(entity_types) > 0: - query_str += 'and entity_type in (' + ', '.join(['%s']*len(entity_types)) + ')' - - return query_str - - @staticmethod - def build_metadata_query(): - return 'select * from TCGA_metadata_data_HG19 where aliquot_barcode=%s ' - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='aliquots') -class TCGAAliquotsAnnotationAPI(AnnotationAPI): - - GET_RESOURCE = endpoints.ResourceContainer(aliquot_barcode=messages.StringField(1, required=True), - entity_type=messages.StringField(2, repeated=True)) - - @endpoints.method(GET_RESOURCE, MetadataAnnotationList, - path='aliquots/{aliquot_barcode}/annotations', http_method='GET') - def annotations(self, request): - """ - Returns TCGA annotations about a specific aliquot, - Takes an aliquot barcode (of length 28, *eg* TCGA-01-0628-11A-01D-0356-01) as a required parameter. - User does not need to be authenticated. - """ - return self.process_annotations(request, 'aliquot_barcode', AliquotsAnnotationsQueryBuilder(), logger) - - - def validate_barcode(self, aliquot_barcode): - # check to make sure aliquot_barcode is in correct form - parts = aliquot_barcode.split('-') - assert len(parts) == 7 - assert len(parts[0]) == 4 - assert len(parts[1]) == 2 - assert len(parts[2]) == 4 - assert len(parts[3]) == 3 - assert len(parts[4]) in [2, 3] - assert len(parts[5]) == 4 - assert len(parts[6]) == 2 - diff --git a/api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json b/api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json deleted file mode 100755 index 5b1bcd63..00000000 --- a/api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json +++ /dev/null @@ -1,835 +0,0 @@ -{ - "Common": { - "disease_code": [ - "ACC", - "BLCA", - "BRCA", - "CESC", - "CHOL", - "COAD", - "DLBC", - "ESCA", - "GBM", - "HNSC", - "KICH", - "KIRC", - "KIRP", - "LAML", - "LGG", - "LIHC", - "LUAD", - "LUSC", - "MESO", - "OV", - "PAAD", - "PCPG", - "PRAD", - "READ", - "SARC", - "SKCM", - "STAD", - "TGCT", - "THCA", - "THYM", - "UCEC", - "UCS", - "UVM" - ], - "endpoint_type": [ - "current", - "legacy" - ], - "program_name": [ - "TCGA" - ], - "project_short_name": [ - "TCGA-ACC", - "TCGA-BLCA", - "TCGA-BRCA", - "TCGA-CESC", - "TCGA-CHOL", - "TCGA-COAD", - "TCGA-DLBC", - "TCGA-ESCA", - "TCGA-GBM", - "TCGA-HNSC", - "TCGA-KICH", - "TCGA-KIRC", - "TCGA-KIRP", - "TCGA-LAML", - "TCGA-LGG", - "TCGA-LIHC", - "TCGA-LUAD", - "TCGA-LUSC", - "TCGA-MESO", - "TCGA-OV", - "TCGA-PAAD", - "TCGA-PCPG", - "TCGA-PRAD", - "TCGA-READ", - "TCGA-SARC", - "TCGA-SKCM", - "TCGA-STAD", - "TCGA-TGCT", - "TCGA-THCA", - "TCGA-THYM", - "TCGA-UCEC", - "TCGA-UCS", - "TCGA-UVM" - ] - }, - "Clinical": { - "anatomic_neoplasm_subdivision": [ - "Alveolar Ridge", - "Antrum/Distal", - "Ascending Colon", - "Base of tongue", - "Bilateral", - "Bladder - NOS", - "Body of Pancreas", - "Bronchial", - "Buccal Mucosa", - "Cardia/Proximal", - "Cecum", - "Descending Colon", - "Dome", - "Endometrium", - "Floor of mouth", - "Fundus uteri", - "Fundus/Body", - "Gastroesophageal Junction", - "Hard Palate", - "Head of Pancreas", - "Hepatic Flexure", - "Hypopharynx", - "L-Lower", - "L-Upper", - "Larynx", - "Left", - "Left Lower Inner Quadrant", - "Left Lower Outer Quadrant", - "Left Upper Inner Quadrant", - "Left Upper Outer Quadrant", - "Lip", - "Lower uterine segment/Isthmus uteri", - "Myometrium", - "Neck", - "Oral Cavity", - "Oral Tongue", - "Oropharynx", - "Other (please specify)", - "R-Lower", - "R-Middle", - "R-Upper", - "Rectosigmoid Junction", - "Rectum", - "Right", - "Right Lower Inner Quadrant", - "Right Lower Outer Quadrant", - "Right Upper Inner Quadrant", - "Right Upper Outer Quadrant", - "Sigmoid Colon", - "Splenic Flexure", - "Stomach (NOS)", - "Tail of Pancreas", - "Tonsil", - "Transverse Colon", - "Trigone", - "Unknown - Uterus NOS", - "Wall Anterior", - "Wall Lateral", - "Wall NOS", - "Wall Posterior" - ], - "bcr": [ - "Nationwide Children's Hospital", - "Washington University" - ], - "clinical_M": [ - "M0", - "M1", - "M1a", - "M1b", - "M1c", - "MX" - ], - "clinical_N": [ - "N0", - "N1", - "N2", - "N2a", - "N2b", - "N2c", - "N3", - "NX" - ], - "clinical_stage": [ - "Stage I", - "Stage IA", - "Stage IA1", - "Stage IA2", - "Stage IB", - "Stage IB1", - "Stage IB2", - "Stage IC", - "Stage II", - "Stage IIA", - "Stage IIA1", - "Stage IIA2", - "Stage IIB", - "Stage IIC", - "Stage III", - "Stage IIIA", - "Stage IIIB", - "Stage IIIC", - "Stage IIIC1", - "Stage IIIC2", - "Stage IS", - "Stage IV", - "Stage IVA", - "Stage IVB", - "Stage IVC" - ], - "clinical_T": [ - "T1", - "T1a", - "T1b", - "T1c", - "T2", - "T2a", - "T2b", - "T2c", - "T3", - "T3a", - "T3b", - "T4", - "T4a", - "T4b", - "T4c", - "T4d", - "T4e", - "TX" - ], - "colorectal_cancer": [ - "NO", - "YES" - ], - "country": [ - "Afghanistan", - "Algeria", - "American Samoa", - "Australia", - "Brazil", - "Bulgaria", - "Canada", - "Croatia", - "Czech Republic", - "France", - "Georgia", - "Germany", - "Hamburg/Germany", - "Israel", - "Italy", - "Korea", - "Korea South", - "Moldova", - "Netherlands", - "Nigeria", - "Ontario Canada", - "Ontario/Canada", - "Pakistan", - "Poland", - "Puerto Rico", - "Republic of Moldova", - "Romania", - "Russia", - "Sao Paulo", - "Singapore", - "Spain", - "Switzerland", - "Ukraine", - "United Kingdom", - "United States", - "Vietnam", - "Yemen" - ], - "ethnicity": [ - "HISPANIC OR LATINO", - "NOT HISPANIC OR LATINO" - ], - "gender": [ - "FEMALE", - "MALE" - ], - "history_of_colon_polyps": [ - "NO", - "YES" - ], - "history_of_neoadjuvant_treatment": [ - "No", - "Yes", - "Yes, Pharmaceutical Treatment Prior to Resection", - "Yes, Radiation Prior to Resection" - ], - "hpv_calls": [ - "HPV16", - "HPV16;HPV18", - "HPV16;HPV18;HPV58", - "HPV16;HPV31", - "HPV16;HPV33", - "HPV16;HPV35", - "HPV16;HPV39", - "HPV16;HPV52", - "HPV16;HPV66", - "HPV18", - "HPV18;HPV31", - "HPV31", - "HPV33", - "HPV35", - "HPV39", - "HPV45", - "HPV51", - "HPV52", - "HPV56", - "HPV58", - "HPV59", - "HPV68", - "HPV73" - ], - "hpv_status": [ - "Indeterminate", - "Negative", - "Positive" - ], - "h_pylori_infection": [ - "Current", - "Never", - "No", - "Yes" - ], - "lymphatic_invasion": [ - "NO", - "YES" - ], - "lymphnodes_examined": [ - "NO", - "YES" - ], - "lymphovascular_invasion_present": [ - "NO", - "YES" - ], - "menopause_status": [ - "Indeterminate (neither Pre or Postmenopausal)", - "Peri (6-12 months since last menstrual period)", - "Post (prior bilateral ovariectomy OR >12 mo since LMP with no prior hysterectomy)", - "Pre (<6 months since LMP AND no prior bilateral ovariectomy AND not on estrogen replacement)" - ], - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": [ - "Indeterminate", - "MSI-H", - "MSI-L", - "MSS", - "Not Tested" - ], - "neoplasm_histologic_grade": [ - "G1", - "G2", - "G3", - "G4", - "GB", - "GX", - "High Grade", - "Low Grade" - ], - "new_tumor_event_after_initial_treatment": [ - "NO", - "YES" - ], - "other_dx": [ - "Both History of Synchronous/ Bilateral and Prior Malignancy", - "No", - "Yes, History of Prior Malignancy", - "Yes, History of Synchronous/Bilateral Malignancy" - ], - "other_malignancy_histological_type": [ - "Adenocarcinoma, Not Otherwise Specified", - "Adenocarcinoma, Not Otherwise Specified, Adenocarcinoma, Not Otherwise Specified", - "Adenocarcinoma, Not Otherwise Specified, Colon Adenocarcinoma", - "Adenocarcinoma, Not Otherwise Specified, Kidney Clear Cell Renal Carcinoma", - "Adenocarcinoma, Not Otherwise Specified, Lung Acinar Adenocarcinoma", - "Adenocarcinoma, Not Otherwise Specified, Other, specify", - "Adenocarcinoma, Not Otherwise Specified, Other, specify, Other, specify", - "Adenocarcinoma, Not Otherwise Specified, Squamous Cell Carcinoma, Not Otherwise Specified", - "Adenosquamous", - "Astrocytoma", - "Basaloid Squamous Cell", - "Basaloid Squamous Cell, Adenocarcinoma, Not Otherwise Specified", - "Clear Cell Adenocarcinoma", - "Clear Cell Squamous Cell", - "Colon Adenocarcinoma", - "Colon Adenocarcinoma, Colon Adenocarcinoma", - "Colon Mucinous Adenocarcinoma", - "Endometrioid endometrial adenocarcinoma (Grade 1 or 2)", - "Endometrioid endometrial adenocarcinoma (Grade 3)", - "Head & Neck Squamous Cell Carcinoma", - "Hepatocellular Carcinoma", - "Kidney Clear Cell Renal Carcinoma", - "Kidney Clear Cell Renal Carcinoma, Kidney Clear Cell Renal Carcinoma", - "Kidney Clear Cell Renal Carcinoma, Kidney Clear Cell Renal Carcinoma, Other, specify", - "Kidney Clear Cell Renal Carcinoma, Kidney Papillary Renal Cell Carcinoma", - "Kidney Clear Cell Renal Carcinoma, Other, specify", - "Kidney Papillary Renal Cell Carcinoma", - "Kidney Papillary Renal Cell Carcinoma, Kidney Papillary Renal Cell Carcinoma", - "Kidney Papillary Renal Cell Carcinoma, Kidney Papillary Renal Cell Carcinoma, Adenocarcinoma, Not Otherwise Specified", - "Lung Adenocarcinoma Mixed Subtype", - "Lung Adenocarcinoma- Not Otherwise Specified (NOS)", - "Lung Adenocarcinoma- Not Otherwise Specified (NOS), Adenocarcinoma, Not Otherwise Specified", - "Lung Bronchioloalveolar Carcinoma Nonmucinous", - "Lung Clear Cell Squamous Cell Carcinoma", - "Lung Clear Cell Squamous Cell Carcinoma, Other, specify", - "Lung Papillary Adenocarcinoma", - "Lung Small Cell Squamous Cell Carcinoma", - "Other, specify", - "Other, specify, Adenocarcinoma, Not Otherwise Specified", - "Other, specify, Adenocarcinoma, Not Otherwise Specified, Other, specify", - "Other, specify, Basaloid Squamous Cell", - "Other, specify, Clear Cell Adenocarcinoma", - "Other, specify, Kidney Papillary Renal Cell Carcinoma", - "Other, specify, Kidney Papillary Renal Cell Carcinoma, Kidney Papillary Renal Cell Carcinoma", - "Other, specify, Lung Mucinous Adenocarcinoma", - "Other, specify, Other, specify", - "Other, specify, Other, specify, Kidney Papillary Renal Cell Carcinoma", - "Other, specify, Other, specify, Other, specify", - "Other, specify, Other, specify, Other, specify, Other, specify", - "Other, specify, Other, specify, Squamous Cell Carcinoma, Not Otherwise Specified", - "Other, specify, Squamous Cell Carcinoma, Not Otherwise Specified", - "Papillary Squamous Cell", - "Rectal Adenocarcinoma", - "Small Cell Squamous Cell", - "Squamous Cell Carcinoma, Not Otherwise Specified", - "Squamous Cell Carcinoma, Not Otherwise Specified, Basaloid Squamous Cell", - "Squamous Cell Carcinoma, Not Otherwise Specified, Kidney Papillary Renal Cell Carcinoma", - "Squamous Cell Carcinoma, Not Otherwise Specified, Lung Adenocarcinoma Mixed Subtype", - "Squamous Cell Carcinoma, Not Otherwise Specified, Other, specify", - "Squamous Cell Carcinoma, Not Otherwise Specified, Other, specify, Other, specify", - "Squamous Cell Carcinoma, Not Otherwise Specified, Squamous Cell Carcinoma, Not Otherwise Specified", - "Squamous Cell Carcinoma, Not Otherwise Specified, Squamous Cell Carcinoma, Not Otherwise Specified, Basaloid Squamous Cell", - "Thyroid Papillary Carcinoma - Classical/usual", - "Thyroid Papillary Carcinoma - Classical/usual, Adenocarcinoma, Not Otherwise Specified", - "Thyroid Papillary Carcinoma - Follicular (>= 99% follicular patterned)", - "Thyroid Papillary Carcinoma - Other, specify", - "Thyroid Papillary Carcinoma - Other, specify, Thyroid Papillary Carcinoma - Other, specify, Other, specify", - "Uterine serous endometrial adenocarcinoma" - ], - "other_malignancy_type": [ - "Prior Malignancy", - "Prior Malignancy, Prior Malignancy", - "Prior Malignancy, Prior Malignancy, Prior Malignancy", - "Prior Malignancy, Prior Malignancy, Prior Malignancy, Synchronous Malignancy", - "Prior Malignancy, Prior Malignancy, Synchronous Malignancy", - "Prior Malignancy, Synchronous Malignancy", - "Prior Malignancy, Synchronous Malignancy, Prior Malignancy", - "Synchronous Malignancy", - "Synchronous Malignancy, Prior Malignancy", - "Synchronous Malignancy, Prior Malignancy, Prior Malignancy, Prior Malignancy", - "Synchronous Malignancy, Prior Malignancy, Synchronous Malignancy", - "Synchronous Malignancy, Synchronous Malignancy", - "Synchronous Malignancy, Synchronous Malignancy, Prior Malignancy" - ], - "pathologic_M": [ - "cM0 (i+)", - "M0", - "M1", - "M1a", - "M1b", - "M1c", - "MX" - ], - "pathologic_N": [ - "N0", - "N0 (i+)", - "N0 (i-)", - "N0 (mol+)", - "N1", - "N1a", - "N1b", - "N1c", - "N1mi", - "N2", - "N2a", - "N2b", - "N2c", - "N3", - "N3a", - "N3b", - "N3c", - "NX" - ], - "pathologic_stage": [ - "I/II NOS", - "IS", - "Stage 0", - "Stage I", - "Stage IA", - "Stage IB", - "Stage II", - "Stage IIA", - "Stage IIB", - "Stage IIC", - "Stage III", - "Stage IIIA", - "Stage IIIB", - "Stage IIIC", - "Stage IV", - "Stage IVA", - "Stage IVB", - "Stage IVC", - "Stage X" - ], - "pathologic_T": [ - "T0", - "T1", - "T1a", - "T1a1", - "T1b", - "T1b1", - "T1b2", - "T1c", - "T2", - "T2a", - "T2a1", - "T2a2", - "T2b", - "T2c", - "T3", - "T3a", - "T3b", - "T3c", - "T4", - "T4a", - "T4b", - "T4c", - "T4d", - "T4e", - "Tis", - "TX" - ], - "person_neoplasm_cancer_status": [ - "TUMOR FREE", - "WITH TUMOR" - ], - "pregnancies": [ - "0", - "1", - "2", - "3", - "4+" - ], - "primary_neoplasm_melanoma_dx": [ - "NO", - "YES" - ], - "primary_therapy_outcome_success": [ - "Complete Remission/Response", - "No Measureable Tumor or Tumor Markers", - "Normalization of Tumor Markers, but Residual Tumor Mass", - "Partial Remission/Response", - "Persistent Disease", - "Progressive Disease", - "Stable Disease" - ], - "race": [ - "AMERICAN INDIAN OR ALASKA NATIVE", - "ASIAN", - "BLACK OR AFRICAN AMERICAN", - "NATIVE HAWAIIAN OR OTHER PACIFIC ISLANDER", - "WHITE" - ], - "residual_tumor": [ - "R0", - "R1", - "R2", - "RX" - ], - "tobacco_smoking_history": [ - "1", - "2", - "3", - "4", - "5" - ], - "tumor_type": [ - "Primary", - "Type 1", - "Type 2" - ], - "venous_invasion": [ - "NO", - "YES" - ], - "vital_status": [ - "Alive", - "Dead" - ] - }, - "Biospecimen": { - "bcr": [ - "Nationwide Children's Hospital", - "Washington University" - ], - "preservation_method": [ - "FFPE", - "frozen" - ], - "sample_type": [ - "01", - "02", - "03", - "05", - "06", - "07", - "10", - "11", - "12", - "14" - ] - }, - "Data_HG19": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - - ], - "center_code": [ - "01", - "02", - "05", - "06", - "07", - "08", - "09", - "10", - "12", - "13", - "21", - "25", - "26", - "31", - "32", - "34", - "36" - ], - "center_name": [ - "Baylor College of Medicine", - "Broad Institute of MIT and Harvard", - "Canada's Michael Smith Genome Sciences Centre", - "Harvard Medical School", - "HudsonAlpha Institute for Biotechnology", - "Johns Hopkins / University of Southern California", - "MD Anderson - Institute for Applied Cancer Science", - "Nationwide Children's Hospital BCR", - "University of California, Santa Cruz", - "University of North Carolina", - "Washington University School of Medicine", - "Wellcome Trust Sanger Institute" - ], - "center_type": [ - "BCR", - "CGCC", - "GSC" - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw sequencing data", - "Simple nucleotide variation" - ], - "data_format": [ - "BAM", - "BCR XML", - "dat", - "SVS", - "TXT", - "VCF", - "ZIP" - ], - "data_type": [ - "Aligned reads", - "Biospecimen Supplement", - "Clinical Supplement", - "Diagnostic image", - "Genotypes", - "Radiology image", - "Simple nucleotide variation", - "Tissue slide image" - ], - "experimental_strategy": [ - "AMPLICON", - "Bisulfite-Seq", - "DNA-Seq", - "Genotyping array", - "miRNA-Seq", - "RNA-Seq", - "VALIDATION", - "WGS", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "false", - "true" - ], - "platform": [ - "ABI SOLiD", - "Affymetrix SNP Array 6.0", - "Clinical", - "HiSeq X Ten", - "Illumina GA", - "Illumina HiSeq", - "Illumina Human 1M Duo", - "Illumina HumanHap550", - "Illumina MiSeq", - "Ion Torrent PGM", - "LS 454" - ], - "sample_type": [ - "01", - "02", - "03", - "05", - "06", - "07", - "10", - "11", - "12", - "14" - ], - "species": [ - "Homo sapien", - "Homo sapiens" - ] - }, - "Data_HG38": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - "BWA with Mark Duplicates and Cocleaning", - "BWA-aln", - "STAR 2-Pass" - ], - "center_code": [ - - ], - "center_name": [ - - ], - "center_type": [ - - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw Sequencing Data" - ], - "data_format": [ - "BAM", - "BCR XML" - ], - "data_type": [ - "Aligned Reads", - "Biospecimen Supplement", - "Clinical Supplement" - ], - "experimental_strategy": [ - "miRNA-Seq", - "RNA-Seq", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "true" - ], - "platform": [ - "Illumina" - ], - "sample_type": [ - "01", - "02", - "03", - "05", - "06", - "07", - "10", - "11", - "12", - "14" - ], - "species": [ - "Homo sapiens" - ] - }, - "Annotation": { - "category": [ - "Acceptable treatment for TCGA tumor", - "Administrative Compliance", - "Alternate sample pipeline", - "Barcode incorrect", - "BCR Notification", - "Case submitted is found to be a recurrence after submission", - "Center QC failed", - "Duplicate item", - "General", - "Genotype mismatch", - "History of acceptable prior treatment related to a prior/other malignancy", - "History of unacceptable prior treatment related to a prior/other malignancy", - "Item does not meet study protocol", - "Item flagged DNU", - "Item Flagged Low Quality", - "Item in special subset", - "Item is noncanonical", - "Item may not meet study protocol", - "Molecular analysis outside specification", - "Neoadjuvant therapy", - "Normal class but appears diseased", - "Normal tissue origin incorrect", - "Pathology outside specification", - "Permanently missing item or object", - "Prior malignancy", - "Qualification metrics changed", - "Qualified in error", - "Sample compromised", - "Subject withdrew consent", - "Synchronous malignancy", - "Tumor class but appears normal", - "Tumor tissue origin incorrect", - "Tumor type incorrect" - ], - "classification": [ - "CenterNotification", - "Notification", - "Observation", - "Redaction" - ], - "entity_type": [ - "aliquot", - "analyte", - "case", - "portion", - "sample", - "slide" - ], - "status": [ - "Approved", - "Rescinded" - ] - } -} diff --git a/api_3/isb_cgc_api_TCGA/annotations_api.py b/api_3/isb_cgc_api_TCGA/annotations_api.py deleted file mode 100755 index 3f9bef9e..00000000 --- a/api_3/isb_cgc_api_TCGA/annotations_api.py +++ /dev/null @@ -1,100 +0,0 @@ -''' -Created on Mar 23, 2017 - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import MySQLdb - -import endpoints -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.cohort_endpoint_helpers import build_constructor_dict_for_message -from api_3.isb_cgc_api_TCGA.message_classes import AnnotationMetadataItem -from api_3.api_helpers import sql_connection - -class MetadataAnnotationList(messages.Message): - items = messages.MessageField(AnnotationMetadataItem, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - -class AnnotationAPI(remote.Service): - def process_annotations(self, request, barcode_type, annotationQueryBuilder, logger): - """ - Base class method to return TCGA annotations about a specific case, sample, or aliquot, - Takes a barcode as a required parameter. User does not need to be authenticated. - """ - try: - cursor = None - db = None - request_barcode = request.get_assigned_value(barcode_type) - query_tuple = str(request_barcode), - try: - self.validate_barcode(request_barcode) - except AssertionError: - raise endpoints.BadRequestException( - '{0} is not the correct format for the {1} barcode.'.format(request_barcode, barcode_type)) - entity_types = request.get_assigned_value('entity_type') - - # check to make sure each entity type is valid - if len(entity_types) > 0: - for itm in entity_types: - itm = itm.strip() - if itm.lower() not in ['case', 'aliquot', 'analyte', 'portion', 'slide', 'sample']: - raise endpoints.BadRequestException("'{}' is not a valid entry for entity_type. " - "Valid entries are 'case', 'aliquot', 'analyte', 'portion', 'slide', and 'sample'". - format(itm)) - query_tuple += itm, - - query_str = annotationQueryBuilder.build_query(entity_types=entity_types) - metadata_query_str = annotationQueryBuilder.build_metadata_query() - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - # build annotation message - cursor.execute(query_str, query_tuple) - rows = cursor.fetchall() - cursor.execute(metadata_query_str, (str(request_barcode), )) - metadata_rows = cursor.fetchall() - if len(rows) == 0: - cursor.close() - db.close() - if len(metadata_rows) == 0: - msg = "{} {} not found in the database.".format(barcode_type, request_barcode) - logger.info(msg) - else: - msg = "No annotations found for {} {}".format(barcode_type, request_barcode) - if entity_types is not None and 0 < len(entity_types): - msg += " and entity_type {}. entity_type name must be one of the following: "\ - "'case', 'aliquot', 'analyte', 'portion', 'slide', 'sample'.".format(entity_types) - logger.info(msg) - raise endpoints.NotFoundException(msg) - items = [] - for row in rows: - constructor_dict = build_constructor_dict_for_message(AnnotationMetadataItem(), row) - items.append(AnnotationMetadataItem(**constructor_dict)) - - return MetadataAnnotationList(items=items, count=len(items)) - - except (IndexError, TypeError), e: - logger.info("{} {} not found. Error: {}".format(barcode_type, request_barcode, e)) - raise endpoints.NotFoundException("{} {} not found.".format(barcode_type, request_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving {} data: {}".format(barcode_type, e)) - raise endpoints.BadRequestException("Error retrieving {} data: {}".format(barcode_type, e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api_3/isb_cgc_api_TCGA/cohorts_create.py b/api_3/isb_cgc_api_TCGA/cohorts_create.py deleted file mode 100644 index 3b8071dd..00000000 --- a/api_3/isb_cgc_api_TCGA/cohorts_create.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsCreateHelper, CreatedCohort -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.isb_cgc_api_TCGA.message_classes import MetadataRangesItem - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cohorts') -class TCGACohortsCreateAPI(CohortsCreateHelper): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='tcga/cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - self.program = 'TCGA' - return super(TCGACohortsCreateAPI, self).create(request) diff --git a/api_3/isb_cgc_api_TCGA/cohorts_preview.py b/api_3/isb_cgc_api_TCGA/cohorts_preview.py deleted file mode 100644 index c1a936df..00000000 --- a/api_3/isb_cgc_api_TCGA/cohorts_preview.py +++ /dev/null @@ -1,38 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsPreviewHelper -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from message_classes import MetadataRangesItem - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cohorts') -class TCGACohortsPreviewAPI(CohortsPreviewHelper): - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, fields=messages.StringField(3)) - - @endpoints.method(POST_RESOURCE, CohortsPreviewHelper.CohortCasesSamplesList, path='tcga/cohorts/preview', http_method='POST') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - self.program = 'TCGA' - return super(TCGACohortsPreviewAPI, self).preview(request) diff --git a/api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py b/api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py deleted file mode 100644 index c9fc093e..00000000 --- a/api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_TCGA_Endpoints = endpoints.api(name='isb_cgc_tcga_api', version='v3', - description="Get information about cohorts, cases, and samples for TCGA. Create cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - package_path='tcga', - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC TCGA API") diff --git a/api_3/isb_cgc_api_TCGA/message_classes.py b/api_3/isb_cgc_api_TCGA/message_classes.py deleted file mode 100644 index 3fba55c9..00000000 --- a/api_3/isb_cgc_api_TCGA/message_classes.py +++ /dev/null @@ -1,532 +0,0 @@ -from protorpc import messages - -class CommonMetadataRangesItem(messages.Message): - disease_code = messages.StringField(1, repeated=True) - endpoint_type = messages.StringField(2, repeated=True) - program_name = messages.StringField(3, repeated=True) - project_short_name = messages.StringField(4, repeated=True) - -class CommonMetadataItem(messages.Message): - disease_code = messages.StringField(1) - endpoint_type = messages.StringField(2) - program_name = messages.StringField(3) - project_short_name = messages.StringField(4) - -class ClinicalMetadataRangesItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, repeated=True, variant=messages.Variant.INT32) - age_at_diagnosis_lte = messages.IntegerField(2, variant=messages.Variant.INT32) - age_at_diagnosis_gte = messages.IntegerField(3, variant=messages.Variant.INT32) - - age_began_smoking_in_years = messages.IntegerField(4, repeated=True, variant=messages.Variant.INT32) - age_began_smoking_in_years_lte = messages.IntegerField(5, variant=messages.Variant.INT32) - age_began_smoking_in_years_gte = messages.IntegerField(6, variant=messages.Variant.INT32) - - anatomic_neoplasm_subdivision = messages.StringField(7, repeated=True) - batch_number = messages.IntegerField(8, repeated=True, variant=messages.Variant.INT32) - batch_number_lte = messages.IntegerField(9, variant=messages.Variant.INT32) - batch_number_gte = messages.IntegerField(10, variant=messages.Variant.INT32) - - bcr = messages.StringField(11, repeated=True) - bmi = messages.FloatField(12, repeated=True) - bmi_lte = messages.FloatField(13) - bmi_gte = messages.FloatField(14) - - case_barcode = messages.StringField(15, repeated=True) - case_gdc_id = messages.StringField(16, repeated=True) - clinical_M = messages.StringField(17, repeated=True) - clinical_N = messages.StringField(18, repeated=True) - clinical_stage = messages.StringField(19, repeated=True) - clinical_T = messages.StringField(20, repeated=True) - colorectal_cancer = messages.StringField(21, repeated=True) - country = messages.StringField(22, repeated=True) - days_to_birth = messages.IntegerField(23, repeated=True, variant=messages.Variant.INT32) - days_to_birth_lte = messages.IntegerField(24, variant=messages.Variant.INT32) - days_to_birth_gte = messages.IntegerField(25, variant=messages.Variant.INT32) - - days_to_death = messages.IntegerField(26, repeated=True, variant=messages.Variant.INT32) - days_to_death_lte = messages.IntegerField(27, variant=messages.Variant.INT32) - days_to_death_gte = messages.IntegerField(28, variant=messages.Variant.INT32) - - days_to_initial_pathologic_diagnosis = messages.IntegerField(29, repeated=True, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_lte = messages.IntegerField(30, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_gte = messages.IntegerField(31, variant=messages.Variant.INT32) - - days_to_last_followup = messages.IntegerField(32, repeated=True, variant=messages.Variant.INT32) - days_to_last_followup_lte = messages.IntegerField(33, variant=messages.Variant.INT32) - days_to_last_followup_gte = messages.IntegerField(34, variant=messages.Variant.INT32) - - days_to_last_known_alive = messages.IntegerField(35, repeated=True, variant=messages.Variant.INT32) - days_to_last_known_alive_lte = messages.IntegerField(36, variant=messages.Variant.INT32) - days_to_last_known_alive_gte = messages.IntegerField(37, variant=messages.Variant.INT32) - - days_to_submitted_specimen_dx = messages.IntegerField(38, repeated=True, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_lte = messages.IntegerField(39, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_gte = messages.IntegerField(40, variant=messages.Variant.INT32) - - ethnicity = messages.StringField(41, repeated=True) - gender = messages.StringField(42, repeated=True) - gleason_score_combined = messages.IntegerField(43, repeated=True, variant=messages.Variant.INT32) - gleason_score_combined_lte = messages.IntegerField(44, variant=messages.Variant.INT32) - gleason_score_combined_gte = messages.IntegerField(45, variant=messages.Variant.INT32) - - height = messages.IntegerField(46, repeated=True, variant=messages.Variant.INT32) - height_lte = messages.IntegerField(47, variant=messages.Variant.INT32) - height_gte = messages.IntegerField(48, variant=messages.Variant.INT32) - - histological_type = messages.StringField(49, repeated=True) - history_of_colon_polyps = messages.StringField(50, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(51, repeated=True) - hpv_calls = messages.StringField(52, repeated=True) - hpv_status = messages.StringField(53, repeated=True) - h_pylori_infection = messages.StringField(54, repeated=True) - icd_10 = messages.StringField(55, repeated=True) - icd_o_3_histology = messages.StringField(56, repeated=True) - icd_o_3_site = messages.StringField(57, repeated=True) - lymphatic_invasion = messages.StringField(58, repeated=True) - lymphnodes_examined = messages.StringField(59, repeated=True) - lymphovascular_invasion_present = messages.StringField(60, repeated=True) - menopause_status = messages.StringField(61, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(62, repeated=True) - neoplasm_histologic_grade = messages.StringField(63, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(64, repeated=True) - number_of_lymphnodes_examined = messages.IntegerField(65, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_lte = messages.IntegerField(66, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_gte = messages.IntegerField(67, variant=messages.Variant.INT32) - - number_of_lymphnodes_positive_by_he = messages.IntegerField(68, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_lte = messages.IntegerField(69, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_gte = messages.IntegerField(70, variant=messages.Variant.INT32) - - number_pack_years_smoked = messages.IntegerField(71, repeated=True, variant=messages.Variant.INT32) - number_pack_years_smoked_lte = messages.IntegerField(72, variant=messages.Variant.INT32) - number_pack_years_smoked_gte = messages.IntegerField(73, variant=messages.Variant.INT32) - - other_dx = messages.StringField(74, repeated=True) - other_malignancy_anatomic_site = messages.StringField(75, repeated=True) - other_malignancy_histological_type = messages.StringField(76, repeated=True) - other_malignancy_type = messages.StringField(77, repeated=True) - pathologic_M = messages.StringField(78, repeated=True) - pathologic_N = messages.StringField(79, repeated=True) - pathologic_stage = messages.StringField(80, repeated=True) - pathologic_T = messages.StringField(81, repeated=True) - person_neoplasm_cancer_status = messages.StringField(82, repeated=True) - pregnancies = messages.StringField(83, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(84, repeated=True) - primary_therapy_outcome_success = messages.StringField(85, repeated=True) - psa_value = messages.FloatField(86, repeated=True) - psa_value_lte = messages.FloatField(87) - psa_value_gte = messages.FloatField(88) - - race = messages.StringField(89, repeated=True) - residual_tumor = messages.StringField(90, repeated=True) - stopped_smoking_year = messages.IntegerField(91, repeated=True, variant=messages.Variant.INT32) - stopped_smoking_year_lte = messages.IntegerField(92, variant=messages.Variant.INT32) - stopped_smoking_year_gte = messages.IntegerField(93, variant=messages.Variant.INT32) - - summary_file_count = messages.IntegerField(94, repeated=True, variant=messages.Variant.INT32) - summary_file_count_lte = messages.IntegerField(95, variant=messages.Variant.INT32) - summary_file_count_gte = messages.IntegerField(96, variant=messages.Variant.INT32) - - tobacco_smoking_history = messages.StringField(97, repeated=True) - tss_code = messages.StringField(98, repeated=True) - tumor_tissue_site = messages.StringField(99, repeated=True) - tumor_type = messages.StringField(100, repeated=True) - venous_invasion = messages.StringField(101, repeated=True) - vital_status = messages.StringField(102, repeated=True) - weight = messages.IntegerField(103, repeated=True, variant=messages.Variant.INT32) - weight_lte = messages.IntegerField(104, variant=messages.Variant.INT32) - weight_gte = messages.IntegerField(105, variant=messages.Variant.INT32) - - year_of_diagnosis = messages.IntegerField(106, repeated=True, variant=messages.Variant.INT32) - year_of_diagnosis_lte = messages.IntegerField(107, variant=messages.Variant.INT32) - year_of_diagnosis_gte = messages.IntegerField(108, variant=messages.Variant.INT32) - - year_of_tobacco_smoking_onset = messages.IntegerField(109, repeated=True, variant=messages.Variant.INT32) - year_of_tobacco_smoking_onset_lte = messages.IntegerField(110, variant=messages.Variant.INT32) - year_of_tobacco_smoking_onset_gte = messages.IntegerField(111, variant=messages.Variant.INT32) - - -class ClinicalMetadataItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, variant=messages.Variant.INT32) - age_began_smoking_in_years = messages.IntegerField(2, variant=messages.Variant.INT32) - anatomic_neoplasm_subdivision = messages.StringField(3) - batch_number = messages.IntegerField(4, variant=messages.Variant.INT32) - bcr = messages.StringField(5) - bmi = messages.FloatField(6) - case_barcode = messages.StringField(7) - case_gdc_id = messages.StringField(8) - clinical_M = messages.StringField(9) - clinical_N = messages.StringField(10) - clinical_stage = messages.StringField(11) - clinical_T = messages.StringField(12) - colorectal_cancer = messages.StringField(13) - country = messages.StringField(14) - days_to_birth = messages.IntegerField(15, variant=messages.Variant.INT32) - days_to_death = messages.IntegerField(16, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis = messages.IntegerField(17, variant=messages.Variant.INT32) - days_to_last_followup = messages.IntegerField(18, variant=messages.Variant.INT32) - days_to_last_known_alive = messages.IntegerField(19, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx = messages.IntegerField(20, variant=messages.Variant.INT32) - ethnicity = messages.StringField(21) - gender = messages.StringField(22) - gleason_score_combined = messages.IntegerField(23, variant=messages.Variant.INT32) - height = messages.IntegerField(24, variant=messages.Variant.INT32) - histological_type = messages.StringField(25) - history_of_colon_polyps = messages.StringField(26) - history_of_neoadjuvant_treatment = messages.StringField(27) - hpv_calls = messages.StringField(28) - hpv_status = messages.StringField(29) - h_pylori_infection = messages.StringField(30) - icd_10 = messages.StringField(31) - icd_o_3_histology = messages.StringField(32) - icd_o_3_site = messages.StringField(33) - lymphatic_invasion = messages.StringField(34) - lymphnodes_examined = messages.StringField(35) - lymphovascular_invasion_present = messages.StringField(36) - menopause_status = messages.StringField(37) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(38) - neoplasm_histologic_grade = messages.StringField(39) - new_tumor_event_after_initial_treatment = messages.StringField(40) - number_of_lymphnodes_examined = messages.IntegerField(41, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he = messages.IntegerField(42, variant=messages.Variant.INT32) - number_pack_years_smoked = messages.IntegerField(43, variant=messages.Variant.INT32) - other_dx = messages.StringField(44) - other_malignancy_anatomic_site = messages.StringField(45) - other_malignancy_histological_type = messages.StringField(46) - other_malignancy_type = messages.StringField(47) - pathologic_M = messages.StringField(48) - pathologic_N = messages.StringField(49) - pathologic_stage = messages.StringField(50) - pathologic_T = messages.StringField(51) - person_neoplasm_cancer_status = messages.StringField(52) - pregnancies = messages.StringField(53) - primary_neoplasm_melanoma_dx = messages.StringField(54) - primary_therapy_outcome_success = messages.StringField(55) - psa_value = messages.FloatField(56) - race = messages.StringField(57) - residual_tumor = messages.StringField(58) - stopped_smoking_year = messages.IntegerField(59, variant=messages.Variant.INT32) - summary_file_count = messages.IntegerField(60, variant=messages.Variant.INT32) - tobacco_smoking_history = messages.StringField(61) - tss_code = messages.StringField(62) - tumor_tissue_site = messages.StringField(63) - tumor_type = messages.StringField(64) - venous_invasion = messages.StringField(65) - vital_status = messages.StringField(66) - weight = messages.IntegerField(67, variant=messages.Variant.INT32) - year_of_diagnosis = messages.IntegerField(68, variant=messages.Variant.INT32) - year_of_tobacco_smoking_onset = messages.IntegerField(69, variant=messages.Variant.INT32) - -class BiospecimenMetadataRangesItem(messages.Message): - avg_percent_lymphocyte_infiltration = messages.FloatField(1, repeated=True) - avg_percent_lymphocyte_infiltration_lte = messages.FloatField(2) - avg_percent_lymphocyte_infiltration_gte = messages.FloatField(3) - - avg_percent_monocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_monocyte_infiltration_lte = messages.FloatField(5) - avg_percent_monocyte_infiltration_gte = messages.FloatField(6) - - avg_percent_necrosis = messages.FloatField(7, repeated=True) - avg_percent_necrosis_lte = messages.FloatField(8) - avg_percent_necrosis_gte = messages.FloatField(9) - - avg_percent_neutrophil_infiltration = messages.FloatField(10, repeated=True) - avg_percent_neutrophil_infiltration_lte = messages.FloatField(11) - avg_percent_neutrophil_infiltration_gte = messages.FloatField(12) - - avg_percent_normal_cells = messages.FloatField(13, repeated=True) - avg_percent_normal_cells_lte = messages.FloatField(14) - avg_percent_normal_cells_gte = messages.FloatField(15) - - avg_percent_stromal_cells = messages.FloatField(16, repeated=True) - avg_percent_stromal_cells_lte = messages.FloatField(17) - avg_percent_stromal_cells_gte = messages.FloatField(18) - - avg_percent_tumor_cells = messages.FloatField(19, repeated=True) - avg_percent_tumor_cells_lte = messages.FloatField(20) - avg_percent_tumor_cells_gte = messages.FloatField(21) - - avg_percent_tumor_nuclei = messages.FloatField(22, repeated=True) - avg_percent_tumor_nuclei_lte = messages.FloatField(23) - avg_percent_tumor_nuclei_gte = messages.FloatField(24) - - batch_number = messages.IntegerField(25, repeated=True, variant=messages.Variant.INT32) - batch_number_lte = messages.IntegerField(26, variant=messages.Variant.INT32) - batch_number_gte = messages.IntegerField(27, variant=messages.Variant.INT32) - - bcr = messages.StringField(28, repeated=True) - case_barcode = messages.StringField(29, repeated=True) - case_gdc_id = messages.StringField(30, repeated=True) - days_to_collection = messages.IntegerField(31, repeated=True, variant=messages.Variant.INT32) - days_to_collection_lte = messages.IntegerField(32, variant=messages.Variant.INT32) - days_to_collection_gte = messages.IntegerField(33, variant=messages.Variant.INT32) - - days_to_sample_procurement = messages.IntegerField(34, repeated=True, variant=messages.Variant.INT32) - days_to_sample_procurement_lte = messages.IntegerField(35, variant=messages.Variant.INT32) - days_to_sample_procurement_gte = messages.IntegerField(36, variant=messages.Variant.INT32) - - max_percent_lymphocyte_infiltration = messages.FloatField(37, repeated=True) - max_percent_lymphocyte_infiltration_lte = messages.FloatField(38) - max_percent_lymphocyte_infiltration_gte = messages.FloatField(39) - - max_percent_monocyte_infiltration = messages.FloatField(40, repeated=True) - max_percent_monocyte_infiltration_lte = messages.FloatField(41) - max_percent_monocyte_infiltration_gte = messages.FloatField(42) - - max_percent_necrosis = messages.FloatField(43, repeated=True) - max_percent_necrosis_lte = messages.FloatField(44) - max_percent_necrosis_gte = messages.FloatField(45) - - max_percent_neutrophil_infiltration = messages.FloatField(46, repeated=True) - max_percent_neutrophil_infiltration_lte = messages.FloatField(47) - max_percent_neutrophil_infiltration_gte = messages.FloatField(48) - - max_percent_normal_cells = messages.FloatField(49, repeated=True) - max_percent_normal_cells_lte = messages.FloatField(50) - max_percent_normal_cells_gte = messages.FloatField(51) - - max_percent_stromal_cells = messages.FloatField(52, repeated=True) - max_percent_stromal_cells_lte = messages.FloatField(53) - max_percent_stromal_cells_gte = messages.FloatField(54) - - max_percent_tumor_cells = messages.FloatField(55, repeated=True) - max_percent_tumor_cells_lte = messages.FloatField(56) - max_percent_tumor_cells_gte = messages.FloatField(57) - - max_percent_tumor_nuclei = messages.FloatField(58, repeated=True) - max_percent_tumor_nuclei_lte = messages.FloatField(59) - max_percent_tumor_nuclei_gte = messages.FloatField(60) - - min_percent_lymphocyte_infiltration = messages.FloatField(61, repeated=True) - min_percent_lymphocyte_infiltration_lte = messages.FloatField(62) - min_percent_lymphocyte_infiltration_gte = messages.FloatField(63) - - min_percent_monocyte_infiltration = messages.FloatField(64, repeated=True) - min_percent_monocyte_infiltration_lte = messages.FloatField(65) - min_percent_monocyte_infiltration_gte = messages.FloatField(66) - - min_percent_necrosis = messages.FloatField(67, repeated=True) - min_percent_necrosis_lte = messages.FloatField(68) - min_percent_necrosis_gte = messages.FloatField(69) - - min_percent_neutrophil_infiltration = messages.FloatField(70, repeated=True) - min_percent_neutrophil_infiltration_lte = messages.FloatField(71) - min_percent_neutrophil_infiltration_gte = messages.FloatField(72) - - min_percent_normal_cells = messages.FloatField(73, repeated=True) - min_percent_normal_cells_lte = messages.FloatField(74) - min_percent_normal_cells_gte = messages.FloatField(75) - - min_percent_stromal_cells = messages.FloatField(76, repeated=True) - min_percent_stromal_cells_lte = messages.FloatField(77) - min_percent_stromal_cells_gte = messages.FloatField(78) - - min_percent_tumor_cells = messages.FloatField(79, repeated=True) - min_percent_tumor_cells_lte = messages.FloatField(80) - min_percent_tumor_cells_gte = messages.FloatField(81) - - min_percent_tumor_nuclei = messages.FloatField(82, repeated=True) - min_percent_tumor_nuclei_lte = messages.FloatField(83) - min_percent_tumor_nuclei_gte = messages.FloatField(84) - - num_portions = messages.IntegerField(85, repeated=True, variant=messages.Variant.INT32) - num_portions_lte = messages.IntegerField(86, variant=messages.Variant.INT32) - num_portions_gte = messages.IntegerField(87, variant=messages.Variant.INT32) - - num_slides = messages.IntegerField(88, repeated=True, variant=messages.Variant.INT32) - num_slides_lte = messages.IntegerField(89, variant=messages.Variant.INT32) - num_slides_gte = messages.IntegerField(90, variant=messages.Variant.INT32) - - pathology_report_uuid = messages.StringField(91, repeated=True) - preservation_method = messages.StringField(92, repeated=True) - sample_barcode = messages.StringField(93, repeated=True) - sample_gdc_id = messages.StringField(94, repeated=True) - sample_type = messages.StringField(95, repeated=True) - -class BiospecimenMetadataItem(messages.Message): - avg_percent_lymphocyte_infiltration = messages.FloatField(1) - avg_percent_monocyte_infiltration = messages.FloatField(2) - avg_percent_necrosis = messages.FloatField(3) - avg_percent_neutrophil_infiltration = messages.FloatField(4) - avg_percent_normal_cells = messages.FloatField(5) - avg_percent_stromal_cells = messages.FloatField(6) - avg_percent_tumor_cells = messages.FloatField(7) - avg_percent_tumor_nuclei = messages.FloatField(8) - batch_number = messages.IntegerField(9, variant=messages.Variant.INT32) - bcr = messages.StringField(10) - case_barcode = messages.StringField(11) - case_gdc_id = messages.StringField(12) - days_to_collection = messages.IntegerField(13, variant=messages.Variant.INT32) - days_to_sample_procurement = messages.IntegerField(14, variant=messages.Variant.INT32) - max_percent_lymphocyte_infiltration = messages.FloatField(15) - max_percent_monocyte_infiltration = messages.FloatField(16) - max_percent_necrosis = messages.FloatField(17) - max_percent_neutrophil_infiltration = messages.FloatField(18) - max_percent_normal_cells = messages.FloatField(19) - max_percent_stromal_cells = messages.FloatField(20) - max_percent_tumor_cells = messages.FloatField(21) - max_percent_tumor_nuclei = messages.FloatField(22) - min_percent_lymphocyte_infiltration = messages.FloatField(23) - min_percent_monocyte_infiltration = messages.FloatField(24) - min_percent_necrosis = messages.FloatField(25) - min_percent_neutrophil_infiltration = messages.FloatField(26) - min_percent_normal_cells = messages.FloatField(27) - min_percent_stromal_cells = messages.FloatField(28) - min_percent_tumor_cells = messages.FloatField(29) - min_percent_tumor_nuclei = messages.FloatField(30) - num_portions = messages.IntegerField(31, variant=messages.Variant.INT32) - num_slides = messages.IntegerField(32, variant=messages.Variant.INT32) - pathology_report_uuid = messages.StringField(33) - preservation_method = messages.StringField(34) - sample_barcode = messages.StringField(35) - sample_gdc_id = messages.StringField(36) - sample_type = messages.StringField(37) - -class Data_HG19MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - aliquot_barcode = messages.StringField(2, repeated=True) - aliquot_gdc_id = messages.StringField(3, repeated=True) - analysis_workflow_type = messages.StringField(4, repeated=True) - archive_file_name = messages.StringField(5, repeated=True) - archive_submitter_id = messages.StringField(6, repeated=True) - case_barcode = messages.StringField(7, repeated=True) - case_gdc_id = messages.StringField(8, repeated=True) - center_code = messages.StringField(9, repeated=True) - center_name = messages.StringField(10, repeated=True) - center_type = messages.StringField(11, repeated=True) - data_category = messages.StringField(12, repeated=True) - data_format = messages.StringField(13, repeated=True) - data_type = messages.StringField(14, repeated=True) - experimental_strategy = messages.StringField(15, repeated=True) - file_name = messages.StringField(16, repeated=True) - file_state = messages.StringField(17, repeated=True) - file_uploaded = messages.StringField(18, repeated=True) - index_file_name = messages.StringField(19, repeated=True) - platform = messages.StringField(20, repeated=True) - sample_barcode = messages.StringField(21, repeated=True) - sample_gdc_id = messages.StringField(22, repeated=True) - sample_type = messages.StringField(23, repeated=True) - species = messages.StringField(24, repeated=True) - -class Data_HG19MetadataItem(messages.Message): - access = messages.StringField(1) - aliquot_barcode = messages.StringField(2) - aliquot_gdc_id = messages.StringField(3) - analysis_workflow_type = messages.StringField(4) - archive_file_name = messages.StringField(5) - archive_submitter_id = messages.StringField(6) - case_barcode = messages.StringField(7) - case_gdc_id = messages.StringField(8) - center_code = messages.StringField(9) - center_name = messages.StringField(10) - center_type = messages.StringField(11) - data_category = messages.StringField(12) - data_format = messages.StringField(13) - data_type = messages.StringField(14) - experimental_strategy = messages.StringField(15) - file_name = messages.StringField(16) - file_state = messages.StringField(17) - file_uploaded = messages.StringField(18) - index_file_name = messages.StringField(19) - platform = messages.StringField(20) - sample_barcode = messages.StringField(21) - sample_gdc_id = messages.StringField(22) - sample_type = messages.StringField(23) - species = messages.StringField(24) - -class Data_HG38MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - aliquot_barcode = messages.StringField(2, repeated=True) - aliquot_gdc_id = messages.StringField(3, repeated=True) - analysis_workflow_type = messages.StringField(4, repeated=True) - archive_file_name = messages.StringField(5, repeated=True) - archive_submitter_id = messages.StringField(6, repeated=True) - case_barcode = messages.StringField(7, repeated=True) - case_gdc_id = messages.StringField(8, repeated=True) - center_code = messages.StringField(9, repeated=True) - center_name = messages.StringField(10, repeated=True) - center_type = messages.StringField(11, repeated=True) - data_category = messages.StringField(12, repeated=True) - data_format = messages.StringField(13, repeated=True) - data_type = messages.StringField(14, repeated=True) - experimental_strategy = messages.StringField(15, repeated=True) - file_name = messages.StringField(16, repeated=True) - file_state = messages.StringField(17, repeated=True) - file_uploaded = messages.StringField(18, repeated=True) - index_file_name = messages.StringField(19, repeated=True) - platform = messages.StringField(20, repeated=True) - sample_barcode = messages.StringField(21, repeated=True) - sample_gdc_id = messages.StringField(22, repeated=True) - sample_type = messages.StringField(23, repeated=True) - species = messages.StringField(24, repeated=True) - -class Data_HG38MetadataItem(messages.Message): - access = messages.StringField(1) - aliquot_barcode = messages.StringField(2) - aliquot_gdc_id = messages.StringField(3) - analysis_workflow_type = messages.StringField(4) - archive_file_name = messages.StringField(5) - archive_submitter_id = messages.StringField(6) - case_barcode = messages.StringField(7) - case_gdc_id = messages.StringField(8) - center_code = messages.StringField(9) - center_name = messages.StringField(10) - center_type = messages.StringField(11) - data_category = messages.StringField(12) - data_format = messages.StringField(13) - data_type = messages.StringField(14) - experimental_strategy = messages.StringField(15) - file_name = messages.StringField(16) - file_state = messages.StringField(17) - file_uploaded = messages.StringField(18) - index_file_name = messages.StringField(19) - platform = messages.StringField(20) - sample_barcode = messages.StringField(21) - sample_gdc_id = messages.StringField(22) - sample_type = messages.StringField(23) - species = messages.StringField(24) - -class MetadataRangesItem(messages.Message): - Common = messages.MessageField(CommonMetadataRangesItem, 1) - Clinical = messages.MessageField(ClinicalMetadataRangesItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataRangesItem, 3) - Data_HG19 = messages.MessageField(Data_HG19MetadataRangesItem, 4) - Data_HG38 = messages.MessageField(Data_HG38MetadataRangesItem, 5) - -class MetadataItem(messages.Message): - Common = messages.MessageField(CommonMetadataItem, 1) - Clinical = messages.MessageField(ClinicalMetadataItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataItem, 3) - Data_HG19 = messages.MessageField(Data_HG19MetadataItem, 4) - Data_HG38 = messages.MessageField(Data_HG38MetadataItem, 5) - -class AnnotationMetadataRangesItem(messages.Message): - aliquot_barcode = messages.StringField(1, repeated=True) - annotation_gdc_id = messages.StringField(2, repeated=True) - annotation_submitter_id = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4, repeated=True) - case_gdc_id = messages.StringField(5, repeated=True) - category = messages.StringField(6, repeated=True) - classification = messages.StringField(7, repeated=True) - entity_barcode = messages.StringField(8, repeated=True) - entity_gdc_id = messages.StringField(9, repeated=True) - entity_type = messages.StringField(10, repeated=True) - notes = messages.StringField(11, repeated=True) - sample_barcode = messages.StringField(12, repeated=True) - status = messages.StringField(13, repeated=True) - -class AnnotationMetadataItem(messages.Message): - aliquot_barcode = messages.StringField(1) - annotation_gdc_id = messages.StringField(2) - annotation_submitter_id = messages.StringField(3) - case_barcode = messages.StringField(4) - case_gdc_id = messages.StringField(5) - category = messages.StringField(6) - classification = messages.StringField(7) - entity_barcode = messages.StringField(8) - entity_gdc_id = messages.StringField(9) - entity_type = messages.StringField(10) - notes = messages.StringField(11) - sample_barcode = messages.StringField(12) - status = messages.StringField(13) - diff --git a/api_3/isb_cgc_api_TCGA/patients_annotations.py b/api_3/isb_cgc_api_TCGA/patients_annotations.py deleted file mode 100644 index a8a9f9be..00000000 --- a/api_3/isb_cgc_api_TCGA/patients_annotations.py +++ /dev/null @@ -1,72 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import logging - -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TCGA.annotations_api import AnnotationAPI, MetadataAnnotationList -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -logger = logging.getLogger(__name__) - - -class CasesAnnotationsQueryBuilder(object): - - @staticmethod - def build_query(entity_types=None): - query_str = 'select * ' \ - 'from TCGA_metadata_annotation ' \ - 'where case_barcode=%s and status = "Approved" ' - if len(entity_types) > 0: - query_str += 'and entity_type in (' + ', '.join(['%s']*len(entity_types)) + ')' - - return query_str - - @staticmethod - def build_metadata_query(): - query_str = 'select case_barcode ' \ - 'from TCGA_metadata_clinical ' \ - 'where case_barcode=%s ' - - return query_str - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cases') -class TCGACasesAnnotationAPI(AnnotationAPI): - - GET_RESOURCE = endpoints.ResourceContainer(case_barcode=messages.StringField(1, required=True), - entity_type=messages.StringField(2, repeated=True)) - - @endpoints.method(GET_RESOURCE, MetadataAnnotationList, - path='cases/{case_barcode}/annotations', http_method='GET') - def annotations(self, request): - """ - Returns TCGA annotations about a specific sample, - Takes a case barcode (of length 12, *eg* TCGA-01-0628) as a required parameter. - User does not need to be authenticated. - """ - return self.process_annotations(request, 'case_barcode', CasesAnnotationsQueryBuilder(), logger) - - - def validate_barcode(self, case_barcode): - # check to make sure case_barcode is in correct form - parts = case_barcode.split('-') - assert len(parts) == 3 - assert len(parts[0]) == 4 - assert len(parts[1]) == 2 - assert len(parts[2]) == 4 diff --git a/api_3/isb_cgc_api_TCGA/patients_get.py b/api_3/isb_cgc_api_TCGA/patients_get.py deleted file mode 100644 index 1694378c..00000000 --- a/api_3/isb_cgc_api_TCGA/patients_get.py +++ /dev/null @@ -1,55 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.patients_get_helper import CasesGetHelper -from api_3.isb_cgc_api_TCGA.message_classes import ClinicalMetadataItem as MetadataItem -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -class CaseDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4) - -class CaseSetDetails(messages.Message): - cases = messages.MessageField(CaseDetails, 1, repeated=True) - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cases') -class TCGACasesGetAPI(CasesGetHelper): - @endpoints.method(CasesGetHelper.GET_RESOURCE, CaseDetails, path='tcga/cases/{case_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - return super(TCGACasesGetAPI, self).get(request, CaseDetails, MetadataItem, 'TCGA') - - @endpoints.method(CasesGetHelper.POST_RESOURCE, CaseSetDetails, path='tcga/cases', http_method='POST') - def get_list(self, request): - """ - Given a list of case barcodes (of length 11, *eg* TCGA-B9-7268), this endpoint returns - all available "biospecimen" information about the cases, including a list of samples and aliquots - derived from them. - Takes a list of case barcodes (of length 12, *eg* TCGA-B9-7268) as a required data payload. - User does not need to be authenticated. - """ - return super(TCGACasesGetAPI, self).get_list(request, CaseSetDetails, CaseDetails, MetadataItem, 'TCGA') diff --git a/api_3/isb_cgc_api_TCGA/samples_annotations.py b/api_3/isb_cgc_api_TCGA/samples_annotations.py deleted file mode 100644 index 1d4ec3c3..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_annotations.py +++ /dev/null @@ -1,74 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import logging - -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TCGA.annotations_api import AnnotationAPI, MetadataAnnotationList -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -logger = logging.getLogger(__name__) - - -class SamplesAnnotationsQueryBuilder(object): - - @staticmethod - def build_query(entity_types=None): - query_str = 'select * ' \ - 'from TCGA_metadata_annotation ' \ - 'where sample_barcode=%s and status = "Approved" ' - if len(entity_types) > 0: - query_str += 'and entity_type in (' + ', '.join(['%s']*len(entity_types)) + ')' - - return query_str - - @staticmethod - def build_metadata_query(): - query_str = 'select sample_barcode ' \ - 'from TCGA_metadata_biospecimen ' \ - 'where sample_barcode=%s ' - - return query_str - - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class TCGASamplesAnnotationAPI(AnnotationAPI): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - entity_type=messages.StringField(2, repeated=True)) - - @endpoints.method(GET_RESOURCE, MetadataAnnotationList, - path='samples/{sample_barcode}/annotations', http_method='GET') - def annotations(self, request): - """ - Returns TCGA annotations about a specific sample, - Takes a sample barcode (of length 16, *eg* TCGA-01-0628-11A) as a required parameter. - User does not need to be authenticated. - """ - return self.process_annotations(request, 'sample_barcode', SamplesAnnotationsQueryBuilder(), logger) - - - def validate_barcode(self, sample_barcode): - # check to make sure sample_barcode is in correct form - parts = sample_barcode.split('-') - assert len(parts) == 4 - assert len(parts[0]) == 4 - assert len(parts[1]) == 2 - assert len(parts[2]) == 4 - assert len(parts[3]) == 3 diff --git a/api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py b/api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py deleted file mode 100644 index 6d8b4f3f..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, SamplesCloudStorageFilePathsHelper - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class TCGASamplesCloudStorageFilePathsAPI(SamplesCloudStorageFilePathsHelper): - - @endpoints.method(SamplesCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, - path='tcga/samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - return super(TCGASamplesCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request, 'TCGA') diff --git a/api_3/isb_cgc_api_TCGA/samples_get.py b/api_3/isb_cgc_api_TCGA/samples_get.py deleted file mode 100644 index fa44d0a6..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_get.py +++ /dev/null @@ -1,60 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.samples_get_helper import SamplesGetAPI, DataDetails -from api_3.isb_cgc_api_TCGA.message_classes import BiospecimenMetadataItem - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(BiospecimenMetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - case_barcode = messages.StringField(3) - case_gdc_id = messages.StringField(4) - data_details = messages.MessageField(DataDetails, 5, repeated=True) - data_details_count = messages.IntegerField(6, variant=messages.Variant.INT32) - sample_barcode = messages.StringField(7) - - -class SampleSetDetails(messages.Message): - samples = messages.MessageField(SampleDetails, 1, repeated=True) - - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class TCGASamplesGetAPI(SamplesGetAPI): - @endpoints.method(SamplesGetAPI.GET_RESOURCE, SampleDetails, path='tcga/samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TCGASamplesGetAPI, self).get(request, 'TCGA', SampleDetails, BiospecimenMetadataItem) - - @endpoints.method(SamplesGetAPI.POST_RESOURCE, SampleSetDetails, path='tcga/samples', http_method='POST') - def get_list(self, request): - """ - Given a list of sample barcodes (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TCGASamplesGetAPI, self).get_list(request, 'TCGA', SampleSetDetails, SampleDetails, BiospecimenMetadataItem) diff --git a/api_3/isb_cgc_api_TCGA/samples_googlegenomics.py b/api_3/isb_cgc_api_TCGA/samples_googlegenomics.py deleted file mode 100644 index 27a1bfc3..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api_3/isb_cgc_api_TCGA/users_get.py b/api_3/isb_cgc_api_TCGA/users_get.py deleted file mode 100644 index a1ee020a..00000000 --- a/api_3/isb_cgc_api_TCGA/users_get.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import message_types - -from api_3.users_get_common import UserGetAPICommon, UserGetAPIReturnJSON -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='users') -class TCGAUserGetAPI(UserGetAPICommon): - - @endpoints.method(message_types.VoidMessage, UserGetAPIReturnJSON, http_method='GET', path='tcga/users') - def get(self, _): - ''' - Returns the dbGaP authorization status of the user. - ''' - return super(TCGAUserGetAPI, self).get('TCGA') diff --git a/api_3/maf.py b/api_3/maf.py deleted file mode 100755 index 19e61845..00000000 --- a/api_3/maf.py +++ /dev/null @@ -1,104 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -from protorpc import remote -from protorpc.messages import Message, MessageField, IntegerField, StringField, Variant -from MySQLdb.cursors import DictCursor - -from api_helpers import * - - -# Generated code - in scripts/maf_import in Python do: -# -# > import import_maf -# > repr(import_maf.FIELD_NAMES_AND_TYPES) -# -FIELD_NAMES_AND_TYPES = [ - {'type': 'varchar(128)', 'name': 'hugo_symbol'}, - {'type': 'int', 'name': 'amino_acid_position'}, - {'type': 'varchar(128)', 'name': 'tumor_type'}, - {'type': 'varchar(12)', 'name': 'uniprot_id'}, - {'type': 'varchar(128)', 'name': 'variant_classification'}, - {'type': 'varchar(128)', 'name': 'c_position'}, - {'type': 'varchar(128)', 'name': 'tumor_sample_barcode'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'reference_allele'}, - {'type': 'varchar(128)', 'name': 'variant_type'}, - {'type': 'varchar(128)', 'name': 'ucsc_cons'} -] - - -class MAFRecord(Message): - hugo_symbol = StringField(1) - tumor_type = StringField(3) - amino_acid_position = IntegerField(2, variant=Variant.INT32) - uniprot_id = StringField(4) - variant_classification = StringField(5) - c_position = StringField(6) - tumor_sample_barcode = StringField(7) - match_norm_seq_allele2 = StringField(8) - tumor_seq_allele1 = StringField(9) - tumor_seq_allele2 = StringField(10) - match_norm_seq_allele1 = StringField(11) - reference_allele = StringField(12) - variant_type = StringField(13) - ucsc_cons = StringField(14) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - - -class MAFRequest(Message): - gene = StringField(1, required=True) - tumor = StringField(2, repeated=True) - -MAFEndpointsAPI = endpoints.api(name='maf_api', version='v3') - - -@MAFEndpointsAPI .api_class(resource_name='maf_endpoints') -class MAFEndpointsAPI(remote.Service): - @endpoints.method(MAFRequest, MAFRecordList, - path='maf_search', http_method='GET', name='maf.getMAF') - def maf_search(self, request): - gene = request.gene - tumor_type_list = request.tumor - tumor_set_template = ', '.join(['%s' for x in range(len(tumor_type_list))]) - query = 'SELECT * FROM maf WHERE hugo_symbol=%s AND tumor_type IN ({0})'.format(tumor_set_template) - - values = [gene] - values.extend(tumor_type_list) - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - data_list = [] - for row in cursor.fetchall(): - data_list.append(MAFRecord(**row)) - - cursor.close() - db.close() - return MAFRecordList(items=data_list) - - except: - raise endpoints.NotFoundException('MAF query error') \ No newline at end of file diff --git a/api_3/message_generator.py b/api_3/message_generator.py deleted file mode 100755 index e35e1f9a..00000000 --- a/api_3/message_generator.py +++ /dev/null @@ -1,243 +0,0 @@ -''' -copyright 2017, Institute for Systems Biology. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' -from argparse import ArgumentParser -from datetime import datetime -import MySQLdb - -def get_sql_connection(args): - try: - connect_options = { - 'host': args.host, - 'db': args.database, - 'user': args.user, - 'passwd': args.password, - 'ssl': { - 'ca': args.ssl_ca, - 'key': args.ssl_key, - 'cert': args.ssl_cert - } - } - db = MySQLdb.connect(**connect_options) - except: - print 'failed' - raise - return db - -FIELD_TYPES ={ - 'varchar': 'StringField', - 'float': 'FloatField', - 'tinyint': 'BooleanField', - 'int': 'IntegerField' -} - -seen_rows = set() -def write_allowed_values(cursor, rows, path, program, metadata_type, column_filter, append_file=True, complete_file=False, write_file=False): - if not write_file: - return - - MAX_VALUES = 70 - try: - mode = 'a+' if append_file else 'w' - with open(path, mode) as av: - if not append_file: - av.write('{\n') - av.write(' "{}": {{\n'.format(metadata_type)) - table_name = 'Clinical' if 'Common' == metadata_type else metadata_type - query = 'select {} from %s_metadata_%s group by 1 order by 1' % (program, table_name[:1].lower() + table_name[1:]) - body = '' - for row in rows: -# if row['DATA_TYPE'] != 'varchar' or row['COLUMN_NAME'] in seen_rows: - if row['DATA_TYPE'] != 'varchar': - continue - if ('Common' in metadata_type and row['COLUMN_NAME'] not in column_filter) or ('Common' not in metadata_type and row['COLUMN_NAME'] in column_filter): - continue - - seen_rows.add(row['COLUMN_NAME']) - cursor.execute(query.format(row['COLUMN_NAME'])) - currows = cursor.fetchall() - if MAX_VALUES < len(currows): - continue - if 1 == len(currows) and 'None' == currows[0][row['COLUMN_NAME']]: - continue - - body += ' "{}": [\n {}\n ],\n'.format(row['COLUMN_NAME'], ' '.join('"{}",\n'.format(currow[row['COLUMN_NAME']]) for currow in currows if not currow[row['COLUMN_NAME']] is None)[:-2]) - av.write(body[:-2]) - av.write('\n }') - - if complete_file: - av.write('\n}\n') - else: - av.write(',\n') - av.flush() - return - except: - cursor.close() - raise - -def create_nesting_class(table_list, path, write_file): - if not write_file: - return - - with open(path, 'a') as f: - ranges_body = 'class MetadataRangesItem(messages.Message):\n' - items_body = 'class MetadataItem(messages.Message):\n' - for index, table in enumerate(table_list): - ranges_body += ' {0} = messages.MessageField({0}MetadataRangesItem, {1})\n'.format(table, index+1) - items_body += ' {0} = messages.MessageField({0}MetadataItem, {1})\n'.format(table, index+1) - f.write('{}\n'.format(ranges_body)) - f.write('{}\n'.format(items_body)) - -def write_metadata_file(class_name, rows, path, append_file=True, write_file=False): - - ranges_text = 'class {}RangesItem(messages.Message):\n '.format(class_name) - - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if not field_type: - continue - ranges_text += '%-65s = messages.%s(%d, repeated=True' % (row['COLUMN_NAME'], field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if field_type == 'IntegerField' or field_type == 'FloatField': - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_lte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_gte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '\n ' - - item_text = '\nclass {}Item(messages.Message):\n '.format(class_name) - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if not field_type: - continue - item_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME'], field_type, i) - item_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if write_file is True: - mode = 'a' if append_file else 'w' - with open(path, mode) as f: - if not append_file: - f.write('from protorpc import messages\n\n') - f.write(ranges_text) - f.write(item_text + '\n') - else: - print path + '\n' - print ranges_text - print '\n\n' - print item_text - -def get_table_column_info(cursor, program, metadata_type, column_filter = []): - key_part = metadata_type.split('_')[0].lower() - table_name = 'Clinical' if 'Common' == metadata_type else metadata_type - query_str = 'SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'FROM INFORMATION_SCHEMA.COLUMNS ' \ - 'WHERE table_name = "{}_metadata_{}" ' \ - 'AND COLUMN_NAME != "metadata_{}_id" ' + 'group by COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'ORDER BY column_name' - cursor.execute(query_str.format(program, table_name[:1].lower() + table_name[1:], key_part)) - rows = cursor.fetchall() - retrows = [] - seen = set() - for row in rows: - if ('Common' in metadata_type and row['COLUMN_NAME'] not in column_filter) or ('Common' not in metadata_type and row['COLUMN_NAME'] in column_filter): - continue - if row['COLUMN_NAME'] in seen: - continue - seen.add(row['COLUMN_NAME']) - retrows += [row] - return retrows - -def create_metadata_file(cursor, program, metadata_type, path, column_filter, append_file=True, complete_file=False, write_file=True): - rows = get_table_column_info(cursor, program, metadata_type, column_filter) - write_metadata_file(metadata_type + 'Metadata', rows, path, append_file, write_file) - path = 'api_3/isb_cgc_api_{0}/allowed_values_v3_{0}.json'.format(program) - write_allowed_values(cursor, rows, path, program, metadata_type, column_filter, append_file, complete_file, write_file) - return rows - -def main(args): - db = get_sql_connection(args) - cursor = None - try: - path_template = 'api_3/isb_cgc_api_%s/message_classes.py' - programs = [ - ('CCLE', False), - ('TARGET', False), - ('TCGA', True), - ] - for program in programs: - print datetime.now(), 'starting program {}'.format(program) - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - write_file = not bool(args.dry_run) - # for the columns that are common to all tables and have the same values from the subsequent tables, get them here and any filters - column_filter = ["disease_code", "endpoint_type", "program_name", "project_short_name"] - create_metadata_file(cursor, program[0], 'Common', path_template % (program[0]), column_filter, False, False, write_file) - column_filter += [ - 'acl', - 'analysis_gdc_id', - 'analysis_workflow_link', - 'archive_gdc_id', - 'archive_revision', - 'archive_revision_lte', - 'archive_revision_gte', - 'file_gdc_id', - 'file_name_key', - 'index_file_id', - 'md5sum', - 'type' - ] - create_metadata_file(cursor, program[0], 'Clinical', path_template % (program[0]), column_filter, True, False, write_file) - column_filter += ["disease_code", "endpoint_type", "program_name", "project_short_name"] - create_metadata_file(cursor, program[0], 'Biospecimen', path_template % (program[0]), column_filter, True, False, write_file) - table_list = ['Common', 'Clinical', 'Biospecimen', 'Data_HG19'] - create_metadata_file(cursor, program[0], 'Data_HG19', path_template % (program[0]), column_filter, True, True if 'CCLE' == program[0] else False, write_file) - if 'CCLE' != program[0]: - create_metadata_file(cursor, program[0], 'Data_HG38', path_template % (program[0]), column_filter, True, False if program[1] else True, write_file) - table_list += ['Data_HG38'] - create_nesting_class(table_list, path_template % (program[0]), write_file) - if program[1]: - create_metadata_file(cursor, program[0], 'Annotation', path_template % (program[0]), column_filter, True, True, write_file) - print datetime.now(), 'finished program {}'.format(program) - finally: - if cursor: - cursor.close() - db.close() - -if __name__ == '__main__': - parser = ArgumentParser() - parser.add_argument('--host', help='database host') - parser.add_argument('--database', '-d', help='database name') - parser.add_argument('--user', '-u', help='database username') - parser.add_argument('--password', '-p', help='database password') - parser.add_argument('--ssl_key') - parser.add_argument('--ssl_cert') - parser.add_argument('--ssl_ca') - parser.add_argument('--dry_run', dest='dry_run', action='store_true') - parser.add_argument('--write_file', dest='dry_run', action='store_false') - parser.set_defaults(feature=True) - args = parser.parse_args() - - print datetime.now(), 'starting...' - main(args) - print datetime.now(), 'finished' - - diff --git a/api_3/metadata.py b/api_3/metadata.py deleted file mode 100755 index 880203a8..00000000 --- a/api_3/metadata.py +++ /dev/null @@ -1,1915 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import time -from protorpc import messages -from protorpc import message_types -from protorpc import remote -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from allauth.socialaccount.models import SocialToken, SocialAccount -from accounts.models import NIH_User -from cohorts.models import Cohort_Perms, Cohort as Django_Cohort, Samples, Filters -from projects.models import Project, User_Feature_Definitions, User_Feature_Counts, User_Data_Tables -from django.core.signals import request_finished -from time import sleep -import django -import sys -import logging -import re -import json -import traceback -import copy -from uuid import uuid4 - -from api_3.api_helpers import * - -logger = logging.getLogger(__name__) - -debug = settings.DEBUG - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID -OPEN_DATA_BUCKET = settings.OPEN_DATA_BUCKET -#CONTROLLED_DATA_BUCKET = settings.CONTROLLED_DATA_BUCKET - -METADATA_SHORTLIST = [ - # 'adenocarcinoma_invasion', - 'age_at_diagnosis', - # 'anatomic_neoplasm_subdivision', - # 'avg_percent_lymphocyte_infiltration', - # 'avg_percent_monocyte_infiltration', - # 'avg_percent_necrosis', - # 'avg_percent_neutrophil_infiltration', - # 'avg_percent_normal_cells', - # 'avg_percent_stromal_cells', - # 'avg_percent_tumor_cells', - # 'avg_percent_tumor_nuclei', - # 'batch_number', - # 'bcr', - 'BMI', - # 'clinical_M', - # 'clinical_N', - # 'clinical_stage', - # 'clinical_T', - # 'colorectal_cancer', - # 'country', - # 'country_of_procurement', - # 'days_to_birth', - # 'days_to_collection', - # 'days_to_death', - # 'days_to_initial_pathologic_diagnosis', - # 'days_to_last_followup', - # 'days_to_submitted_specimen_dx', - 'Study', - # 'ethnicity', - # 'frozen_specimen_anatomic_site', - 'gender', - # 'gleason_score_combined', - 'has_27k', - 'has_450k', - 'has_BCGSC_GA_RNASeq', - 'has_BCGSC_HiSeq_RNASeq', - 'has_GA_miRNASeq', - 'has_HiSeq_miRnaSeq', - 'has_Illumina_DNASeq', - 'has_RPPA', - 'has_SNP6', - 'has_UNC_GA_RNASeq', - 'has_UNC_HiSeq_RNASeq', - # 'height', - 'histological_type', - # 'history_of_colon_polyps', - # 'history_of_neoadjuvant_treatment', - # 'history_of_prior_malignancy', - # 'hpv_calls', - 'hpv_status', - 'icd_10', - 'icd_o_3_histology', - 'icd_o_3_site', - # 'lymph_node_examined_count', - # 'lymphatic_invasion', - # 'lymphnodes_examined', - # 'lymphovascular_invasion_present', - # 'max_percent_lymphocyte_infiltration', - # 'max_percent_monocyte_infiltration', - # 'max_percent_necrosis', - # 'max_percent_neutrophil_infiltration', - # 'max_percent_normal_cells', - # 'max_percent_stromal_cells', - # 'max_percent_tumor_cells', - # 'max_percent_tumor_nuclei', - # 'menopause_status', - # 'min_percent_lymphocyte_infiltration', - # 'min_percent_monocyte_infiltration', - # 'min_percent_necrosis', - # 'min_percent_neutrophil_infiltration', - # 'min_percent_normal_cells', - # 'min_percent_stromal_cells', - # 'min_percent_tumor_cells', - # 'min_percent_tumor_nuclei', - # 'mononucleotide_and_dinucleotide_marker_panel_analysis_status', - # 'mononucleotide_marker_panel_analysis_status', - 'neoplasm_histologic_grade', - 'new_tumor_event_after_initial_treatment', - # 'number_of_lymphnodes_examined', - # 'number_of_lymphnodes_positive_by_he', - # 'number_pack_years_smoked', - # 'other_dx', - # 'case_barcode', - # 'pathologic_M', - # 'pathologic_N', - 'pathologic_stage', - # 'pathologic_T', - 'person_neoplasm_cancer_status', - # 'pregnancies', - # 'preservation_method', - # 'primary_neoplasm_melanoma_dx', - # 'primary_therapy_outcome_success', - 'Project', - # 'psa_value', - # 'race', - 'residual_tumor', - # 'sample_barcode', - 'SampleTypeCode', - # 'Study', - 'tobacco_smoking_history', - # 'total_number_of_pregnancies', - # 'tumor_pathology', - 'tumor_tissue_site', - 'tumor_type', - # 'weiss_venous_invasion', - 'vital_status' - # 'weight', - # 'year_of_initial_pathologic_diagnosis', -] - -metadata_dict = { - 'age_at_diagnosis': 'INTEGER', - 'anatomic_neoplasm_subdivision': 'VARCHAR(63)', - 'avg_percent_lymphocyte_infiltration': 'FLOAT', - 'avg_percent_monocyte_infiltration': 'FLOAT', - 'avg_percent_necrosis': 'FLOAT', - 'avg_percent_neutrophil_infiltration': 'FLOAT', - 'avg_percent_normal_cells': 'FLOAT', - 'avg_percent_stromal_cells': 'FLOAT', - 'avg_percent_tumor_cells': 'FLOAT', - 'avg_percent_tumor_nuclei': 'FLOAT', - 'batch_number': 'INTEGER', - 'bcr': 'VARCHAR(63)', - 'BMI': 'FLOAT', - 'clinical_M': 'VARCHAR(12)', - 'clinical_N': 'VARCHAR(12)', - 'clinical_T': 'VARCHAR(12)', - 'clinical_stage': 'VARCHAR(12)', - 'colorectal_cancer': 'VARCHAR(10)', - 'country': 'VARCHAR(63)', - 'days_to_birth': 'INTEGER', - 'days_to_collection': 'INTEGER', - 'days_to_death': 'INTEGER', - 'days_to_initial_pathologic_diagnosis': 'INTEGER', - 'days_to_last_followup': 'INTEGER', - 'days_to_submitted_specimen_dx': 'INTEGER', - 'Study': 'VARCHAR(4)', - 'ethnicity': 'VARCHAR(20)', - 'frozen_specimen_anatomic_site': 'VARCHAR(63)', - 'gender': 'VARCHAR(15)', - 'gleason_score_combined': 'INTEGER', - 'height': 'INTEGER', - 'histological_type': 'VARCHAR(63)', - 'history_of_colon_polyps': 'VARCHAR(8)', - 'history_of_neoadjuvant_treatment': 'VARCHAR(63)', - 'history_of_prior_malignancy': 'VARCHAR(25)', - 'hpv_calls': 'VARCHAR(20)', - 'hpv_status': 'VARCHAR(20)', - 'icd_10': 'VARCHAR(8)', - 'icd_o_3_histology': 'VARCHAR(10)', - 'icd_o_3_site': 'VARCHAR(8)', - 'lymphatic_invasion': 'VARCHAR(8)', - 'lymphnodes_examined': 'VARCHAR(8)', - 'lymphovascular_invasion_present': 'VARCHAR(63)', - 'max_percent_lymphocyte_infiltration': 'INTEGER', - 'max_percent_monocyte_infiltration': 'INTEGER', - 'max_percent_necrosis': 'INTEGER', - 'max_percent_neutrophil_infiltration': 'INTEGER', - 'max_percent_normal_cells': 'INTEGER', - 'max_percent_stromal_cells': 'INTEGER', - 'max_percent_tumor_cells': 'INTEGER', - 'max_percent_tumor_nuclei': 'INTEGER', - 'menopause_status': 'VARCHAR(30)', - 'min_percent_lymphocyte_infiltration': 'INTEGER', - 'min_percent_monocyte_infiltration': 'INTEGER', - 'min_percent_necrosis': 'INTEGER', - 'min_percent_neutrophil_infiltration': 'INTEGER', - 'min_percent_normal_cells': 'INTEGER', - 'min_percent_stromal_cells': 'INTEGER', - 'min_percent_tumor_cells': 'INTEGER', - 'min_percent_tumor_nuclei': 'INTEGER', - 'mononucleotide_and_dinucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'mononucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'neoplasm_histologic_grade': 'VARCHAR(15)', - 'new_tumor_event_after_initial_treatment': 'VARCHAR(8)', - 'number_of_lymphnodes_examined': 'INTEGER', - 'number_of_lymphnodes_positive_by_he': 'INTEGER', - 'number_pack_years_smoked': 'INTEGER', - 'case_barcode': 'VARCHAR(12)', - 'pathologic_M': 'VARCHAR(5)', - 'pathologic_N': 'VARCHAR(5)', - 'pathologic_T': 'VARCHAR(5)', - 'pathologic_stage': 'VARCHAR(10)', - 'person_neoplasm_cancer_status': 'VARCHAR(15)', - 'pregnancies': 'VARCHAR(35)', - 'primary_neoplasm_melanoma_dx': 'VARCHAR(10)', - 'primary_therapy_outcome_success': 'VARCHAR(35)', - 'prior_dx': 'VARCHAR(50)', - 'Project': 'VARCHAR(4)', - 'psa_value': 'FLOAT', - 'race': 'VARCHAR(30)', - 'residual_tumor': 'VARCHAR(5)', - 'sample_barcode': 'VARCHAR(16)', - 'Study': 'VARCHAR(4)', - 'tobacco_smoking_history': 'VARCHAR(30)', - 'tumor_tissue_site': 'VARCHAR(20)', - 'tumor_type': 'VARCHAR(4)', - 'weiss_venous_invasion': 'VARCHAR(63)', - 'vital_status': 'VARCHAR(63)', - 'weight': 'VARCHAR(63)', - 'year_of_initialPY_pathologic_diagnosis': 'VARCHAR(63)', - 'SampleTypeCode': 'VARCHAR(3)', - 'has_Illumina_DNASeq': 'TINYINT', - 'has_BCGSC_HiSeq_RNASeq': 'TINYINT', - 'has_UNC_HiSeq_RNASeq': 'TINYINT', - 'has_BCGSC_GA_RNASeq': 'TINYINT', - 'has_UNC_GA_RNASeq': 'TINYINT', - 'has_HiSeq_miRnaSeq': 'TINYINT', - 'has_GA_miRNASeq': 'TINYINT', - 'has_RPPA': 'TINYINT', - 'has_SNP6': 'TINYINT', - 'has_27k': 'TINYINT', - 'has_450k': 'TINYINT', -} - - -class MetaValueListCount(messages.Message): - value = messages.StringField(1) # note: this means converting booleans to strings - count = messages.IntegerField(2) - - -class MetaAttrValuesList(messages.Message): - adenocarcinoma_invasion = messages.MessageField(MetaValueListCount, 1, repeated=True) - age_at_diagnosis = messages.MessageField(MetaValueListCount, 2, repeated=True) - anatomic_neoplasm_subdivision = messages.MessageField(MetaValueListCount, 3, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_necrosis = messages.FloatField(6, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(7, repeated=True) - avg_percent_normal_cells = messages.FloatField(8, repeated=True) - avg_percent_stromal_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_cells = messages.FloatField(10, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(11, repeated=True) - batch_number = messages.MessageField(MetaValueListCount, 12, repeated=True) - bcr = messages.MessageField(MetaValueListCount, 13, repeated=True) - clinical_M = messages.MessageField(MetaValueListCount, 14, repeated=True) - clinical_N = messages.MessageField(MetaValueListCount, 15, repeated=True) - clinical_stage = messages.MessageField(MetaValueListCount, 16, repeated=True) - clinical_T = messages.MessageField(MetaValueListCount, 17, repeated=True) - colorectal_cancer = messages.MessageField(MetaValueListCount, 18, repeated=True) - country = messages.MessageField(MetaValueListCount, 19, repeated=True) - country_of_procurement = messages.MessageField(MetaValueListCount, 20, repeated=True) - days_to_birth = messages.MessageField(MetaValueListCount, 21, repeated=True) - days_to_collection = messages.MessageField(MetaValueListCount, 22, repeated=True) - days_to_death = messages.MessageField(MetaValueListCount, 23, repeated=True) - days_to_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 24, repeated=True) - days_to_last_followup = messages.MessageField(MetaValueListCount, 25, repeated=True) - days_to_submitted_specimen_dx = messages.MessageField(MetaValueListCount, 26, repeated=True) - Study = messages.MessageField(MetaValueListCount, 27, repeated=True) - ethnicity = messages.MessageField(MetaValueListCount, 28, repeated=True) - frozen_specimen_anatomic_site = messages.MessageField(MetaValueListCount, 29, repeated=True) - gender = messages.MessageField(MetaValueListCount, 30, repeated=True) - height = messages.MessageField(MetaValueListCount, 31, repeated=True) - histological_type = messages.MessageField(MetaValueListCount, 32, repeated=True) - history_of_colon_polyps = messages.MessageField(MetaValueListCount, 33, repeated=True) - history_of_neoadjuvant_treatment = messages.MessageField(MetaValueListCount, 34, repeated=True) - history_of_prior_malignancy = messages.MessageField(MetaValueListCount, 35, repeated=True) - hpv_calls = messages.MessageField(MetaValueListCount, 36, repeated=True) - hpv_status = messages.MessageField(MetaValueListCount, 37, repeated=True) - icd_10 = messages.MessageField(MetaValueListCount, 38, repeated=True) - icd_o_3_histology = messages.MessageField(MetaValueListCount, 39, repeated=True) - icd_o_3_site = messages.MessageField(MetaValueListCount, 40, repeated=True) - lymph_node_examined_count = messages.MessageField(MetaValueListCount, 41, repeated=True) - lymphatic_invasion = messages.MessageField(MetaValueListCount, 42, repeated=True) - lymphnodes_examined = messages.MessageField(MetaValueListCount, 43, repeated=True) - lymphovascular_invasion_present = messages.MessageField(MetaValueListCount, 44, repeated=True) - max_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 45, repeated=True) - max_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 46, repeated=True) - max_percent_necrosis = messages.MessageField(MetaValueListCount, 47, repeated=True) - max_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 48, repeated=True) - max_percent_normal_cells = messages.MessageField(MetaValueListCount, 49, repeated=True) - max_percent_stromal_cells = messages.MessageField(MetaValueListCount, 50, repeated=True) - max_percent_tumor_cells = messages.MessageField(MetaValueListCount, 51, repeated=True) - max_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 52, repeated=True) - menopause_status = messages.MessageField(MetaValueListCount, 53, repeated=True) - min_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 54, repeated=True) - min_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 55, repeated=True) - min_percent_necrosis = messages.MessageField(MetaValueListCount, 56, repeated=True) - min_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 57, repeated=True) - min_percent_normal_cells = messages.MessageField(MetaValueListCount, 58, repeated=True) - min_percent_stromal_cells = messages.MessageField(MetaValueListCount, 59, repeated=True) - min_percent_tumor_cells = messages.MessageField(MetaValueListCount, 60, repeated=True) - min_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 61, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 62, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 63, repeated=True) - neoplasm_histologic_grade = messages.MessageField(MetaValueListCount, 64, repeated=True) - new_tumor_event_after_initial_treatment = messages.MessageField(MetaValueListCount, 65, repeated=True) - number_of_lymphnodes_examined = messages.MessageField(MetaValueListCount, 66, repeated=True) - number_of_lymphnodes_positive_by_he = messages.MessageField(MetaValueListCount, 67, repeated=True) - case_barcode = messages.MessageField(MetaValueListCount, 68, repeated=True) - pathologic_M = messages.MessageField(MetaValueListCount, 69, repeated=True) - pathologic_N = messages.MessageField(MetaValueListCount, 70, repeated=True) - pathologic_stage = messages.MessageField(MetaValueListCount, 71, repeated=True) - pathologic_T = messages.MessageField(MetaValueListCount, 72, repeated=True) - person_neoplasm_cancer_status = messages.MessageField(MetaValueListCount, 73, repeated=True) - pregnancies = messages.MessageField(MetaValueListCount, 74, repeated=True) - preservation_method = messages.MessageField(MetaValueListCount, 75, repeated=True) - primary_neoplasm_melanoma_dx = messages.MessageField(MetaValueListCount, 76, repeated=True) - primary_therapy_outcome_success = messages.MessageField(MetaValueListCount, 77, repeated=True) - prior_dx = messages.MessageField(MetaValueListCount, 78, repeated=True) - Project = messages.MessageField(MetaValueListCount, 79, repeated=True) - psa_value = messages.FloatField(80, repeated=True) - race = messages.MessageField(MetaValueListCount, 81, repeated=True) - residual_tumor = messages.MessageField(MetaValueListCount, 82, repeated=True) - sample_barcode = messages.MessageField(MetaValueListCount, 83, repeated=True) - tobacco_smoking_history = messages.MessageField(MetaValueListCount, 86, repeated=True) - total_number_of_pregnancies = messages.MessageField(MetaValueListCount, 87, repeated=True) - tumor_tissue_site = messages.MessageField(MetaValueListCount, 88, repeated=True) - tumor_pathology = messages.MessageField(MetaValueListCount, 89, repeated=True) - tumor_type = messages.MessageField(MetaValueListCount, 90, repeated=True) - weiss_venous_invasion = messages.MessageField(MetaValueListCount, 91, repeated=True) - vital_status = messages.MessageField(MetaValueListCount, 92, repeated=True) - weight = messages.MessageField(MetaValueListCount, 93, repeated=True) - year_of_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 94, repeated=True) - SampleTypeCode = messages.MessageField(MetaValueListCount, 95, repeated=True) - has_Illumina_DNASeq = messages.MessageField(MetaValueListCount, 96, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 97, repeated=True) - has_UNC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 98, repeated=True) - has_BCGSC_GA_RNASeq = messages.MessageField(MetaValueListCount, 99, repeated=True) - has_UNC_GA_RNASeq = messages.MessageField(MetaValueListCount, 100, repeated=True) - has_HiSeq_miRnaSeq = messages.MessageField(MetaValueListCount, 101, repeated=True) - has_GA_miRNASeq = messages.MessageField(MetaValueListCount, 102, repeated=True) - has_RPPA = messages.MessageField(MetaValueListCount, 103, repeated=True) - has_SNP6 = messages.MessageField(MetaValueListCount, 104, repeated=True) - has_27k = messages.MessageField(MetaValueListCount, 105, repeated=True) - has_450k = messages.MessageField(MetaValueListCount, 106, repeated=True) - BMI = messages.FloatField(107, repeated=True) - - -class MetadataItem(messages.Message): - adenocarcinoma_invasion = messages.StringField(1) - age_at_diagnosis = messages.IntegerField(2) - anatomic_neoplasm_subdivision = messages.StringField(3) - avg_percent_lymphocyte_infiltration = messages.FloatField(4) - avg_percent_monocyte_infiltration = messages.FloatField(5) - avg_percent_necrosis = messages.FloatField(6) - avg_percent_neutrophil_infiltration = messages.FloatField(7) - avg_percent_normal_cells = messages.FloatField(8) - avg_percent_stromal_cells = messages.FloatField(9) - avg_percent_tumor_cells = messages.FloatField(10) - avg_percent_tumor_nuclei = messages.FloatField(11) - batch_number = messages.IntegerField(12) - bcr = messages.StringField(13) - clinical_M = messages.StringField(14) - clinical_N = messages.StringField(15) - clinical_stage = messages.StringField(16) - clinical_T = messages.StringField(17) - colorectal_cancer = messages.StringField(18) - country = messages.StringField(19) - country_of_procurement = messages.StringField(20) - days_to_birth = messages.IntegerField(21) - days_to_collection = messages.IntegerField(22) - days_to_death = messages.IntegerField(23) - days_to_initial_pathologic_diagnosis = messages.IntegerField(24) - days_to_last_followup = messages.IntegerField(25) - days_to_submitted_specimen_dx = messages.IntegerField(26) - Study = messages.StringField(27) - ethnicity = messages.StringField(28) - frozen_specimen_anatomic_site = messages.StringField(29) - gender = messages.StringField(30) - height = messages.IntegerField(31) - histological_type = messages.StringField(32) - history_of_colon_polyps = messages.StringField(33) - history_of_neoadjuvant_treatment = messages.StringField(34) - history_of_prior_malignancy = messages.StringField(35) - hpv_calls = messages.StringField(36) - hpv_status = messages.StringField(37) - icd_10 = messages.StringField(38) - icd_o_3_histology = messages.StringField(39) - icd_o_3_site = messages.StringField(40) - lymph_node_examined_count = messages.IntegerField(41) - lymphatic_invasion = messages.StringField(42) - lymphnodes_examined = messages.StringField(43) - lymphovascular_invasion_present = messages.StringField(44) - max_percent_lymphocyte_infiltration = messages.IntegerField(45) - max_percent_monocyte_infiltration = messages.IntegerField(46) - max_percent_necrosis = messages.IntegerField(47) - max_percent_neutrophil_infiltration = messages.IntegerField(48) - max_percent_normal_cells = messages.IntegerField(49) - max_percent_stromal_cells = messages.IntegerField(50) - max_percent_tumor_cells = messages.IntegerField(51) - max_percent_tumor_nuclei = messages.IntegerField(52) - menopause_status = messages.StringField(53) - min_percent_lymphocyte_infiltration = messages.IntegerField(54) - min_percent_monocyte_infiltration = messages.IntegerField(55) - min_percent_necrosis = messages.IntegerField(56) - min_percent_neutrophil_infiltration = messages.IntegerField(57) - min_percent_normal_cells = messages.IntegerField(58) - min_percent_stromal_cells = messages.IntegerField(59) - min_percent_tumor_cells = messages.IntegerField(60) - min_percent_tumor_nuclei = messages.IntegerField(61) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(62) - mononucleotide_marker_panel_analysis_status = messages.StringField(63) - neoplasm_histologic_grade = messages.StringField(64) - new_tumor_event_after_initial_treatment = messages.StringField(65) - number_of_lymphnodes_examined = messages.IntegerField(66) - number_of_lymphnodes_positive_by_he = messages.IntegerField(67) - case_barcode = messages.StringField(68) - pathologic_M = messages.StringField(69) - pathologic_N = messages.StringField(70) - pathologic_stage = messages.StringField(71) - pathologic_T = messages.StringField(72) - person_neoplasm_cancer_status = messages.StringField(73) - pregnancies = messages.StringField(74) - preservation_method = messages.StringField(75) - primary_neoplasm_melanoma_dx = messages.StringField(76) - primary_therapy_outcome_success = messages.StringField(77) - prior_dx = messages.StringField(78) - Project = messages.StringField(79) - psa_value = messages.FloatField(80) - race = messages.StringField(81) - residual_tumor = messages.StringField(82) - sample_barcode = messages.StringField(83) - tobacco_smoking_history = messages.StringField(86) - total_number_of_pregnancies = messages.IntegerField(87) - tumor_tissue_site = messages.StringField(88) - tumor_pathology = messages.StringField(89) - tumor_type = messages.StringField(90) - weiss_venous_invasion = messages.StringField(91) - vital_status = messages.StringField(92) - weight = messages.IntegerField(93) - year_of_initial_pathologic_diagnosis = messages.StringField(94) - SampleTypeCode = messages.StringField(95) - has_Illumina_DNASeq = messages.StringField(96) - has_BCGSC_HiSeq_RNASeq = messages.StringField(97) - has_UNC_HiSeq_RNASeq = messages.StringField(98) - has_BCGSC_GA_RNASeq = messages.StringField(99) - has_UNC_GA_RNASeq = messages.StringField(100) - has_HiSeq_miRnaSeq = messages.StringField(101) - has_GA_miRNASeq = messages.StringField(102) - has_RPPA = messages.StringField(103) - has_SNP6 = messages.StringField(104) - has_27k = messages.StringField(105) - has_450k = messages.StringField(106) - BMI = messages.FloatField(107) - -''' -Incoming object needs to use age and BMI that's a string (eg. 10_to_39) -''' -class IncomingMetadataItem(messages.Message): - age_at_diagnosis = messages.StringField(1, repeated=True) - anatomic_neoplasm_subdivision = messages.StringField(2, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(3, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_necrosis = messages.FloatField(5, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(6, repeated=True) - avg_percent_normal_cells = messages.FloatField(7, repeated=True) - avg_percent_stromal_cells = messages.FloatField(8, repeated=True) - avg_percent_tumor_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(10, repeated=True) - batch_number = messages.IntegerField(11, repeated=True) - bcr = messages.StringField(12, repeated=True) - clinical_M = messages.StringField(13, repeated=True) - clinical_N = messages.StringField(14, repeated=True) - clinical_stage = messages.StringField(15, repeated=True) - clinical_T = messages.StringField(16, repeated=True) - colorectal_cancer = messages.StringField(17, repeated=True) - country = messages.StringField(18, repeated=True) - days_to_birth = messages.IntegerField(19, repeated=True) - days_to_collection = messages.IntegerField(20, repeated=True) - days_to_death = messages.IntegerField(21, repeated=True) - days_to_initial_pathologic_diagnosis = messages.IntegerField(22, repeated=True) - days_to_last_followup = messages.IntegerField(23, repeated=True) - days_to_submitted_specimen_dx = messages.IntegerField(24, repeated=True) - Study = messages.StringField(25, repeated=True) - ethnicity = messages.StringField(26, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(27, repeated=True) - gender = messages.StringField(28, repeated=True) - height = messages.IntegerField(29, repeated=True) - histological_type = messages.StringField(30, repeated=True) - history_of_colon_polyps = messages.StringField(31, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(32, repeated=True) - history_of_prior_malignancy = messages.StringField(33, repeated=True) - hpv_calls = messages.StringField(34, repeated=True) - hpv_status = messages.StringField(35, repeated=True) - icd_10 = messages.StringField(36, repeated=True) - icd_o_3_histology = messages.StringField(37, repeated=True) - icd_o_3_site = messages.StringField(38, repeated=True) - lymphatic_invasion = messages.StringField(39, repeated=True) - lymphnodes_examined = messages.StringField(40, repeated=True) - lymphovascular_invasion_present = messages.StringField(41, repeated=True) - max_percent_lymphocyte_infiltration = messages.IntegerField(42, repeated=True) - max_percent_monocyte_infiltration = messages.IntegerField(43, repeated=True) - max_percent_necrosis = messages.IntegerField(44, repeated=True) - max_percent_neutrophil_infiltration = messages.IntegerField(45, repeated=True) - max_percent_normal_cells = messages.IntegerField(46, repeated=True) - max_percent_stromal_cells = messages.IntegerField(47, repeated=True) - max_percent_tumor_cells = messages.IntegerField(48, repeated=True) - max_percent_tumor_nuclei = messages.IntegerField(49, repeated=True) - menopause_status = messages.StringField(50, repeated=True) - min_percent_lymphocyte_infiltration = messages.IntegerField(51, repeated=True) - min_percent_monocyte_infiltration = messages.IntegerField(52, repeated=True) - min_percent_necrosis = messages.IntegerField(53, repeated=True) - min_percent_neutrophil_infiltration = messages.IntegerField(54, repeated=True) - min_percent_normal_cells = messages.IntegerField(55, repeated=True) - min_percent_stromal_cells = messages.IntegerField(56, repeated=True) - min_percent_tumor_cells = messages.IntegerField(57, repeated=True) - min_percent_tumor_nuclei = messages.IntegerField(58, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(59, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(60, repeated=True) - neoplasm_histologic_grade = messages.StringField(61, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(62, repeated=True) - number_of_lymphnodes_examined = messages.IntegerField(63, repeated=True) - number_of_lymphnodes_positive_by_he = messages.IntegerField(64, repeated=True) - case_barcode = messages.StringField(65, repeated=True) - pathologic_M = messages.StringField(66, repeated=True) - pathologic_N = messages.StringField(67, repeated=True) - pathologic_stage = messages.StringField(68, repeated=True) - pathologic_T = messages.StringField(69, repeated=True) - person_neoplasm_cancer_status = messages.StringField(70, repeated=True) - pregnancies = messages.StringField(71, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(72, repeated=True) - primary_therapy_outcome_success = messages.StringField(73, repeated=True) - prior_dx = messages.StringField(74, repeated=True) - Project = messages.StringField(75, repeated=True) - psa_value = messages.FloatField(76, repeated=True) - race = messages.StringField(77, repeated=True) - residual_tumor = messages.StringField(78, repeated=True) - sample_barcode = messages.StringField(79, repeated=True) - tobacco_smoking_history = messages.StringField(80, repeated=True) - tumor_tissue_site = messages.StringField(81, repeated=True) - tumor_type = messages.StringField(82, repeated=True) - weiss_venous_invasion = messages.StringField(83, repeated=True) - vital_status = messages.StringField(84, repeated=True) - weight = messages.IntegerField(85, repeated=True) - year_of_initial_pathologic_diagnosis = messages.StringField(86, repeated=True) - SampleTypeCode = messages.StringField(87, repeated=True) - has_Illumina_DNASeq = messages.StringField(88, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(89, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(90, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(91, repeated=True) - has_UNC_GA_RNASeq = messages.StringField(92, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(93, repeated=True) - has_GA_miRNASeq = messages.StringField(94, repeated=True) - has_RPPA = messages.StringField(95, repeated=True) - has_SNP6 = messages.StringField(96, repeated=True) - has_27k = messages.StringField(97, repeated=True) - has_450k = messages.StringField(98, repeated=True) - BMI = messages.StringField(99, repeated=True) - -class MetadataAttributeValues(messages.Message): - name = messages.StringField(1) - id = messages.StringField(2) - values = messages.MessageField(MetaValueListCount, 3, repeated=True) - total = messages.IntegerField(4) - -class MetadataItemList(messages.Message): - items = messages.MessageField(MetadataItem, 1, repeated=True) - count = messages.MessageField(MetaAttrValuesList, 2) - total = messages.IntegerField(3) - -class MetadataCountsItem(messages.Message): - count = messages.MessageField(MetadataAttributeValues, 1, repeated=True) - total = messages.IntegerField(2) - -class MetaDomainsList(messages.Message): - gender = messages.StringField(1, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(2, repeated=True) - country = messages.StringField(3, repeated=True) - Study = messages.StringField(4, repeated=True) - ethnicity = messages.StringField(5, repeated=True) - histological_type = messages.StringField(6, repeated=True) - icd_10 = messages.StringField(7, repeated=True) - icd_o_3_histology = messages.StringField(8, repeated=True) - icd_o_3_site = messages.StringField(9, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(10, repeated=True) - neoplasm_histologic_grade = messages.StringField(11, repeated=True) - pathologic_N = messages.StringField(12, repeated=True) - pathologic_T = messages.StringField(13, repeated=True) - pathologic_stage = messages.StringField(14, repeated=True) - person_neoplasm_cancer_status = messages.StringField(15, repeated=True) - prior_dx = messages.StringField(16, repeated=True) - Project = messages.StringField(17, repeated=True) - race = messages.StringField(18, repeated=True) - residual_tumor = messages.StringField(19, repeated=True) - SampleTypeCode = messages.StringField(20, repeated=True) - tumor_tissue_site = messages.StringField(21, repeated=True) - tumor_type = messages.StringField(22, repeated=True) - vital_status = messages.StringField(23, repeated=True) - has_Illumina_DNASeq = messages.StringField(24, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(25, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(26, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(27, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(28, repeated=True) - has_GA_miRNASeq = messages.StringField(29, repeated=True) - has_RPPA = messages.StringField(30, repeated=True) - has_SNP6 = messages.StringField(31, repeated=True) - has_27k = messages.StringField(32, repeated=True) - has_450k = messages.StringField(33, repeated=True) - - -class SampleBarcodeItem(messages.Message): - sample_barcode = messages.StringField(1) - study_id = messages.IntegerField(2) - -class MetadataAttr(messages.Message): - attribute = messages.StringField(1) - code = messages.StringField(2) - spec = messages.StringField(3) - key = messages.StringField(4) - - -class MetadataAttrList(messages.Message): - items = messages.MessageField(MetadataAttr, 1, repeated=True) - count = messages.IntegerField(2) - - -class SampleBarcodeList(messages.Message): - items = messages.MessageField(SampleBarcodeItem, 1, repeated=True) - count = messages.IntegerField(2) - - -class MetadataPlatformItem(messages.Message): - DNAseq_data = messages.StringField(1) - cnvrPlatform = messages.StringField(2) - gexpPlatform = messages.StringField(3) - methPlatform = messages.StringField(4) - mirnPlatform = messages.StringField(5) - rppaPlatform = messages.StringField(6) - - -class MetadataPlatformItemList(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - - -class MetadataCountsPlatformItem(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - count = messages.MessageField(MetadataAttributeValues, 2, repeated=True) - participants = messages.IntegerField(3) - total = messages.IntegerField(4) - - -def createDataItem(data, selectors): - if len(selectors): - item = MetadataItem() - for attr in selectors: - attr = attr.encode('utf-8') - if data[attr] is not None: - if type(data[attr]) is not long and type(data[attr]) is not int: - item.__setattr__(attr, data[attr].encode('utf-8')) - if attr.startswith('has_'): - item.__setattr__(attr, str(bool(data[attr]))) - else: - item.__setattr__(attr, data[attr]) - else: - item.__setattr__(attr, None) - return item - - -class MetadataDomainItem(messages.Message): - attribute = messages.StringField(1) - domains = messages.StringField(2, repeated=True) - - -def submit_bigquery_job(bq_service, project_id, query_body, batch=False): - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid4()) - }, - 'configuration': { - 'query': { - 'query': query_body, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - - return bq_service.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=5) - - -def is_bigquery_job_finished(bq_service, project_id, job_id): - job = bq_service.jobs().get(projectId=project_id, - jobId=job_id).execute() - - return job['status']['state'] == 'DONE' - - -def get_bq_job_results(bq_service, job_reference): - result = [] - page_token = None - - while True: - page = bq_service.jobs().getQueryResults( - pageToken=page_token, - **job_reference).execute(num_retries=2) - - if int(page['totalRows']) == 0: - break - - rows = page['rows'] - result.extend(rows) - - page_token = page.get('pageToken') - if not page_token: - break - - return result - - -def generateSQLQuery(request): - db = sql_connection() - query_dict = {} - select = '*' - sample_ids = () - selector_list = [] - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s AND project_id IS NULL;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - cursor.close() - - except (TypeError, IndexError) as e: - print e - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - if request.__getattribute__('selectors') is not None and len(request.__getattribute__('selectors')): - select = ','.join(request.selectors) - selector_list = select.split(',') # request.selectors - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None: - if key.startswith('has_'): - query_dict[key] = 1 if request.__getattribute__(key) == 'True' else 0 - else: - query_dict[key] = request.__getattribute__(key).replace('_', ' ') # values coming in with _ replaced with spaces - - query_str = 'SELECT %s FROM metadata_samples' % select - value_tuple = () - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - if request.__getattribute__('limit') is not None: - query_str += ' LIMIT %s' % request.__getattribute__('limit') - - query_str += ';' - db.close() - - return query_str, value_tuple, selector_list - - -class MetadataDomainList(messages.Message): - items = messages.MessageField(MetadataDomainItem, 1, repeated=True) - - -def normalize_metadata_ages(ages): - result = [] - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age in ages: - if type(age) != dict: - - if age.value != 'None': - int_age = float(age.value) - if int_age < 40: - new_age_list['10 to 39'] += int(age.count) - elif int_age < 50: - new_age_list['40 to 49'] += int(age.count) - elif int_age < 60: - new_age_list['50 to 59'] += int(age.count) - elif int_age < 70: - new_age_list['60 to 69'] += int(age.count) - elif int_age < 80: - new_age_list['70 to 79'] += int(age.count) - else: - new_age_list['Over 80'] += int(age.count) - else: - new_age_list['None'] += int(age.count) - - for key, value in new_age_list.items(): - result.append({'count': value, 'value': key}) - return result - - -class PlatformCount(messages.Message): - platform = messages.StringField(1) - count = messages.IntegerField(2) - - -class FileDetails(messages.Message): - sample = messages.StringField(1) - filename = messages.StringField(2) - pipeline = messages.StringField(3) - platform = messages.StringField(4) - datalevel = messages.StringField(5) - datatype = messages.StringField(6) - gg_readgroupset_id = messages.StringField(7) - cloudstorage_location = messages.StringField(8) - access = messages.StringField(9) - - -class SampleFiles(messages.Message): - total_file_count = messages.IntegerField(1) - page = messages.IntegerField(2) - platform_count_list = messages.MessageField(PlatformCount, 3, repeated=True) - file_list = messages.MessageField(FileDetails, 4, repeated=True) - - -class SampleFileCount(messages.Message): - sample_id = messages.StringField(1) - count = messages.IntegerField(2) - - -class CohortFileCountSampleList(messages.Message): - sample_list = messages.MessageField(SampleFileCount, 1, repeated=True) - - -class IncomingPlatformSelection(messages.Message): - ABSOLiD_DNASeq = messages.StringField(1) - Genome_Wide_SNP_6 = messages.StringField(2) - HumanMethylation27 = messages.StringField(3) - HumanMethylation450 = messages.StringField(4) - IlluminaGA_DNASeq = messages.StringField(5) - IlluminaGA_DNASeq_automated = messages.StringField(6) - IlluminaGA_DNASeq_Cont_automated = messages.StringField(7) - IlluminaGA_DNASeq_curated = messages.StringField(8) - IlluminaGA_miRNASeq = messages.StringField(9) - IlluminaGA_None = messages.StringField(10) - IlluminaGA_RNASeq = messages.StringField(11) - IlluminaGA_RNASeqV2 = messages.StringField(12) - IlluminaHiSeq_DNASeq = messages.StringField(13) - IlluminaHiSeq_DNASeq_automated = messages.StringField(14) - IlluminaHiSeq_DNASeq_Cont_automated = messages.StringField(15) - IlluminaHiSeq_miRNASeq = messages.StringField(16) - IlluminaHiSeq_None = messages.StringField(17) - IlluminaHiSeq_RNASeq = messages.StringField(18) - IlluminaHiSeq_RNASeqV2 = messages.StringField(19) - IlluminaHiSeq_TotalRNASeqV2 = messages.StringField(20) - IlluminaMiSeq_DNASeq = messages.StringField(21) - IlluminaMiSeq_None = messages.StringField(22) - LifeIonTorrentPGM_None = messages.StringField(23) - LifeIonTorrentProton_None = messages.StringField(24) - MDA_RPPA_Core = messages.StringField(25) - microsat_i = messages.StringField(26) - Mixed_DNASeq_Cont = messages.StringField(27) - Mixed_DNASeq_Cont_curated = messages.StringField(28) - Mixed_DNASeq_curated = messages.StringField(29) - RocheGSFLX_DNASeq = messages.StringField(30) - - -class IncomingMetadataCount(messages.Message): - filters = messages.StringField(1) - cohort_id = messages.IntegerField(2) - token = messages.StringField(3) - - -def get_current_user(request): - """ - Returns a Django_User object from either endpoints.get_current_user() or an access token. - Anytime this is used in an endpoint, request_finished.send(self) must be used - """ - user_email = None - access_token = request.__getattribute__('token') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - django.setup() - try: - if access_token is not None and user_email is None: - social_account_id = SocialToken.objects.get(token=access_token).account_id - user_id = SocialAccount.objects.get(id=social_account_id).user_id - return Django_User.objects.get(id=user_id) - elif user_email is not None: - return Django_User.objects.get(email=user_email) - else: - return None - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - return None - - -# TODO: needs to be refactored to use other samples tables -def get_participant_list(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - return results - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_list') - -# TODO: needs to be refactored to use other samples tables -def get_participant_count(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT COUNT(DISTINCT case_barcode) AS ParticipantCount FROM metadata_samples WHERE sample_barcode IN (' - first = True - samples = () - for barcode in sample_ids: - samples += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - count = 0; - cursor.execute(participant_query, samples) - for row in cursor.fetchall(): - count = row['ParticipantCount'] - - return count - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_count') - - -def count_metadata(user, cohort_id=None, sample_ids=None, filters=None): - counts_and_total = {} - sample_tables = {} - valid_attrs = {} - project_ids = () - table_key_map = {} - - if filters is None: - filters = {} - - if sample_ids is None: - sample_ids = {} - - for key in sample_ids: - samples_by_project = sample_ids[key] - sample_ids[key] = { - 'sample_barcode': build_where_clause({'sample_barcode': samples_by_project}), - } - - db = sql_connection() - django.setup() - - try: - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'sample_ids': None} - if sample_ids and None in sample_ids: - sample_tables['metadata_samples']['sample_ids'] = sample_ids[None] - - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - if row['attribute'] in METADATA_SHORTLIST: - valid_attrs[row['spec'] + ':' + row['attribute']] = { - 'name': row['attribute'], - 'tables': ('metadata_samples',), - 'sample_ids': None - } - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = {'sample_ids': None} - if sample_ids and project.id in sample_ids: - sample_tables[tables.metadata_samples_table]['sample_ids'] = sample_ids[project.id] - - features = User_Feature_Definitions.objects.filter(project__in=project_ids) - for feature in features: - if ' ' in feature.feature_name: - # It is not a column name and comes from molecular data, ignore it - continue - - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name, 'tables': (), 'sample_ids': None} - - for tables in User_Data_Tables.objects.filter(project_id=feature.project_id): - valid_attrs[key]['tables'] += (tables.metadata_samples_table,) - - if tables.metadata_samples_table not in table_key_map: - table_key_map[tables.metadata_samples_table] = {} - table_key_map[tables.metadata_samples_table][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - - if sample_ids and feature.project_id in sample_ids: - valid_attrs[key]['sample_ids'] = sample_ids[feature.project_id] - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - # For filters with no tables at this point, assume its the TCGA metadata_samples table - for key, obj in filters.items(): - if not obj['tables']: - filters[key]['tables'].append('metadata_samples') - - resulting_samples = {} - - # Loop through the features - for key, feature in valid_attrs.items(): - # Get a count for each feature - table_values = {} - feature['total'] = 0 - for table in feature['tables']: - # Check if the filters make this table 0 anyway - # We do this to avoid SQL errors for columns that don't exist - should_be_queried = True - if cohort_id and sample_tables[table]['sample_ids'] is None: - should_be_queried = False - - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - # Build Filter Where Clause - filter_copy = copy.deepcopy(filters) - key_map = table_key_map[table] if table in table_key_map else False - where_clause = build_where_clause(filter_copy, alt_key_map=key_map) - col_name = feature['name'] - if key_map and key in key_map: - col_name = key_map[key] - - cursor = db.cursor() - if should_be_queried: - # Query the table for counts and values - query = ('SELECT DISTINCT %s, COUNT(1) as count FROM %s') % (col_name, table) - sample_query = ('SELECT DISTINCT %s AS sample_id FROM %s') % ('sample_barcode', table) - query_clause = '' - if where_clause['query_str']: - query_clause = ' WHERE ' + where_clause['query_str'] - if sample_tables[table]['sample_ids']: - barcode_key = 'sample_barcode' - addt_cond = sample_tables[table]['sample_ids'][barcode_key]['query_str'] - if addt_cond and where_clause['query_str']: - query_clause += ' AND ' + addt_cond - elif addt_cond: - query_clause = ' WHERE ' + addt_cond - where_clause['value_tuple'] += sample_tables[table]['sample_ids'][barcode_key]['value_tuple'] - query += query_clause + (' GROUP BY %s ' % col_name) - sample_query += query_clause - - logger.debug("In api count_metadata, executing query "+query) - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - table_values[row[0]] += int(row[1]) - feature['total'] += int(row[1]) - - cursor.execute(sample_query, where_clause['value_tuple']) - for row in cursor.fetchall(): - resulting_samples[row[0]] = 1 - else: - # Just get the values so we can have them be 0 - cursor.execute(('SELECT DISTINCT %s FROM %s') % (col_name, table)) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - - cursor.close() - - feature['values'] = table_values - - sample_set = () - for sample in resulting_samples: - sample_set += (sample,) - - counts_and_total['participants'] = get_participant_count(sample_set) if sample_set.__len__() > 0 else 0 - counts_and_total['counts'] = [] - counts_and_total['total'] = 0 - for key, feature in valid_attrs.items(): - value_list = [] - - # Special case for age ranges and BMI - if key == 'CLIN:age_at_diagnosis': - feature['values'] = normalize_ages(feature['values']) - elif key == 'CLIN:BMI': - feature['values'] = normalize_BMI(feature['values']) - - - for value, count in feature['values'].items(): - if feature['name'].startswith('has_'): - value = 'True' if value else 'False' - - value_list.append(MetaValueListCount(value=str(value), count=count)) - - counts_and_total['counts'].append( - MetadataAttributeValues(name=feature['name'], values=value_list, id=key, total=feature['total'])) - if feature['total'] > counts_and_total['total']: - counts_and_total['total'] = feature['total'] - - db.close() - - return counts_and_total - - except (Exception) as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in count_metadata.') - - -def query_samples_and_studies(parameter, bucket_by=None): - - query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - if bucket_by is not None and bucket_by not in query_str: - logging.error("Cannot group barcodes: column '" + bucket_by + - "' not found in query string '"+query_str+"'. Barcodes will not be grouped.") - bucket_by = None - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, (parameter,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs for cohort '" + parameter + "': " + (stop-start).__str__()) - - samples = () - - if bucket_by is not None: - samples = {} - - for row in cursor.fetchall(): - if bucket_by is not None: - if row[bucket_by] not in samples: - samples[row[bucket_by]] = [] - samples[row[bucket_by]].append(row['sample_barcode']) - else: - samples += ({"sample_id": row['sample_barcode'], "project_id": row['project_id']},) - cursor.close() - db.close() - - return samples - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - -""" -Metadata Endpoints v3 - -Includes User Uploaded Data -""" -Meta_Endpoints_v3 = endpoints.api(name='meta_api', version='v3', - description='Retrieve metadata information relating to projects, cohorts, and other data', - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - -@Meta_Endpoints_v3.api_class(resource_name='meta_endpoints') -class Meta_Endpoints_API_v3(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer( - MetadataAttr, - token=messages.StringField(4), - ) - @endpoints.method(GET_RESOURCE, MetadataAttrList, - path='attributes', http_method='GET', - name='meta.attr_list') - def metadata_attr_list(self, request): - - cursor = None - db = None - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - query_dict = {} - value_tuple = () - for key, value in MetadataAttr.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) != None: - query_dict[key] = request.__getattribute__(key) - - if len(query_dict) == 0: - query_str = 'SELECT * FROM metadata_attr' - else: - query_str = 'SELECT * FROM metadata_attr where' - where_clause = build_where_clause(query_dict) - query_str += where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - data.append(MetadataAttr(attribute=str(row['attribute']), - code=str(row['code']), - spec=str(row['spec']), - key=str(row['spec']) + ':' + str(row['attribute']) - )) - - if user: - studies = Project.get_user_studies(user) - feature_defs = User_Feature_Definitions.objects.filter(project__in=studies) - for feature in feature_defs: - data_table = User_Data_Tables.objects.get(project=feature.project).metadata_samples_table - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - - data.append(MetadataAttr(attribute=name, - code='N' if feature.is_numeric else 'C', - spec='USER', - key=key - )) - - cursor.close() - db.close() - request_finished.send(self) - return MetadataAttrList(items=data, count=len(data)) - - except (IndexError, TypeError): - raise endpoints.InternalServerErrorException('Error retrieving attribute list') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsItem, - path='metadata_counts', http_method='POST', - name='meta.metadata_counts') - def metadata_counts(self, request): - filters = {} - sample_ids = None - cohort_id = None - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for reqFilter in tmp: - key = reqFilter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(reqFilter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - start = time.time() - counts_and_totals = count_metadata(user, cohort_id, sample_ids, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts" - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - request_finished.send(self) - return MetadataCountsItem(count=counts_and_totals['counts'], total=counts_and_totals['total']) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - @endpoints.method(POST_RESOURCE, SampleBarcodeList, - path='metadata_sample_list', http_method='POST', - name='meta.metadata_sample_list') - def metadata_list(self, request): - filters = {} - valid_attrs = {} - sample_tables = {} - table_key_map = {} - sample_ids = None - project_ids = () - cohort_id = None - cursor = None - db = None - mutation_filters = None - mutation_results = {} - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if 'MUT:' in key: - if not mutation_filters: - mutation_filters = {} - if not key in mutation_filters: - mutation_filters[key] = [] - mutation_filters[key].append(filter['value']) - else: - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - django.setup() - - # TODO enable filtering based off of this - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - if mutation_filters: - mutation_where_clause = build_where_clause(mutation_filters) - print >> sys.stdout, mutation_where_clause - cohort_join_str = '' - cohort_where_str = '' - bq_cohort_table = '' - bq_cohort_dataset = '' - cohort = '' - query_template = None - - if cohort_id is not None: - query_template = \ - ("SELECT ct.sample_barcode" - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] ct" - " JOIN (SELECT Tumor_SampleBarcode AS barcode " - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY barcode) mt" - " ON mt.barcode = ct.sample_barcode" - " WHERE ct.cohort_id = {cohort};") - bq_cohort_table = settings.BIGQUERY_COHORT_TABLE_ID - bq_cohort_dataset = settings.COHORT_DATASET_ID - cohort = cohort_id - else: - query_template = \ - ("SELECT Tumor_SampleBarcode" - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY Tumor_SampleBarcode; ") - - params = mutation_where_clause['value_tuple'][0] - - query = query_template.format(dataset_name=settings.BIGQUERY_DATASET, - project_name=settings.BIGQUERY_PROJECT_NAME, - table_name="Somatic_Mutation_calls", hugo_symbol=str(params['gene']), - var_class=params['var_class'], cohort_dataset=bq_cohort_dataset, - cohort_table=bq_cohort_table, cohort=cohort) - - bq_service = authorize_credentials_with_Google() - query_job = submit_bigquery_job(bq_service, settings.BQ_PROJECT_ID, query) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - retries = 0 - - while not job_is_done and retries < 10: - retries += 1 - sleep(1) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - results = get_bq_job_results(bq_service, query_job['jobReference']) - - # for-each result, add to list - - if results.__len__() > 0: - for barcode in results: - mutation_results[str(barcode['f'][0]['v'])] = 1 - - else: - print >> sys.stdout, "Mutation filter result was empty!" - # Put in one 'not found' entry to zero out the rest of the queries - barcodes = ['NONE_FOUND', ] - - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'features': {}, 'barcode': 'sample_barcode', 'project_id': None} - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - key = row['spec'] + ':' + row['attribute'] - valid_attrs[key] = {'name': row['attribute']} - sample_tables['metadata_samples']['features'][key] = row['attribute'] - if key in filters: - filters[key]['tables'] += ('metadata_samples',) - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - # Add all tables from each project - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = { - 'features':{}, - 'barcode':'sample_barcode', - 'project_id': project.id - } - - # Record features that should be in each sample table so we can know how and when we need to query - for feature in User_Feature_Definitions.objects.filter(project=project): - name = feature.feature_name - key = 'study:' + str(project.id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name} - - for tables in User_Data_Tables.objects.filter(project=feature.project_id): - sample_tables[tables.metadata_samples_table]['features'][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - results = [] - # Loop through the sample tables - for table, table_settings in sample_tables.items(): - # Make sure we should run the query here, or if we have filters that won't return anything, skip - should_be_queried = True - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - if not should_be_queried: - continue - - filter_copy = copy.deepcopy(filters) - where_clause = build_where_clause(filter_copy, table_settings['features']) - query = 'SELECT DISTINCT %s FROM %s' % (table_settings['barcode'], table) - if where_clause['query_str']: - query += ' WHERE ' + where_clause['query_str'] - cursor = db.cursor() - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - project_id = table_settings['project_id'] - if cohort_id and (project_id not in sample_ids or row[0] not in sample_ids[project_id]): - # This barcode was not in our cohort's list of barcodes, skip it - continue - if mutation_filters: - if row[0] in mutation_results: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id'])) - else: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id']) ) - cursor.close() - - db.close() - request_finished.send(self) - return SampleBarcodeList( items=results, count=len(results) ) - - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - ) - @endpoints.method(GET_RESOURCE, SampleBarcodeList, - path='metadata_participant_list', http_method='GET', - name='meta.metadata_participant_list') - def metadata_participant_list(self, request): - cursor = None - db = None - - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = [] - - for row in cursor.fetchall(): - sample_ids.append(row['sample_barcode']) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - cursor.close() - db.close() - return SampleBarcodeList(items=results, count=len(results)) - - except (TypeError, IndexError) as e: - raise endpoints.NotFoundException('Error in retrieving barcodes.') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataPlatformItemList, - path='metadata_platform_list', http_method='POST', - name='meta.metadata_platform_list') - def metadata_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - cursor = None - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(sample_query_str, (cohort_id,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs in metadata_platform_list for cohort '" + cohort_id + "': " + (stop - start).__str__()) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - - except (TypeError, IndexError) as e: - print e - cursor.close() - db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_platform_list for cohort '" + str(request.cohort_id) + "': " + (stop - start).__str__()) - data = [] - for row in cursor.fetchall(): - - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataPlatformItemList(items=data) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsPlatformItem, - path='metadata_counts_platform_list', http_method='POST', - name='meta.metadata_counts_platform_list') - def metadata_counts_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - samples_by_project = None - cohort_id = None - participants = 0 - user = get_current_user(request) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - samples = query_samples_and_studies(cohort_id, ) - - sample_ids = () - samples_by_project = {} - - for sample in samples: - sample_ids += (sample['sample_id'],) - if sample['project_id'] not in samples_by_project: - samples_by_project[sample['project_id']] = [] - samples_by_project[sample['project_id']].append(sample['sample_id']) - - participants = get_participant_count(sample_ids) - - start = time.time() - counts_and_total = count_metadata(user, cohort_id, samples_by_project, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts " - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - db = sql_connection() - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - data = [] - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_counts_platform_list for cohort '" + str( - request.cohort_id) + "': " + (stop - start).__str__()) - for row in cursor.fetchall(): - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataCountsPlatformItem(items=data, count=counts_and_total['counts'], - participants=counts_and_total['participants'], - total=counts_and_total['total']) - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Exception in metadata_counts_platforms_list.') diff --git a/api_3/pairwise.py b/api_3/pairwise.py deleted file mode 100755 index acb7f8ba..00000000 --- a/api_3/pairwise.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import json -import base64 -import logging -import urllib -import traceback - -from google.appengine.api import urlfetch -from django.conf import settings - -from bq_data_access.data_access import get_feature_vector -from bq_data_access.feature_value_types import ValueType -from bq_data_access.utils import VectorMergeSupport - -logger = logging.getLogger(__name__) - -class PairwiseInputVector(object): - def __init__(self, feature_id, value_type, data): - self.feature_id = feature_id - self.value_type = value_type - self.data = data - - -class Pairwise(object): - def __init__(self): - pass - - @classmethod - def prepare_features(self, cohort_id, features): - # Get the feature data - feature_vector_mapping = {} - vectors = [] - for feature in features: - value_type, vector = get_feature_vector(feature, cohort_id) - - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - vms = VectorMergeSupport('NA', 'sample_id', row_ids=features) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def prepare_feature_vector(self, input_vectors): - feature_vector_mapping = {} - vectors = [] - for item in input_vectors: - feature_id, value_type, vector = item.feature_id, item.value_type, item.data - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature_id] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - feature_ids = [v.feature_id for v in input_vectors] - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', row_ids=feature_ids) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def run_pairwise(self, feature_rows): - url = settings.PAIRWISE_SERVICE_URL - - data_dict = {} - row_count = 1 - for row in feature_rows: - label = "row_{count}".format(count=row_count) - data_dict[label] = row - row_count += 1 - - # Encode the data to be sent to the service - data = urllib.urlencode(data_dict) - decoded_response = None - - try: - pairwise_response = urlfetch.fetch(url=url, payload=data, method=urlfetch.POST) - response = pairwise_response.content - decoded_response = json.loads(base64.b64decode(response)) - except Exception as e: - decoded_response = None - logger.error(traceback.format_exc()) - - return decoded_response diff --git a/api_3/pairwise_api.py b/api_3/pairwise_api.py deleted file mode 100644 index 2e5a5e19..00000000 --- a/api_3/pairwise_api.py +++ /dev/null @@ -1,166 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -import endpoints -from protorpc import messages, remote, message_types -from django.conf import settings - -from api_3.api_helpers import authorize_credentials_with_Google -from api_3.pairwise import Pairwise - -package = 'pairwise' - - -class PairwiseJobRequest(messages.Message): - cohort_id = messages.StringField(1, required=True) - feature = messages.StringField(2, repeated=True) - - -class PairwiseResultVector(messages.Message): - feature_1 = messages.StringField(1, required=True) - feature_2 = messages.StringField(2, required=True) - comparison_type = messages.StringField(3, required=True) - correlation_coefficient = messages.StringField(4, required=True) - n = messages.IntegerField(5, required=True) - _logp = messages.FloatField(6, required=True) - n_A = messages.IntegerField(7, required=True) - p_A = messages.FloatField(8, required=True) - n_B = messages.IntegerField(9, required=True) - p_B = messages.FloatField(10, required=True) - exclusion_rules = messages.StringField(11, required=True) - - -class PairwiseFilterMessage(messages.Message): - filter_message = messages.StringField(1, required=True) - - -class PairwiseResults(messages.Message): - result_vectors = messages.MessageField(PairwiseResultVector, 1, repeated=True) - filter_messages = messages.MessageField(PairwiseFilterMessage, 2, repeated=True) - - -class Feature(messages.Message): - annotated_type = messages.StringField(1) - chr = messages.StringField(2) - start = messages.IntegerField(3) - end = messages.IntegerField(4) - label = messages.StringField(5) - mutation_count = messages.IntegerField(6) - source = messages.StringField(7) - - -class Association(messages.Message): - node1 = messages.MessageField(Feature, 1) - node2 = messages.MessageField(Feature, 2) - logged_pvalue = messages.FloatField(3) - - -class CircvizOutput(messages.Message): - items = messages.MessageField(Association, 1, repeated=True) - - -Pairwise_Endpoints = endpoints.api(name='pairwise', version='v3') - - -@Pairwise_Endpoints.api_class(resource_name='pairwise_api') -class PairwiseApi(remote.Service): - """Pairwise API v1""" - - @endpoints.method(PairwiseJobRequest, PairwiseResults, name="run", http_method="POST") - def run_job(self, request): - """ Used by the web application.""" - features = [] - count = len(request.feature) - 1 - while count >= 0: - features.append(str(request.feature[count])) - count -= 1 - - prepped_features = Pairwise.prepare_features(request.cohort_id, features) - outputs = Pairwise.run_pairwise(prepped_features) - - results = PairwiseResults(result_vectors=[], filter_messages=[]) - logging.info(results) - - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - - return results - - @endpoints.method(message_types.VoidMessage, CircvizOutput, - path='precomp', http_method='GET', name='precomputed') - def precomputed_results(self, request): - """ Used by the web application.""" - bq_table = 'brca_pwpv' - query = 'SELECT A_valueType, A_chr, A_startPos, A_endPos, A_featureName, A_N, A_dataType,' \ - 'B_valueType, B_chr, B_startPos, B_endPos, B_featureName, B_N, B_dataType,' \ - 'logP FROM [isb-cgc:test.brca_pwpv] ' \ - 'where B_chr != "null" ' \ - 'and A_chr != "null"' \ - 'and A_startPos != "null" and A_endPos != "null"' \ - 'and B_startPos != "null" and B_endPos != "null"' \ - 'LIMIT 50;' - query_body = { - 'query': query - } - bigquery_service = authorize_credentials_with_Google() - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=settings.BQ_PROJECT_ID, body=query_body).execute() - association_list = [] - feature_list = [] - for row in query_response['rows']: - node1 = Feature( - annotated_type=row['f'][0]['v'].encode('utf-8') if row['f'][0]['v'] else None, - chr=row['f'][1]['v'].encode('utf-8').replace('chr','') if row['f'][1]['v'] else None, - start=int(row['f'][2]['v']) if row['f'][2]['v'] else None, - end=int(row['f'][3]['v']) if row['f'][3]['v'] else None, - label=row['f'][4]['v'].encode('utf-8') if row['f'][4]['v'] else '', - mutation_count=int(row['f'][5]['v']) if row['f'][5]['v'] else None, - source=row['f'][6]['v'].encode('utf-8') if row['f'][6]['v'] else None - ) - node2 = Feature( - annotated_type=row['f'][7]['v'].encode('utf-8') if row['f'][7]['v'] else None, - chr=row['f'][8]['v'].encode('utf-8').replace('chr','') if row['f'][8]['v'] else None, - start=int(row['f'][9]['v']) if row['f'][9]['v'] else None, - end=int(row['f'][10]['v']) if row['f'][10]['v'] else None, - label=row['f'][11]['v'].encode('utf-8') if row['f'][11]['v'] else '', - mutation_count=int(row['f'][12]['v']) if row['f'][12]['v'] else None, - source=row['f'][13]['v'].encode('utf-8') if row['f'][13]['v'] else None - ) - logP = float(row['f'][14]['v']) - association_list.append(Association(node1=node1, node2=node2, logged_pvalue=logP)) - feature_list.append(node1) - feature_list.append(node2) - return CircvizOutput(items=association_list) - - -APPLICATION = endpoints.api_server([PairwiseApi]) diff --git a/api_3/patients_get_helper.py b/api_3/patients_get_helper.py deleted file mode 100755 index 1b664228..00000000 --- a/api_3/patients_get_helper.py +++ /dev/null @@ -1,203 +0,0 @@ -''' -Created on Apr 5, 2017 - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.cohort_endpoint_helpers import build_constructor_dict_for_message -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class CasesGetQueryBuilder(object): - - def build_queries(self, program, genomic_builds, count=1): - - clin_case_clause = 'case_barcode=%s' - case_clause = 'case_barcode=%s' - - if count > 1: - clin_case_clause = 'case_barcode in ({})'.format(",".join(['%s']*count)) - - clinical_query_str = 'select * ' \ - 'from {}_metadata_clinical ' \ - 'where {}'.format(program, clin_case_clause) - - sample_query_str = 'select sample_barcode ' \ - 'from {}_metadata_biospecimen ' \ - 'where {} ' \ - 'group by sample_barcode ' \ - 'order by sample_barcode'.format(program, case_clause) - - aliquot_query_str = '' - for genomic_build in genomic_builds: - part_aliquot_query_str = 'select aliquot_barcode ' \ - 'from {}_metadata_data_{} ' \ - 'where {} and aliquot_barcode is not null ' \ - 'group by aliquot_barcode '.format(program, genomic_build, case_clause) - if 0 < len(aliquot_query_str): - aliquot_query_str += ' UNION DISTINCT ' - aliquot_query_str += part_aliquot_query_str - - aliquot_query_str += 'order by aliquot_barcode' - - return clinical_query_str, sample_query_str, aliquot_query_str - - -class CaseGetListFilters(messages.Message): - case_barcodes = messages.StringField(1, repeated=True) - -class CasesGetHelper(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(case_barcode=messages.StringField(1, required=True)) - - POST_RESOURCE = endpoints.ResourceContainer( - CaseGetListFilters - ) - - def get(self, request, CaseDetails, MetadataItem, program): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case (*eg* TCGA-B9-7268 for TCGA) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - case_barcode = request.get_assigned_value('case_barcode') - query_tuple = (str(case_barcode),) - genomic_builds = ['HG19'] - if program != 'CCLE': - genomic_builds.append('HG38') - - clinical_query_str, sample_query_str, aliquot_query_str = CasesGetQueryBuilder().build_queries(program, genomic_builds) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build clinical data message - cursor.execute(clinical_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - logger.warn("Case barcode {} not found in {}_metadata_clinical table.".format(case_barcode, program)) - raise endpoints.NotFoundException("Case barcode {} not found".format(case_barcode)) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - clinical_data_item = MetadataItem(**constructor_dict) - - # get list of samples - cursor.execute(sample_query_str, query_tuple) - sample_list = [row['sample_barcode'] for row in cursor.fetchall()] - - # get list of aliquots - cursor.execute(aliquot_query_str, (query_tuple * len(genomic_builds))) - aliquot_list = [row['aliquot_barcode'] for row in cursor.fetchall()] - - return CaseDetails(clinical_data=clinical_data_item, samples=sample_list, aliquots=aliquot_list if aliquot_list else []) - except (IndexError, TypeError), e: - logger.info("Case {} not found. Error: {}".format(case_barcode, e)) - raise endpoints.NotFoundException("Case {} not found.".format(case_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving case, sample, or aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving case, sample, or aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - - def get_list(self, request, CaseSetDetails, CaseDetails, MetadataItem, program): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case (*eg* TCGA-B9-7268 for TCGA) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - case_barcodes = request.get_assigned_value('case_barcodes') if 'case_barcodes' in [k.name for k in request.all_fields()] else None - - if not case_barcodes or not len(case_barcodes): - raise endpoints.BadRequestException("A list of case barcodes is required.") - elif len(case_barcodes) > 500: - raise endpoints.BadRequestException("The limit on barcodes per request is 500.") - - query_tuple = [x for x in case_barcodes] - genomic_builds = ['HG19'] - if program != 'CCLE': - genomic_builds.append('HG38') - - clinical_query_str, sample_query_str, aliquot_query_str = CasesGetQueryBuilder().build_queries(program, genomic_builds, len(case_barcodes)) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build clinical data message - cursor.execute(clinical_query_str, query_tuple) - rows = cursor.fetchall() - if not len(rows): - cursor.close() - db.close() - logger.warn("None of the case barcodes were found in the {}_metadata_clinical table.".format(program)) - raise endpoints.NotFoundException("None of the case barcodes were found") - - case_details = [] - - for row in rows: - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - clinical_data_item = MetadataItem(**constructor_dict) - - # get list of samples - cursor.execute(sample_query_str, (row['case_barcode'],)) - sample_list = [sample_row['sample_barcode'] for sample_row in cursor.fetchall()] - - # get list of aliquots - cursor.execute(aliquot_query_str, ((row['case_barcode'],) * len(genomic_builds))) - aliquot_list = [aliquot_row['aliquot_barcode'] for aliquot_row in cursor.fetchall()] - - case_details.append( - CaseDetails( - clinical_data=clinical_data_item, samples=sample_list, - aliquots=aliquot_list if aliquot_list else [], case_barcode=row['case_barcode']) - ) - - return CaseSetDetails(cases=case_details) - except (IndexError, TypeError), e: - logger.exception(e) - logger.info("The cases supplied were not found. Error: {}".format(e)) - raise endpoints.NotFoundException("The cases provided were not found not found.") - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving case, sample, or aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving case, sample, or aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api_3/samples_get_helper.py b/api_3/samples_get_helper.py deleted file mode 100755 index 735ac6ed..00000000 --- a/api_3/samples_get_helper.py +++ /dev/null @@ -1,345 +0,0 @@ -''' -Created on Apr 5, 2017 - -opyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages -from api_3.isb_cgc_api_TCGA.message_classes import BiospecimenMetadataItem - -from api_3.api_helpers import sql_connection -from api_3.cohort_endpoint_helpers import build_constructor_dict_for_message - -logger = logging.getLogger(__name__) - -class SamplesGetQueryBuilder(object): - def build_aliquot_query(self, program, param_list, count=1): - if 'CCLE' == program: - genomic_builds = ['HG19'] - else: - genomic_builds = ['HG19', 'HG38'] - - aliquot_query_str = '' - for genomic_build in genomic_builds: - part_aliquot_query_str = 'select sample_barcode, case_barcode, aliquot_barcode, aliquot_gdc_id ' \ - 'from {}_metadata_data_{} ' \ - 'where file_name_key is not null and file_name_key !="" '.format(program, genomic_build) - for column in param_list: - if column == 'sample_barcode' and count>1: - part_aliquot_query_str += ' and {} IN ({}) '.format(column, ','.join(["%s"]*count)) - else: - part_aliquot_query_str += ' and {}=%s '.format(column) - if 0 < len(aliquot_query_str): - aliquot_query_str += ' union ' - aliquot_query_str += part_aliquot_query_str - - return aliquot_query_str - - def build_biospecimen_query(self, program, count=1): - - biospecimen_query_str = 'select * ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode=%s'.format(program) - if count>1: - biospecimen_query_str = 'select * ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode in ({})'.format(program, ','.join(["%s"] * count)) - - return biospecimen_query_str - - def build_data_query(self, program, datadict_class, param_list, count=1): - if 'CCLE' == program: - genomic_builds = ['HG19'] - else: - genomic_builds = ['HG19', 'HG38'] - - data_query_str = '' - for genomic_build in genomic_builds: - part_data_query_str = 'select {0} ' \ - 'from {1}_metadata_data_{2} ' \ - 'where file_name_key is not null and file_name_key !="" '.format(', '.join(field.name for field in datadict_class.all_fields()), program, genomic_build) - for column in param_list: - if column == 'sample_barcode' and count > 1: - part_data_query_str += ' and {} IN ({}) '.format(column, ','.join(["%s"]*count)) - else: - part_data_query_str += ' and {}=%s '.format(column) - if 0 < len(data_query_str): - data_query_str += ' union ' - data_query_str += part_data_query_str - - return data_query_str - - def build_case_query(self, program, count=1): - - case_query_str = 'select case_barcode, case_gdc_id ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode=%s ' \ - 'group by case_barcode, case_gdc_id'.format(program) - - if count>1: - case_query_str = 'select case_barcode, case_gdc_id ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode in ({}) ' \ - 'group by case_barcode, case_gdc_id'.format(program, ','.join(["%s"] * count)) - - return case_query_str - - -class DataDetails(messages.Message): - file_gdc_id = messages.StringField(1) - file_name = messages.StringField(2) - file_name_key = messages.StringField(3) - file_size = messages.IntegerField(4, variant=messages.Variant.INT64) - sample_gdc_id = messages.StringField(5) - sample_barcode = messages.StringField(6) - sample_type = messages.StringField(7) - project_short_name = messages.StringField(8) - disease_code = messages.StringField(9) - program_name = messages.StringField(11) - data_type = messages.StringField(12) - data_category = messages.StringField(13) - experimental_strategy = messages.StringField(14) - data_format = messages.StringField(15) - access = messages.StringField(16) - platform = messages.StringField(17) - endpoint_type = messages.StringField(18) - analysis_workflow_type = messages.StringField(19) - index_file_name = messages.StringField(20) - - -class SampleGetListFilters(messages.Message): - sample_barcodes = messages.StringField(1, repeated=True) - disease_code = messages.StringField(2, repeated=True) - experimental_strategy = messages.StringField(3, repeated=True) - platform = messages.StringField(4, repeated=True) - data_category = messages.StringField(5, repeated=True) - data_type = messages.StringField(6, repeated=True) - data_format = messages.StringField(7, repeated=True) - project_short_name = messages.StringField(8, repeated=True) - analysis_workflow_type = messages.StringField(9, repeated=True) - - -class SamplesGetAPI(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer( - sample_barcode=messages.StringField(1, required=True), - data_type = messages.StringField(2), - data_category = messages.StringField(3), - experimental_strategy = messages.StringField(4), - data_format = messages.StringField(5), - platform = messages.StringField(6), - endpoint_type = messages.StringField(7), - analysis_workflow_type = messages.StringField(8) - ) - - POST_RESOURCE = endpoints.ResourceContainer( - SampleGetListFilters - ) - - def get(self, request, program, SampleDetails, MetadataItem): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A, for TCGA), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - param_list = ['sample_barcode'] - query_tuple = [sample_barcode] - extra_query_tuple = [sample_barcode] - for field in request.all_fields(): - if 'sample_barcode' != field.name and request.get_assigned_value(field.name): - param_list += [field.name] - extra_query_tuple += [request.get_assigned_value(field.name)] - - if 'CCLE' != program: - extra_query_tuple += extra_query_tuple - - # need to take into account params used in the union between the genomic builds - - aliquot_query_str = SamplesGetQueryBuilder().build_aliquot_query(program, param_list) - biospecimen_query_str = SamplesGetQueryBuilder().build_biospecimen_query(program) - data_query_str = SamplesGetQueryBuilder().build_data_query(program, DataDetails(), param_list) - case_query_str = SamplesGetQueryBuilder().build_case_query(program) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build biospecimen data message - cursor.execute(biospecimen_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - error_message = "Sample barcode {} not found in {}_metadata_biospecimen table.".format(query_tuple[0], program) - raise endpoints.NotFoundException(error_message) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - biospecimen_data_item = MetadataItem(**constructor_dict) - - # get list of aliquots - cursor.execute(aliquot_query_str, extra_query_tuple) - aliquot_list = [row['aliquot_barcode'] for row in cursor.fetchall()] - - # get case barcode (superfluous?) - cursor.execute(case_query_str, query_tuple) - row = cursor.fetchone() - case_barcode = str(row["case_barcode"]) - case_gdc_id = str(row["case_gdc_id"]) - - # prepare to build list of data details messages - cursor.execute(data_query_str, extra_query_tuple) - cursor_rows = cursor.fetchall() - - # build a data details message for each row returned from metadata_data table - data_details_list = [] - for row in cursor_rows: - constructor_dict = build_constructor_dict_for_message(DataDetails(), row) - data_details_item = DataDetails(**constructor_dict) - data_details_list.append(data_details_item) - - return SampleDetails(aliquots=aliquot_list, - biospecimen_data=biospecimen_data_item, - data_details=data_details_list, - data_details_count=len(data_details_list), - case_barcode=case_barcode, - case_gdc_id=case_gdc_id) - - except (IndexError, TypeError) as e: - logger.info("Sample details for barcode {} not found. Error: {}, \nSQL: {}, \nParams: {}".format(sample_barcode, e, aliquot_query_str, extra_query_tuple)) - raise endpoints.NotFoundException( - "Sample details for barcode {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn(e) - raise endpoints.BadRequestException("Error retrieving biospecimen, case, or other data. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - def get_list(self, request, program, SampleSetDetails, SampleDetails, MetadataItem): - """ - Given a set of sample barcodes (of length 16, *eg* TCGA-B9-7268-01A, for TCGA), this endpoint returns - all available "biospecimen" information about these samples, the associated case barcodes, - a list of associated aliquots, and a list of "data_details" blocks describing each of the data files - associated with this sample - """ - cursor = None - db = None - sample_barcodes = request.get_assigned_value('sample_barcodes') if 'sample_barcodes' in [k.name for k in - request.all_fields()] else None - - if not sample_barcodes: - raise endpoints.BadRequestException("A list of sample barcodes is required.") - elif len(sample_barcodes) > 500: - raise endpoints.BadRequestException("There is a 500 barcode limit per quest.") - - param_list = ['sample_barcode'] - query_tuple = [x for x in sample_barcodes] - extra_query_tuple = [x for x in sample_barcodes] - for field in request.all_fields(): - if 'sample_barcodes' not in field.name and request.get_assigned_value(field.name): - param_list += [field.name] - extra_query_tuple += [request.get_assigned_value(field.name)] - - if 'CCLE' != program: - extra_query_tuple += extra_query_tuple - - # need to take into account params used in the union between the genomic builds - - aliquot_query_str = SamplesGetQueryBuilder().build_aliquot_query(program, param_list, len(sample_barcodes)) - biospecimen_query_str = SamplesGetQueryBuilder().build_biospecimen_query(program, len(sample_barcodes)) - data_query_str = SamplesGetQueryBuilder().build_data_query(program, DataDetails(), param_list, len(sample_barcodes)) - - samples = SampleSetDetails() - - sample_data = {} - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build biospecimen data message - cursor.execute(biospecimen_query_str, query_tuple) - rows = cursor.fetchall() - if not len(rows): - cursor.close() - db.close() - error_message = "These sample barcodes were not found in the {}_metadata_biospecimen table.".format(program,) - raise endpoints.NotFoundException(error_message) - - for row in rows: - sample_data[row['sample_barcode']] = {} - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - biospecimen_data_item = MetadataItem(**constructor_dict) - sample_data[row['sample_barcode']]['biospecimen'] = biospecimen_data_item - sample_data[row['sample_barcode']]['case_barcode'] = row['case_barcode'] - sample_data[row['sample_barcode']]['case_gdc_id'] = row['case_gdc_id'] - - # get list of aliquots - cursor.execute(aliquot_query_str, extra_query_tuple) - rows = cursor.fetchall() - for row in rows: - if 'aliquots' not in sample_data[row['sample_barcode']]: - sample_data[row['sample_barcode']]['aliquots'] = [] - sample_data[row['sample_barcode']]['aliquots'].append(row['aliquot_barcode']) - - # prepare to build list of data details messages - cursor.execute(data_query_str, extra_query_tuple) - cursor_rows = cursor.fetchall() - - # build a data details message for each row returned from metadata_data table - for row in cursor_rows: - if 'data_rows' not in sample_data[row['sample_barcode']]: - sample_data[row['sample_barcode']]['data_rows'] = [] - constructor_dict = build_constructor_dict_for_message(DataDetails(), row) - data_details_item = DataDetails(**constructor_dict) - sample_data[row['sample_barcode']]['data_rows'].append(data_details_item) - - for sample in sample_data: - samples.samples.append( - SampleDetails( - aliquots=sample_data[sample]['aliquots'] if 'aliquots' in sample_data[sample] else [], - biospecimen_data=sample_data[sample]['biospecimen'], - data_details=sample_data[sample]['data_rows'] if 'data_rows' in sample_data[sample] else [], - data_details_count=len(sample_data[sample]['data_rows']) if 'data_rows' in sample_data[sample] else 0, - case_barcode=sample_data[sample]['case_barcode'], - case_gdc_id=sample_data[sample]['case_gdc_id'], - sample_barcode=sample - ) - ) - - return samples - - except (IndexError, TypeError) as e: - logger.exception(e) - logger.error("Error: {}, \nSQL: {}, \nParams: {}".format(e,aliquot_query_str,extra_query_tuple)) - raise endpoints.NotFoundException( - "There was an error while processing your request--please contact us at feedback@isb-cgc.org" - ) - except MySQLdb.ProgrammingError as e: - logger.warn(e) - raise endpoints.BadRequestException("Error retrieving biospecimen, case, or other data. {}".format(e)) - except Exception as e: - logger.error("[ERROR] While processing samples.get_list: ") - logger.exception(e) - finally: - if cursor: cursor.close() - if db and db.open: db.close() diff --git a/api_3/schema/__init__.py b/api_3/schema/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/schema/tcga_clinical.py b/api_3/schema/tcga_clinical.py deleted file mode 100755 index 0725770f..00000000 --- a/api_3/schema/tcga_clinical.py +++ /dev/null @@ -1,283 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -# Updated from -# https://github.com/isb-cgc/data-prototyping/blob/537c5c24646f87bda804ca95dee6cf479f0b1fb9/tcga_etl_pipeline/schemas/clinical.json - -schema = [ - { - "type": "STRING", - "name": "ParticipantBarcode" - }, - { - "type": "STRING", - "name": "Study" - }, - { - "type": "STRING", - "name": "Project" - }, - { - "type": "STRING", - "name": "ParticipantUUID" - }, - { - "type": "STRING", - "name": "TSSCode" - }, - { - "type": "INTEGER", - "name": "age_at_diagnosis" - }, - { - "type": "STRING", - "name": "anatomic_neoplasm_subdivision" - }, - { - "type": "INTEGER", - "name": "batch_number" - }, - { - "type": "STRING", - "name": "bcr" - }, - { - "type": "STRING", - "name": "clinical_M" - }, - { - "type": "STRING", - "name": "clinical_N" - }, - { - "type": "STRING", - "name": "clinical_T" - }, - { - "type": "STRING", - "name": "clinical_stage" - }, - { - "type": "STRING", - "name": "colorectal_cancer" - }, - { - "type": "STRING", - "name": "country" - }, - { - "type": "STRING", - "name": "vital_status" - }, - { - "type": "INTEGER", - "name": "days_to_birth" - }, - { - "type": "INTEGER", - "name": "days_to_death" - }, - { - "type": "INTEGER", - "name": "days_to_last_known_alive" - }, - { - "type": "INTEGER", - "name": "days_to_last_followup" - }, - { - "type": "INTEGER", - "name": "days_to_initial_pathologic_diagnosis" - }, - { - "type": "INTEGER", - "name": "days_to_submitted_specimen_dx" - }, - { - "type": "STRING", - "name": "ethnicity" - }, - { - "type": "STRING", - "name": "frozen_specimen_anatomic_site" - }, - { - "type": "STRING", - "name": "gender" - }, - { - "type": "FLOAT", - "name": "gleason_score_combined" - }, - { - "type": "STRING", - "name": "histological_type" - }, - { - "type": "STRING", - "name": "history_of_colon_polyps" - }, - { - "type": "STRING", - "name": "history_of_neoadjuvant_treatment" - }, - { - "type": "STRING", - "name": "hpv_calls" - }, - { - "type": "STRING", - "name": "hpv_status" - }, - { - "type": "STRING", - "name": "icd_10" - }, - { - "type": "STRING", - "name": "icd_o_3_histology" - }, - { - "type": "STRING", - "name": "icd_o_3_site" - }, - { - "type": "STRING", - "name": "lymphatic_invasion" - }, - { - "type": "STRING", - "name": "lymphnodes_examined" - }, - { - "type": "STRING", - "name": "lymphovascular_invasion_present" - }, - { - "type": "STRING", - "name": "menopause_status" - }, - { - "type": "STRING", - "name": "mononucleotide_and_dinucleotide_marker_panel_analysis_status" - }, - { - "type": "FLOAT", - "name": "mononucleotide_marker_panel_analysis_status" - }, - { - "type": "STRING", - "name": "neoplasm_histologic_grade" - }, - { - "type": "STRING", - "name": "new_tumor_event_after_initial_treatment" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_examined" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_positive_by_he" - }, - { - "type": "FLOAT", - "name": "number_pack_years_smoked" - }, - { - "type": "INTEGER", - "name": "year_of_initial_pathologic_diagnosis" - }, - { - "type": "STRING", - "name": "pathologic_M" - }, - { - "type": "STRING", - "name": "pathologic_N" - }, - { - "type": "STRING", - "name": "pathologic_T" - }, - { - "type": "STRING", - "name": "pathologic_stage" - }, - { - "type": "STRING", - "name": "person_neoplasm_cancer_status" - }, - { - "type": "STRING", - "name": "pregnancies" - }, - { - "type": "STRING", - "name": "primary_neoplasm_melanoma_dx" - }, - { - "type": "STRING", - "name": "primary_therapy_outcome_success" - }, - { - "type": "STRING", - "name": "prior_dx" - }, - { - "type": "FLOAT", - "name": "psa_value" - }, - { - "type": "STRING", - "name": "race" - }, - { - "type": "STRING", - "name": "residual_tumor" - }, - { - "type": "STRING", - "name": "tobacco_smoking_history" - }, - { - "type": "STRING", - "name": "tumor_tissue_site" - }, - { - "type": "STRING", - "name": "tumor_type" - }, - { - "type": "STRING", - "name": "venous_invasion" - }, - { - "type": "FLOAT", - "name": "weight" - }, - { - "type": "FLOAT", - "name": "height" - }, - { - "type": "FLOAT", - "name": "BMI" - } -] \ No newline at end of file diff --git a/api_3/seqpeek_api.py b/api_3/seqpeek_api.py deleted file mode 100644 index 7aee1aa7..00000000 --- a/api_3/seqpeek_api.py +++ /dev/null @@ -1,94 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from api_3.data_access import PlotDataCohortInfo - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class MAFRecord(Message): - sample_id = StringField(1) - patient_id = StringField(2) - aliquot_id = StringField(3) - hugo_symbol = StringField(4) - uniprot_aapos = IntegerField(5, variant=Variant.INT32) - uniprot_id = StringField(6) - variant_classification = StringField(7) - cohort = IntegerField(8, repeated=True) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 2, repeated=True) - -SeqPeekDataEndpointsAPI = endpoints_api(name='seqpeek_data_api', version='v1', - description='Endpoints used by the seqpeek visualization in the web application.') - - -def maf_array_to_record(maf_array): - data_points = [] - for item in maf_array: - data_points.append(MAFRecord(**item)) - - return data_points - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekDataAccessAPI(remote.Service): - - def create_response(self, maf_with_cohorts): - - data_points = maf_array_to_record(maf_with_cohorts.maf_vector) - - cohort_info_obj_array = [] - for item in maf_with_cohorts.cohort_info: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return MAFRecordList(items=data_points, cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, MAFRecordList, - path='by_gnab_feature', http_method='GET', name='seqpeek.getMAFDataWithCohorts') - def data_access_by_gnab_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - - maf_with_cohorts = SeqPeekMAFDataFormatter().format_maf_vector_for_view(feature_id, cohort_id_array) - response = self.create_response(maf_with_cohorts) - return response - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/seqpeek_view_api.py b/api_3/seqpeek_view_api.py deleted file mode 100644 index 8e69d7be..00000000 --- a/api_3/seqpeek_view_api.py +++ /dev/null @@ -1,238 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from endpoints import method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant - -from bq_data_access.seqpeek.seqpeek_view import SeqPeekViewDataBuilder -from bq_data_access.data_access import get_feature_vectors_tcga_only -from api_3.seqpeek_api import SeqPeekDataEndpointsAPI, MAFRecord, maf_array_to_record -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter -from bq_data_access.seqpeek_maf_data import SeqPeekDataProvider -from bq_data_access.data_access import ProviderClassQueryDescription -from api_3.data_access import fetch_isbcgc_project_set -from api_3.api_helpers import sql_connection - -from projects.models import Project - -class SeqPeekViewDataRequest(Message): - hugo_symbol = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class InterproMatchLocation(Message): - # TODO this should likely be a float - score = IntegerField(1, variant=Variant.INT32) - start = IntegerField(2, variant=Variant.INT32) - end = IntegerField(3, variant=Variant.INT32) - - -class InterproMatch(Message): - status = StringField(1) - name = StringField(2) - evd = StringField(3) - locations = MessageField(InterproMatchLocation, 4, repeated=True) - dbname = StringField(5) - id = StringField(6) - - -class InterproJson(Message): - matches = MessageField(InterproMatch, 1, repeated=True) - uniprot_id = StringField(2) - length = IntegerField(3, variant=Variant.INT32) - name = StringField(4) - - -class InterproItem(Message): - uniprot_id = StringField(1) - interpro_json = MessageField(InterproJson, 2, repeated=False) - - -class SeqPeekRegionRecord(Message): - type = StringField(1, required=True) - start = IntegerField(2, required=True, variant=Variant.INT32) - end = IntegerField(3, required=True, variant=Variant.INT32) - - -class SeqPeekTrackRecord(Message): - mutations = MessageField(MAFRecord, 1, repeated=True) - type = StringField(2, required=True) - label = StringField(3, required=True) - number_of_samples = IntegerField(4, required=True) - mutated_positions = IntegerField(5, required=True) - cohort_size = IntegerField(6, required=False) - row_id = StringField(7, required=True) - - -class SeqPeekViewPlotDataRecord(Message): - tracks = MessageField(SeqPeekTrackRecord, 1, repeated=True) - protein = MessageField(InterproItem, 2, required=False) - regions = MessageField(SeqPeekRegionRecord, 3, repeated=True) - - -class SeqPeekRemovedRow(Message): - name = StringField(1, required=True) - num = IntegerField(2, required=True) - - -class SeqPeekViewRecord(Message): - cohort_id_list = StringField(1, repeated=True) - hugo_symbol = StringField(2, required=True) - plot_data = MessageField(SeqPeekViewPlotDataRecord, 3, required=True) - removed_row_statistics = MessageField(SeqPeekRemovedRow, 4, repeated=True) - - -def create_interpro_record(interpro_literal): - match_data = [] - for match in interpro_literal['matches']: - - match_location_data = [] - for location in match['locations']: - match_location_data.append(InterproMatchLocation( - score=int(location['score']), - start=int(location['start']), - end=int(location['end']) - )) - - match_data.append(InterproMatch( - status=str(match['status']), - name=str(match['name']), - evd=str(match['evd']), - locations=match_location_data, - dbname=str(match['dbname']), - id=str(match['id']), - )) - - interpro_json = InterproJson( - matches=match_data, - uniprot_id=str(interpro_literal['uniprot_id']), - length=int(interpro_literal['length']), - name=str(interpro_literal['name']) - ) - - return InterproItem( - interpro_json=interpro_json - ) - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekViewDataAccessAPI(remote.Service): - def build_gnab_feature_id(self, gene_label): - return "GNAB:{gene_label}:variant_classification".format(gene_label=gene_label) - - def create_response(self, seqpeek_view_data): - plot_data = seqpeek_view_data['plot_data'] - tracks = [] - for track in plot_data['tracks']: - mutations = maf_array_to_record(track['mutations']) - tracks.append(SeqPeekTrackRecord(mutations=mutations, label=track['label'], type=track["type"], - row_id=track['render_info']['row_id'], - number_of_samples=track['statistics']['samples']['numberOf'], - mutated_positions=track['statistics']['samples']['mutated_positions'], - cohort_size=track['statistics']['cohort_size'])) - - region_records = [] - for region in plot_data['regions']: - region_records.append(SeqPeekRegionRecord(**region)) - - protein = create_interpro_record(plot_data['protein']) - plot_data_record = SeqPeekViewPlotDataRecord(tracks=tracks, protein=protein, regions=region_records) - - removed_row_statistics = [] - for item in seqpeek_view_data['removed_row_statistics']: - removed_row_statistics.append(SeqPeekRemovedRow(**item)) - - return SeqPeekViewRecord(plot_data=plot_data_record, hugo_symbol=seqpeek_view_data['hugo_symbol'], - cohort_id_list=seqpeek_view_data['cohort_id_list'], - removed_row_statistics=removed_row_statistics) - - @endpoints_method(SeqPeekViewDataRequest, SeqPeekViewRecord, - path='view_data', http_method='GET', name='seqpeek.getViewData') - def seqpeek_view_data(self, request): - try: - hugo_symbol = request.hugo_symbol - cohort_id_array = request.cohort_id - - gnab_feature_id = self.build_gnab_feature_id(hugo_symbol) - logging.debug("GNAB feature ID for SeqPeke: {0}".format(gnab_feature_id)) - - # Get the project IDs these cohorts' samples come from - cohort_vals = tuple(int(i) for i in cohort_id_array) - cohort_params = ('%s,' * len(cohort_id_array))[:-1] - - db = sql_connection() - cursor = db.cursor() - - isbcgc_projects = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN (%s);" % cohort_params, - cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_project_ids = [] - unconfirmed_project_ids = [] - - for row in cursor.fetchall(): - if row[0] in isbcgc_projects: - if row[0] not in confirmed_project_ids: - confirmed_project_ids.append(row[0]) - elif row[0] not in unconfirmed_project_ids: - unconfirmed_project_ids.append(row[0]) - - if len(unconfirmed_project_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_project_ids) - - for proj in projects: - if proj.get_my_root_and_depth()['root'] in isbcgc_projects: - confirmed_project_ids.append(proj.id) - - async_params = [ProviderClassQueryDescription(SeqPeekDataProvider, gnab_feature_id, cohort_id_array, confirmed_project_ids)] - maf_data_result = get_feature_vectors_tcga_only(async_params, skip_formatting_for_plot=True) - - maf_data_vector = maf_data_result[gnab_feature_id]['data'] - - if len(maf_data_vector) > 0: - # Since the gene (hugo_symbol) parameter is part of the GNAB feature ID, - # it will be sanity-checked in the SeqPeekMAFDataAccess instance. - seqpeek_data = SeqPeekMAFDataFormatter().format_maf_vector_for_view(maf_data_vector, cohort_id_array) - - seqpeek_maf_vector = seqpeek_data.maf_vector - seqpeek_cohort_info = seqpeek_data.cohort_info - removed_row_statistics_dict = seqpeek_data.removed_row_statistics - - seqpeek_view_data = SeqPeekViewDataBuilder().build_view_data(hugo_symbol, - seqpeek_maf_vector, - seqpeek_cohort_info, - cohort_id_array, - removed_row_statistics_dict) - - response = self.create_response(seqpeek_view_data) - return response - else: - # No data found - return SeqPeekViewRecord(plot_data=SeqPeekViewPlotDataRecord(tracks=[], protein=None, regions=[]), - hugo_symbol=hugo_symbol, cohort_id_list=[str(i) for i in cohort_id_array], - removed_row_statistics=[]) - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - diff --git a/api_3/single_feature_access.py b/api_3/single_feature_access.py deleted file mode 100644 index 916cb48c..00000000 --- a/api_3/single_feature_access.py +++ /dev/null @@ -1,131 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import time - -from api_3.data_access import FeatureDataEndpointsAPI, PlotDataCohortInfo - -from endpoints import method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class DataPointList(Message): - type = EnumField(ValueType, 1, required=True) - items = MessageField(DataPoint, 2, repeated=True) - label = StringField(3, required=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class SingleFeatureDataAccess(remote.Service): - def get_feature_vector(self, feature_id, cohort_id_array): - start = time.time() - - async_params = [(feature_id, cohort_id_array)] - async_result = get_feature_vectors_with_user_data(async_params) - - feature_type, feature_vec = async_result[feature_id]['type'], async_result[feature_id]['data'] - - end = time.time() - time_elapsed = end-start - logging.info('Time elapsed: ' + str(time_elapsed)) - - vms = VectorMergeSupport('NA', 'sample_id', [feature_id]) - vms.add_dict_array(feature_vec, feature_id, 'value') - - merged = vms.get_merged_dict() - - return feature_type, merged - - def annotate_vector_with_cohorts(self, cohort_id_array, merged): - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - value_bundle['cohort'] = cohort_set - - def get_cohort_information(self, cohort_id_array): - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - - return cohort_info_array - - def create_response(self, feature_id, vector_type, vector, cohort_info_array): - data_points = [] - for item in vector: - data_points.append(DataPoint( - sample_id=item['sample_id'], - value=item[feature_id], - cohort=item['cohort'] - )) - - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return DataPointList(type=vector_type, items=data_points, label=feature_id, - cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, DataPointList, - path='feature_data', http_method='GET', name='feature_access.getFeatureData') - def data_access_by_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - vector_type, vector = self.get_feature_vector(feature_id, cohort_id_array) - - self.annotate_vector_with_cohorts(cohort_id_array, vector) - - cohort_info = self.get_cohort_information(cohort_id_array) - response = self.create_response(feature_id, vector_type, vector, cohort_info) - return response - except FeatureNotFoundException as fnf: - logging.error("Invalid internal feature ID '{}'".format(str(fnf))) - raise NotFoundException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/test/__init__.py b/api_3/test/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/users_get_common.py b/api_3/users_get_common.py deleted file mode 100755 index 91aa9e61..00000000 --- a/api_3/users_get_common.py +++ /dev/null @@ -1,102 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import logging - -import endpoints -from protorpc import remote, messages - -from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned - -from accounts.models import AuthorizedDataset, NIH_User, UserAuthorizedDatasets -from dataset_utils.dataset_access_support_factory import DatasetAccessSupportFactory - -logger = logging.getLogger(__name__) - -class UserGetAPIReturnJSON(messages.Message): - message = messages.StringField(1) - dbGaP_authorized = messages.BooleanField(2) - dbGaP_allowed = messages.BooleanField(3) - -class UserGetAPICommon(remote.Service): - def get(self, program): - ''' - Returns the dbGaP authorization status of the user. - ''' - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException("Authentication unsuccessful.") - - authorized = False - allowed = False - try: - user = User.objects.get(email=user_email) - except ObjectDoesNotExist, MultipleObjectsReturned: - user = None - - if user: - # FIND NIH_USER FOR USER - try: - nih_user = NIH_User.objects.filter(user=user).first() - except: - nih_user = None - - # IF USER HAS LINKED ERA COMMONS ID - if nih_user: - # FIND ALL DATASETS USER HAS ACCESS TO - user_auth_datasets = AuthorizedDataset.objects.filter(id__in=UserAuthorizedDatasets.objects.filter(nih_user_id=nih_user.id).values_list('authorized_dataset', flat=True)) - for dataset in user_auth_datasets: - if program in dataset.name: - authorized = True - allowed = True - if not allowed: - das = DatasetAccessSupportFactory.from_webapp_django_settings() - authorized_datasets = das.get_datasets_for_era_login(nih_user.NIH_username) - for dataset in authorized_datasets: - try: - ad = AuthorizedDataset.objects.get(whitelist_id=dataset.dataset_id) - if program in ad.name: - allowed = True - except (ObjectDoesNotExist) as e: - logger.exception('didn\'t find an expected authorized dataset for {}: {}'.format(dataset, e)) - raise - except (MultipleObjectsReturned) as e: - authdatasets = AuthorizedDataset.objects.filter(whitelist_id=dataset.dataset_id).values_list('name',flat=True) - logger.exception('found more than one expected authorized dataset for {} named {}: {}'.format(dataset, ", ".join(authdatasets), e)) - raise - - if not allowed: - return UserGetAPIReturnJSON(message="{} is not on the controlled-access google group.".format(user_email), - dbGaP_authorized=False, - dbGaP_allowed=False) - # since user has access to the program controlled data, include a warning - warn_message = 'You are reminded that when accessing controlled access information you are bound by the dbGaP DATA USE CERTIFICATION AGREEMENT (DUCA) for this dataset.' - if authorized: - return UserGetAPIReturnJSON(message="{} has dbGaP authorization for {} and is a member of the controlled-access google group. {}" - .format(user_email, program, warn_message), - dbGaP_authorized=True, - dbGaP_allowed=True) - else: - return UserGetAPIReturnJSON(message="{} has dbGaP authorization for {} but is not currently a member of the controlled-access google group. {}" - .format(user_email, program, warn_message), - dbGaP_authorized=False, - dbGaP_allowed=True) diff --git a/appengine_config.py b/appengine_config.py index 565aaa4f..4c2433ed 100644 --- a/appengine_config.py +++ b/appengine_config.py @@ -12,9 +12,6 @@ def unload_module(module_name): # Add any libraries installed in the "lib" folder. vendor.add('lib') -if bool(os.environ.get('LOCAL_DEV', 'False') == 'True'): - print "Local development detected, vendoring in endpoints_lib..." - vendor.add('endpoints_lib') # The default endpoints/GAE oauth2 is way too old. unload_module('oauth2client') diff --git a/bq_data_access/__init__.py b/bq_data_access/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/bq_data_access/clinical_data.py b/bq_data_access/clinical_data.py deleted file mode 100755 index 6a6afba6..00000000 --- a/bq_data_access/clinical_data.py +++ /dev/null @@ -1,190 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import sys -from re import compile as re_compile - -from api.schema.tcga_clinical import schema as clinical_schema - -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import DataTypes, BigQuerySchemaToValueTypeConverter -from bq_data_access.utils import DurationLogged - -CLINICAL_FEATURE_TYPE = 'CLIN' - -BSP_TABLE_NAME = 'Biospecimen_data' - -class InvalidClinicalFeatureIDException(Exception): - def __init__(self, feature_id, reason): - self.feature_id = feature_id - self.reason = reason - - def __str__(self): - return "Invalid internal clinical feature ID '{feature_id}', reason '{reason}'".format( - feature_id=self.feature_id, - reason=self.reason - ) - - -class ClinicalFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: CLIN:vital_status - regex = re_compile("^CLIN:" - # column name - "([a-zA-Z0-9_\-]+)$") - - def __init__(self, table_field, value_type): - self.table_field = table_field - self.value_type = value_type - - @classmethod - def get_table_field_and_value_type(cls, column_id): - table_field = None - value_type = None - for clinical_field in clinical_schema: - name, schema_field_type = clinical_field['name'], clinical_field['type'] - if name == column_id: - table_field= name - value_type = BigQuerySchemaToValueTypeConverter.get_value_type(schema_field_type) - - return table_field, value_type - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - column_name = feature_fields[0] - - table_field, value_type = cls.get_table_field_and_value_type(column_name) - if table_field is None: - raise FeatureNotFoundException(feature_id) - - return cls(table_field, value_type) - - -class ClinicalFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Clinical_data', - 'info': 'Clinical', - 'id': 'tcga_clinical' - } - ] - - def __init__(self, feature_id, **kwargs): - self.table_name = '' - self.feature_def = None - self.parse_internal_feature_id(feature_id) - super(ClinicalFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return self.feature_def.value_type - - def get_feature_type(self): - return DataTypes.CLIN - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT clin.ParticipantBarcode, biospec.sample_id, clin.{column_name} " - "FROM ( " - " SELECT ParticipantBarcode, {column_name} " - " FROM [{project_name}:{dataset_name}.{table_name}] " - " ) AS clin " - " JOIN ( " - " SELECT ParticipantBarcode, SampleBarcode as sample_id " - " FROM [{project_name}:{dataset_name}.{bsp_table_name}] " - " ) AS biospec " - " ON clin.ParticipantBarcode = biospec.ParticipantBarcode " - "WHERE biospec.sample_id IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - query_template += " GROUP BY clin.ParticipantBarcode, biospec.sample_id, clin.{column_name}" - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - column_name=feature_def.table_field, bsp_table_name=BSP_TABLE_NAME, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_CLIN: " + query) - return query - - @DurationLogged('CLIN', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Always None - - 'value': Value of the selected column from the clinical data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'case_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': None, - 'value': row['f'][2]['v'] - }) - - return result - - def get_table_name(self): - return self.TABLES[0]['name'] - - def parse_internal_feature_id(self, feature_id): - self.feature_def = ClinicalFeatureDef.from_feature_id(feature_id) - self.table_name = self.get_table_name() - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - ClinicalFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # ClinicalFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/cohort_bigquery.py b/bq_data_access/cohort_bigquery.py deleted file mode 100755 index f5ba29fe..00000000 --- a/bq_data_access/cohort_bigquery.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from copy import deepcopy -import sys - -from api.api_helpers import authorize_credentials_with_Google - -COHORT_DATASETS = { - 'prod': 'cloud_deployment_cohorts', - 'staging': 'cloud_deployment_cohorts', - 'dev': 'dev_deployment_cohorts' -} - -COHORT_TABLES = { - 'prod': 'prod_cohorts', - 'staging': 'staging_cohorts' -} - -class BigQueryCohortSupport(object): - cohort_schema = [ - { - "name": "cohort_id", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "case_barcode", - "type": "STRING" - }, - { - "name": "sample_barcode", - "type": "STRING" - }, - { - "name": "aliquot_barcode", - "type": "STRING" - }, - { - "name": "project_id", - "type": "INTEGER" - } - ] - - patient_type = 'patient' - sample_type = 'sample' - aliquot_type = 'aliquot' - - @classmethod - def get_schema(cls): - return deepcopy(cls.cohort_schema) - - def __init__(self, project_id, dataset_id, table_id): - self.project_id = project_id - self.dataset_id = dataset_id - self.table_id = table_id - - def _build_request_body_from_rows(self, rows): - insertable_rows = [] - for row in rows: - insertable_rows.append({ - 'json': row - }) - - return { - "rows": insertable_rows - } - - def _streaming_insert(self, rows): - bigquery_service = authorize_credentials_with_Google() - table_data = bigquery_service.tabledata() - - body = self._build_request_body_from_rows(rows) - - print >> sys.stdout, self.project_id+":"+self.dataset_id+":"+self.table_id - - response = table_data.insertAll(projectId=self.project_id, - datasetId=self.dataset_id, - tableId=self.table_id, - body=body).execute() - - return response - - def _build_cohort_row(self, cohort_id, - case_barcode=None, sample_barcode=None, aliquot_barcode=None, project_id=None): - return { - 'cohort_id': cohort_id, - 'case_barcode': case_barcode, - 'sample_barcode': sample_barcode, - 'aliquot_barcode': aliquot_barcode, - 'project_id': project_id - } - - # Create a cohort based on a dictionary of sample, patient/case/participant, and project IDs - def add_cohort_to_bq(self, cohort_id, samples): - rows = [] - for sample in samples: - rows.append(self._build_cohort_row(cohort_id, case_barcode=sample['case_barcode'], sample_barcode=sample['sample_barcode'], project_id=sample['project'].id)) - - response = self._streaming_insert(rows) - - print >> sys.stdout, response.__str__() - - return response - - # Create a cohort based only on sample and optionally project IDs (patient/participant/case ID is NOT added) - def add_cohort_with_sample_barcodes(self, cohort_id, samples): - rows = [] - for sample in samples: - # TODO This is REALLY specific to TCGA. This needs to be changed - # patient_barcode = sample_barcode[:12] - barcode = sample - project_id = None - if isinstance(sample, tuple): - barcode = sample[0] - if len(sample) > 1: - project_id = sample[1] - elif isinstance(sample, dict): - barcode = sample['sample_id'] - if 'project_id' in sample: - project_id = sample['project_id'] - - rows.append(self._build_cohort_row(cohort_id, sample_barcode=barcode, project_id=project_id)) - - response = self._streaming_insert(rows) - return response diff --git a/bq_data_access/cohort_cloudsql.py b/bq_data_access/cohort_cloudsql.py deleted file mode 100755 index ef097704..00000000 --- a/bq_data_access/cohort_cloudsql.py +++ /dev/null @@ -1,152 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import sys -import traceback - -from api.api_helpers import sql_connection - -from MySQLdb.cursors import DictCursor - -DJANGO_COHORT_TABLE = 'cohorts_samples' -DJANGO_COHORT_INFO_TABLE = 'cohorts_cohort' -DJANGO_COHORT_SAMPLES_TABLE = 'cohorts_samples' - -logger = logging - - -class CohortException(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return 'Cohort error: ' + self.message - -class CloudSQLCohortAccess(object): - @classmethod - def parse_barcodes(cls, barcode_string): - codes = barcode_string.replace('[', '').replace(']', '').replace('\'', '').replace(' ', '').split(',') - return codes - - @classmethod - def get_cohort_barcodes(cls, cohort_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join(['%s' for x in xrange(len(cohort_id_array))]) - query = 'SELECT sample_barcode AS barcode FROM {cohort_table} WHERE cohort_id IN ({cohort_id_stmt})'.format( - cohort_table=DJANGO_COHORT_TABLE, - cohort_id_stmt=cohort_id_stmt) - values = cohort_id_array - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - result = cursor.fetchall() - barcodes = [] - for row in result: - barcodes.append(row['barcode']) - cursor.close() - db.close() - - # Return only unique barcodes - return list(set(barcodes)) - - except Exception as e: - raise CohortException('get_cohort_barcodes CloudSQL error, cohort IDs {cohort_ids}: {message}'.format( - cohort_ids=cohort_id_array, - message=str(e.message))) - raise CohortException('bad cohort: ' + str(cohort_id_array)) - - @classmethod - def get_cohorts_for_datapoints(cls, cohort_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join(['%s' for x in xrange(len(cohort_id_array))]) - - query = 'SELECT sample_barcode, cohort_id FROM {cohort_samples_table} WHERE cohort_id IN ({cohort_id_stmt})'.format( - cohort_samples_table=DJANGO_COHORT_SAMPLES_TABLE, - cohort_id_stmt=cohort_id_stmt) - - values = cohort_id_array - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - result = cursor.fetchall() - cohort_per_samples = {} - - for row in result: - cohort_id, sample_barcode = row['cohort_id'], row['sample_barcode'] - if sample_barcode not in cohort_per_samples: - cohort_per_samples[sample_barcode] = [] - cohort_per_samples[sample_barcode].append(cohort_id) - - cursor.close() - db.close() - - return cohort_per_samples - - except Exception as e: - logger.exception(e) - raise CohortException('get_cohorts_for_datapoints CloudSQL error, cohort IDs {cohort_ids}: {message}'.format( - cohort_ids=cohort_id_array, - message=str(e.message))) - - @classmethod - def get_cohort_info(cls, cohort_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join(['%s' for x in xrange(len(cohort_id_array))]) - - query_template = ("SELECT ti.id AS cohort_id, ti.name, COUNT(ts.sample_barcode) AS size " - "FROM {cohort_info_table} ti " - " LEFT JOIN {cohort_samples_table} ts ON ts.cohort_id = ti.id " - "WHERE ti.id IN ({cohort_id_stmt}) " - "GROUP BY ti.id, ti.name") - - query = query_template.format( - cohort_info_table=DJANGO_COHORT_INFO_TABLE, - cohort_samples_table=DJANGO_COHORT_SAMPLES_TABLE, - cohort_id_stmt=cohort_id_stmt) - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(cohort_id_array)) - - result = [] - - for row in cursor.fetchall(): - result.append({ - 'id': row['cohort_id'], - 'name': row['name'], - 'size': row['size'] - }) - - cursor.close() - db.close() - - return result - - except Exception as e: - print >> sys.stdout, "[ERROR] In get_cohort_info: " - print >> sys.stdout, e - print >> sys.stdout, traceback.format_exc() - raise CohortException('get_cohort_info CloudSQL error, cohort IDs {cohort_ids}: {message}'.format( - cohort_ids=cohort_id_array, - message=str(e.message))) diff --git a/bq_data_access/cohort_utils.py b/bq_data_access/cohort_utils.py deleted file mode 100755 index 051d2640..00000000 --- a/bq_data_access/cohort_utils.py +++ /dev/null @@ -1,61 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from bq_data_access.feature_value_types import DataTypes, IdentifierTypes - -class BarcodeAdapter(object): - identifier_length = { - IdentifierTypes.PATIENT: 12, - IdentifierTypes.SAMPLE: 16, - IdentifierTypes.ALIQUOT: 28 - } - - supported_identifers = { - DataTypes.CLIN: IdentifierTypes.PATIENT, - DataTypes.GEXP: IdentifierTypes.PATIENT, - DataTypes.METH: IdentifierTypes.PATIENT, - DataTypes.CNVR: IdentifierTypes.PATIENT, - DataTypes.RPPA: IdentifierTypes.PATIENT, - DataTypes.MIRN: IdentifierTypes.PATIENT, - DataTypes.GNAB: IdentifierTypes.PATIENT - } - - @classmethod - def get_identifier_type_for_data_type(cls, data_type): - return cls.supported_identifers[data_type] - - @classmethod - def convert(cls, data_type, barcode_list): - identifier_type = cls.get_identifier_type_for_data_type(data_type) - truncate_length = cls.identifier_length[identifier_type] - result = [] - for barcode in barcode_list: - truncated_barcode = barcode[:truncate_length] - result.append(truncated_barcode) - - return result - -class CohortQueryBuilder(object): - @classmethod - def build(cls, field_name, barcodes_list, delimiter=',', barcode_quote='\''): - barcodes_quoted = [] - for barcode in barcodes_list: - barcodes_quoted.append('{quote}{barcode}{quote}'.format(quote=barcode_quote, barcode=barcode)) - result = field_name + ' IN ({identifiers})' - result = result.format(identifiers=delimiter.join(barcodes_quoted)) - return result diff --git a/bq_data_access/copynumber_data.py b/bq_data_access/copynumber_data.py deleted file mode 100755 index ac885e8a..00000000 --- a/bq_data_access/copynumber_data.py +++ /dev/null @@ -1,164 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -CNVR_FEATURE_TYPE = 'CNVR' -IDENTIFIER_COLUMN_NAME = 'sample_id' - - -def get_feature_type(): - return CNVR_FEATURE_TYPE - - -class CNVRFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: CNVR:max_segment_mean:X:133276258:133276370 - regex = re_compile("^CNVR:" - # value - "(avg_segment_mean|std_dev_segment_mean|min_segment_mean|max_segment_mean|num_segments):" - # validate outside - chromosome 1-23, X, Y, M - "(\d|\d\d|X|Y|M):" - # coordinates start:end - "(\d+):(\d+)$") - - def __init__(self, value_field, chromosome, start, end): - self.value_field = value_field - self.chromosome = chromosome - self.start = start - self.end = end - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - logging.debug(feature_fields) - value_field, chromosome, start, end = feature_fields[0] - - valid_chr_set = frozenset([str(x) for x in xrange(1, 24)] + ['X', 'Y', 'M']) - if chromosome not in valid_chr_set: - raise FeatureNotFoundException(feature_id) - - return cls(value_field, chromosome, start, end) - - -class CNVRFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Copy_Number_segments', - 'info': 'CNV', - 'id': 'cnv' - } - ] - - def __init__(self, feature_id, **kwargs): - self.table_name = '' - self.feature_def = None - self.parse_internal_feature_id(feature_id) - super(CNVRFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.CNVR - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, study_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - study_id_stmt = '' - if study_id_array is not None: - study_id_stmt = ', '.join([str(study_id) for study_id in study_id_array]) - - value_field_bqsql = { - 'avg_segment_mean': 'AVG(Segment_Mean)', - 'std_dev_segment_mean': 'STDDEV(Segment_Mean)', - 'min_segment_mean': 'MIN(Segment_Mean)', - 'max_segment_mean': 'MAX(Segment_Mean)', - 'num_segments': 'COUNT(*)' - } - - query_template = "SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, {value_field} AS value " \ - "FROM [{project_name}:{dataset_name}.{table_name}] " \ - "WHERE ( Chromosome='{chr}' AND ( " \ - " ( Start<{start} AND End>{start} ) OR " \ - " ( Start>{start}-1 AND Start<{end}+1 ) ) ) " \ - "AND SampleBarcode IN ( " \ - " SELECT sample_barcode " \ - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " \ - " WHERE cohort_id IN ({cohort_id_list})" \ - " AND (study_id IS NULL" - - query_template += (" OR study_id IN ({study_id_list})))" if study_id_array is not None else "))") - query_template += " GROUP BY ParticipantBarcode, SampleBarcode, AliquotBarcode" - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - value_field=value_field_bqsql[feature_def.value_field], - chr=feature_def.chromosome, - start=feature_def.start, end=feature_def.end, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, study_id_list=study_id_stmt) - - logging.debug("BQ_QUERY_CNVR: " + query) - return query - - @DurationLogged('CNVR', 'UNPACK') - def unpack_query_response(self, query_result_array): - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': row['f'][3]['v'] - }) - - return result - - def get_table_name(self): - return self.TABLES[0]['name'] - - def parse_internal_feature_id(self, feature_id): - self.feature_def = CNVRFeatureDef.from_feature_id(feature_id) - self.table_name = self.get_table_name() - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - CNVRFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # CNVRFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/data_access.py b/bq_data_access/data_access.py deleted file mode 100755 index b0bfa265..00000000 --- a/bq_data_access/data_access.py +++ /dev/null @@ -1,436 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile -from time import sleep - -from django.conf import settings - -from api.api_helpers import authorize_credentials_with_Google -from api.api_helpers import sql_connection, MySQLdb - -from bq_data_access.feature_value_types import ValueType - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.gexp_data import GEXPFeatureProvider, GEXP_FEATURE_TYPE -from bq_data_access.methylation_data import METHFeatureProvider, METH_FEATURE_TYPE -from bq_data_access.copynumber_data import CNVRFeatureProvider, CNVR_FEATURE_TYPE -from bq_data_access.protein_data import RPPAFeatureProvider, RPPA_FEATURE_TYPE -from bq_data_access.mirna_data import MIRNFeatureProvider, MIRN_FEATURE_TYPE -from bq_data_access.clinical_data import ClinicalFeatureProvider, CLINICAL_FEATURE_TYPE -from bq_data_access.gnab_data import GNABFeatureProvider, GNAB_FEATURE_TYPE -from bq_data_access.user_data import UserFeatureProvider, USER_FEATURE_TYPE - - -class FeatureProviderFactory(object): - @classmethod - def get_feature_type_string(cls, feature_id): - regex = re_compile("^(CLIN|GEXP|METH|CNVR|RPPA|MIRN|GNAB|USER):") - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - return None - - feature_type = feature_fields[0] - return feature_type - - @classmethod - def get_provider_class_from_feature_id(cls, feature_id): - """ - Args: - feature_id: Feature identifier - - Returns: - Feature data provider class for the datatype defined in the - feature identifier. - - Raises: - FeatureNotFoundException: If the datatype part of the feature - identifier is unknown. - - """ - feature_type = cls.get_feature_type_string(feature_id) - if feature_type is None: - logging.debug("FeatureProviderFactory.from_feature_id: invalid feature ID: " + str(feature_id)) - raise FeatureNotFoundException(feature_id) - - if feature_type == CLINICAL_FEATURE_TYPE: - return ClinicalFeatureProvider - elif feature_type == GEXP_FEATURE_TYPE: - return GEXPFeatureProvider - elif feature_type == METH_FEATURE_TYPE: - return METHFeatureProvider - elif feature_type == CNVR_FEATURE_TYPE: - return CNVRFeatureProvider - elif feature_type == RPPA_FEATURE_TYPE: - return RPPAFeatureProvider - elif feature_type == MIRN_FEATURE_TYPE: - return MIRNFeatureProvider - elif feature_type == GNAB_FEATURE_TYPE: - return GNABFeatureProvider - elif feature_type == USER_FEATURE_TYPE: - return UserFeatureProvider - - @classmethod - def from_feature_id(cls, feature_id, **kwargs): - provider_class = cls.get_provider_class_from_feature_id(feature_id) - return provider_class(feature_id, **kwargs) - - @classmethod - def from_parameters(cls, parameters_obj, **kwargs): - if isinstance(parameters_obj, FeatureIdQueryDescription): - return cls.from_feature_id(parameters_obj.feature_id, **kwargs) - elif isinstance(parameters_obj, ProviderClassQueryDescription): - class_type = parameters_obj.feature_data_provider_class - feature_id = parameters_obj.feature_id - return class_type(feature_id, **kwargs) - - -class FeatureIdQueryDescription(object): - def __init__(self, feature_id, cohort_id_array, project_id_array): - self.feature_id = feature_id - self.cohort_id_array = cohort_id_array - self.project_id_array = project_id_array - - -class ProviderClassQueryDescription(object): - def __init__(self, feature_data_provider_class, feature_id, cohort_id_array, project_id_array): - self.feature_data_provider_class = feature_data_provider_class - self.feature_id = feature_id - self.cohort_id_array = cohort_id_array - self.project_id_array = project_id_array - - -def is_valid_feature_identifier(feature_id): - """ - Answers if given internal feature identifier is valid. - - Args: - feature_id: Internal feature identifier - - Returns: - True if feature id is valid, otherwise False. - """ - is_valid = False - try: - provider_class = FeatureProviderFactory.get_provider_class_from_feature_id(feature_id) - is_valid = provider_class.is_valid_feature_id(feature_id) - except FeatureNotFoundException: - # FeatureProviderFactory.get_provider_class_from_feature_id raises FeatureNotFoundException - # if the feature identifier does not look valid. Nothing needs to be done here, - # since is_valid is already False. - pass - finally: - return is_valid - - -def get_feature_vector(feature_id, cohort_id_array): - """ - Fetches the data from BigQuery tables for a given feature identifier and - one or more stored cohorts. Returns the intersection of the samples defined - by the feature identifier and the stored cohort. - - Each returned data point is represented as a dict containing patient, sample and - aliquot barcodes, and the value as defined by the feature identifier. - - Args: - feature_id: Feature identifier - cohort_id_array: Array of cohort identifiers (integers) - - Returns: - Data as an array of dicts. - """ - provider = FeatureProviderFactory.from_feature_id(feature_id) - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - - result = provider.get_data(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - - items = [] - for data_point in result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = provider.process_data_point(data_point) - # TODO refactor missing value logic - if value is None: - value = 'NA' - data_item['value'] = value - items.append(data_item) - - return provider.get_value_type(), items - - -def submit_tcga_job(param_obj, bigquery_service, cohort_settings): - provider = FeatureProviderFactory.from_parameters(param_obj, bigquery_service=bigquery_service) - feature_id = param_obj.feature_id - cohort_id_array = param_obj.cohort_id_array - project_id_array = param_obj.project_id_array - job_reference = provider.get_data_job_reference(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id, project_id_array) - - logging.info("Submitted TCGA {job_id}: {fid} - {cohorts}".format(job_id=job_reference['jobId'], fid=feature_id, - cohorts=str(cohort_id_array))) - job_item = { - 'feature_id': feature_id, - 'provider': provider, - 'ready': False, - 'job_reference': job_reference - } - - return job_item - - -def submit_jobs_with_user_data(params_array): - bigquery_service = authorize_credentials_with_Google() - provider_array = [] - - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - - # Submit jobs - for parameter_object in params_array: - feature_id = parameter_object.feature_id - cohort_id_array = parameter_object.cohort_id_array - - user_data = user_feature_handler(feature_id, cohort_id_array) - - if user_data['include_tcga']: - job_item = submit_tcga_job(parameter_object, bigquery_service, cohort_settings) - provider_array.append(job_item) - - if len(user_data['user_studies']) > 0: - converted_feature_id = user_data['converted_feature_id'] - user_feature_id = user_data['user_feature_id'] - logging.debug("user_feature_id: {0}".format(user_feature_id)) - provider = UserFeatureProvider(converted_feature_id, user_feature_id=user_feature_id) - - # The UserFeatureProvider instance might not generate a BigQuery query and job at all given the combination - # of cohort(s) and feature identifiers. The provider is not added to the array, and therefore to the - # polling loop below, if it would not submit a BigQuery job. - if provider.is_queryable(cohort_id_array): - job_reference = provider.get_data_job_reference(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - - logging.info("Submitted USER {job_id}: {fid} - {cohorts}".format(job_id=job_reference['jobId'], fid=feature_id, - cohorts=str(cohort_id_array))) - provider_array.append({ - 'feature_id': feature_id, - 'provider': provider, - 'ready': False, - 'job_reference': job_reference - }) - else: - logging.debug("No UserFeatureDefs for '{0}'".format(converted_feature_id)) - - return provider_array - - -def get_submitted_job_results(provider_array, project_id, poll_retry_limit, skip_formatting_for_plot): - result = {} - all_done = False - total_retries = 0 - poll_count = 0 - - # Poll for completion - while all_done is False and total_retries < poll_retry_limit: - poll_count += 1 - total_retries += 1 - - for item in provider_array: - provider = item['provider'] - feature_id = item['feature_id'] - is_finished = provider.is_bigquery_job_finished(project_id) - logging.info("Status {job_id}: {status}".format(job_id=item['job_reference']['jobId'], - status=str(is_finished))) - - if item['ready'] is False and is_finished: - item['ready'] = True - query_result = provider.download_and_unpack_query_result() - - if not skip_formatting_for_plot: - data = format_query_result_for_plot(provider, query_result) - - value_type = provider.get_value_type() - if feature_id not in result: - result[feature_id] = { - 'type': value_type, - 'data': data - } - else: - # TODO fix possible bug: - # The ValueType of the data from the user feature provider may not match that of the TCGA - # provider above. - result[feature_id]['data'].extend(data) - else: - result[feature_id] = { - 'data': query_result, - 'type': None - } - - sleep(1) - - all_done = all([j['ready'] for j in provider_array]) - logging.debug("Done: {done} retry: {retry}".format(done=str(all_done), retry=total_retries)) - - return result - - -def format_query_result_for_plot(provider_instance, query_result): - data = [] - - for data_point in query_result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = str(provider_instance.process_data_point(data_point)) - - if value is None: - value = 'NA' - data_item['value'] = value - data.append(data_item) - - return data - - -def get_feature_vectors_with_user_data(params_array, poll_retry_limit=20, skip_formatting_for_plot=False): - provider_array = submit_jobs_with_user_data(params_array) - - project_id = settings.BQ_PROJECT_ID - result = get_submitted_job_results(provider_array, project_id, poll_retry_limit, skip_formatting_for_plot) - - return result - - -def get_feature_vectors_tcga_only(params_array, poll_retry_limit=20, skip_formatting_for_plot=False): - bigquery_service = authorize_credentials_with_Google() - provider_array = [] - - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - - # Submit jobs - for parameter_object in params_array: - job_item = submit_tcga_job(parameter_object, bigquery_service, cohort_settings) - provider_array.append(job_item) - - project_id = settings.BQ_PROJECT_ID - result = get_submitted_job_results(provider_array, project_id, poll_retry_limit, skip_formatting_for_plot) - - return result - - -def user_feature_handler(feature_id, cohort_id_array): - include_tcga = False - user_studies = () - for cohort_id in cohort_id_array: - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - cursor.execute("SELECT project_id FROM cohorts_samples WHERE cohort_id = %s GROUP BY project_id", (cohort_id,)) - for row in cursor.fetchall(): - if row['project_id'] is None: - include_tcga = True - else: - user_studies += (row['project_id'],) - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - user_feature_id = None - if feature_id.startswith('USER:'): - # Try and convert it with a shared ID to a TCGA queryable id - user_feature_id = feature_id - feature_id = UserFeatureProvider.convert_user_feature_id(feature_id) - if feature_id is None: - # Querying user specific data, don't include TCGA - include_tcga = False - - return { - 'converted_feature_id': feature_id, - 'include_tcga': include_tcga, - 'user_studies': user_studies, - 'user_feature_id': user_feature_id - } - - -def get_feature_vector(feature_id, cohort_id_array): - include_tcga = False - user_studies = () - for cohort_id in cohort_id_array: - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - cursor.execute("SELECT project_id FROM cohorts_samples WHERE cohort_id = %s GROUP BY project_id", (cohort_id,)) - for row in cursor.fetchall(): - if row['project_id'] is None: - include_tcga = True - else: - user_studies += (row['project_id'],) - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - # ex: feature_id 'CLIN:Disease_Code' - user_feature_id = None - if feature_id.startswith('USER:'): - # Try and convert it with a shared ID to a TCGA queryable id - user_feature_id = feature_id - feature_id = UserFeatureProvider.convert_user_feature_id(feature_id) - if feature_id is None: - # Querying user specific data, don't include TCGA - include_tcga = False - - items = [] - type = None - result = [] - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - if include_tcga: - provider = FeatureProviderFactory.from_feature_id(feature_id) - result = provider.get_data(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - - # ex: result[0] - # {'aliquot_id': None, 'case_id': u'TCGA-BH-A0B1', 'sample_id': u'TCGA-BH-A0B1-10A', 'value': u'BRCA'} - for data_point in result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = provider.process_data_point(data_point) - # TODO refactor missing value logic - if value is None: - value = 'NA' - data_item['value'] = value - items.append(data_item) - - type = provider.get_value_type() - - if len(user_studies) > 0: - # Query User Data - user_provider = UserFeatureProvider(feature_id, user_feature_id=user_feature_id) - user_result = user_provider.get_data(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - result.extend(user_result) - - for data_point in user_result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = provider.process_data_point(data_point) - # TODO refactor missing value logic - if value is None: - value = 'NA' - data_item['value'] = value - items.append(data_item) - - if not type: - type = user_provider.get_value_type() - - return type, items diff --git a/bq_data_access/errors.py b/bq_data_access/errors.py deleted file mode 100644 index 6369cac4..00000000 --- a/bq_data_access/errors.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -class FeatureAccessError(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return self.message - -class FeatureNotFoundException(FeatureAccessError): - """Exception raised when the queried internal feature identifier is invalid""" - diff --git a/bq_data_access/feature_data_provider.py b/bq_data_access/feature_data_provider.py deleted file mode 100644 index 26dac2d5..00000000 --- a/bq_data_access/feature_data_provider.py +++ /dev/null @@ -1,143 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import sys -from uuid import uuid4 -from time import sleep - -from django.conf import settings -from api.api_helpers import authorize_credentials_with_Google -from bq_data_access.utils import DurationLogged - - -class FeatureDataProvider(object): - """ - Class for building data access modules for different datatypes. - - TODO: Document interface - """ - def __init__(self, bigquery_service=None): - self.job_reference = None - self.bigquery_service = bigquery_service - - def get_bq_service(self): - if self.bigquery_service is None: - self.bigquery_service = authorize_credentials_with_Google() - - return self.bigquery_service - - @DurationLogged('FEATURE', 'BQ_SUBMIT') - def submit_bigquery_job(self, bigquery, project_id, query_body, batch=False): - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid4()) - }, - 'configuration': { - 'query': { - 'query': query_body, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - - return bigquery.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=5) - - @DurationLogged('FEATURE', 'BQ_POLL') - def poll_async_job(self, bigquery_service, project_id, job_id, poll_interval=5): - job_collection = bigquery_service.jobs() - - poll = True - - while poll: - sleep(poll_interval) - job = job_collection.get(projectId=project_id, - jobId=job_id).execute() - - if job['status']['state'] == 'DONE': - poll = False - - if 'errorResult' in job['status']: - raise Exception(job['status']) - - @DurationLogged('FEATURE', 'BQ_FETCH') - def download_query_result(self, bigquery, job_reference): - result = [] - page_token = None - total_rows = 0 - - while True: - page = bigquery.jobs().getQueryResults( - pageToken=page_token, - **job_reference).execute(num_retries=2) - - if int(page['totalRows']) == 0: - break - - rows = page['rows'] - result.extend(rows) - total_rows += len(rows) - - page_token = page.get('pageToken') - if not page_token: - break - - return result - - def is_bigquery_job_finished(self, project_id): - job_collection = self.get_bq_service().jobs() - bigquery_job_id = self.job_reference['jobId'] - - job = job_collection.get(projectId=project_id, - jobId=bigquery_job_id).execute() - - return job['status']['state'] == 'DONE' - - def download_and_unpack_query_result(self): - bigquery_service = self.get_bq_service() - query_result_array = self.download_query_result(bigquery_service, self.job_reference) - - result = self.unpack_query_response(query_result_array) - return result - - def submit_query_and_get_job_ref(self, project_id, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - bigquery_service = self.get_bq_service() - - query_body = self.build_query(project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array) - query_job = self.submit_bigquery_job(bigquery_service, project_id, query_body) - - # Poll for completion of the query - self.job_reference = query_job['jobReference'] - job_id = query_job['jobReference']['jobId'] - logging.debug("JOBID {id}".format(id=job_id)) - - return self.job_reference - - def get_data_job_reference(self, cohort_id_array, cohort_dataset, cohort_table, project_id_array): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - - result = self.submit_query_and_get_job_ref(project_id, project_name, dataset_name, self.table_name, - self.feature_def, cohort_dataset, cohort_table, cohort_id_array, - project_id_array) - return result - diff --git a/bq_data_access/feature_search/__init__.py b/bq_data_access/feature_search/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/bq_data_access/feature_search/clinical_searcher.py b/bq_data_access/feature_search/clinical_searcher.py deleted file mode 100755 index c8dfd7b0..00000000 --- a/bq_data_access/feature_search/clinical_searcher.py +++ /dev/null @@ -1,92 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from api.schema.tcga_clinical import schema as clinical_schema - -from bq_data_access.clinical_data import CLINICAL_FEATURE_TYPE -from bq_data_access.feature_search.common import InvalidFieldException, EmptyQueryException - -class ClinicalSearcher(object): - feature_search_valid_fields = ['keyword'] - - @classmethod - def get_datatype_identifier(cls): - return CLINICAL_FEATURE_TYPE - - @classmethod - def get_searchable_fields(cls): - return [ - {'name': 'keyword', - 'label': 'Keyword', - 'static': False}, - ] - - @classmethod - def build_feature_label(cls, column_name): - name_parts = [] - for part in column_name.split('_'): - name_parts.append(part.capitalize()) - human_readable_name = ' '.join(name_parts) - return human_readable_name - - def filter_by_name(self, keyword): - result = [] - - for item in clinical_schema: - name = item['name'] - if name.find(keyword.lower()) != -1: - result.append(item) - - return result - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(self.get_datatype_identifier(), keyword, field) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def search(self, parameters): - self.validate_feature_search_input(parameters) - search_result = self.filter_by_name(parameters['keyword']) - - found_features = [] - for feature_item in search_result: - column_name = feature_item['name'] - human_readable_name = self.build_feature_label(column_name) - internal_id = 'CLIN:' + column_name - type = "N" - if feature_item['type'] == "STRING" : - type = "C" - found_features.append({ - 'feature_type': 'CLIN', - 'internal_feature_id': internal_id, - 'label': human_readable_name, - 'type' : type - }) - - return found_features diff --git a/bq_data_access/feature_search/common.py b/bq_data_access/feature_search/common.py deleted file mode 100755 index d9b66f51..00000000 --- a/bq_data_access/feature_search/common.py +++ /dev/null @@ -1,41 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -class FeatureSearchError(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return self.message - -class InvalidDataTypeException(FeatureSearchError): - """Exception raised when requested data type does not exist""" - -class InvalidQueryException(FeatureSearchError): - """Exception raised when a query for a datatype is invalid""" - -class InvalidFieldException(InvalidQueryException): - """Exception raised when the requested search field does not exist for a datatype""" - -class EmptyQueryException(InvalidQueryException): - """Exception raised when a query contain only empty keywords""" - -class BackendException(FeatureSearchError): - """Exception raised when feature search fails because of backend datasource""" - -FOUND_FEATURE_LIMIT = 20 diff --git a/bq_data_access/feature_search/copynumber_search.py b/bq_data_access/feature_search/copynumber_search.py deleted file mode 100644 index 7d6c4966..00000000 --- a/bq_data_access/feature_search/copynumber_search.py +++ /dev/null @@ -1,145 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.copynumber_data import CNVR_FEATURE_TYPE - - -class CNVRSearcher(object): - feature_search_valid_fields = set(['gene_name', 'value_field']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, - 'values': ['avg_segment_mean', 'std_dev_segment_mean', 'min_segment_mean', 'max_segment_mean', 'num_segments']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return CNVR_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_cnvr" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("CNVR", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Copy Number | Gene:EGFR, Value:avg_segment_mean' - label = "Copy Number | Gene:" + row['gene_name'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, value_field, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name=%s'\ - ' AND value_field LIKE %s' \ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = [input['gene_name'], - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = CNVR_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError as mse: - raise BackendException('database error') - diff --git a/bq_data_access/feature_search/gexp_searcher.py b/bq_data_access/feature_search/gexp_searcher.py deleted file mode 100755 index e39fe52a..00000000 --- a/bq_data_access/feature_search/gexp_searcher.py +++ /dev/null @@ -1,155 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -import logging - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.gexp_data import GEXP_FEATURE_TYPE - - -class GEXPSearcher(object): - feature_search_valid_fields = set(['gene_name', 'platform', 'center']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'platform', - 'label': 'Platform', - 'static': True, - 'values': ['Illumina GA', 'Illumina HiSeq']}, - {'name': 'center', - 'label': 'Center', - 'static': True, 'values': ['UNC']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return GEXP_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_gexp" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("GEXP: '%s', '%s'", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - logging.debug("CLOUDSQL_QUERY_GEXP_FIELDS: {}".format(query)) - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException("MySQLError: {}".format(str(mse))) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, gene, info): - # print info - # Example: 'EGFR mRNA (Illumina HiSeq, UNC RSEM)' - label = gene + " mRNA (" + info['platform'] + ", " + info['generating_center'] + " " + info['value_label'] + ")" - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, platform, generating_center, value_label, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name=%s'\ - ' AND platform LIKE %s' \ - ' AND generating_center LIKE %s'\ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - logging.debug("CLOUDSQL_QUERY_GEXP_SEARCH: {}".format(query)) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = [input['gene_name'], - '%' + input['platform'] + '%', - '%' + input['center'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = GEXP_FEATURE_TYPE - item['label'] = self.build_feature_label(item['gene_name'], item) - - return items - - except MySQLError as mse: - raise BackendException("MySQLError: {}".format(str(mse))) - diff --git a/bq_data_access/feature_search/gnab_searcher.py b/bq_data_access/feature_search/gnab_searcher.py deleted file mode 100644 index f9193e86..00000000 --- a/bq_data_access/feature_search/gnab_searcher.py +++ /dev/null @@ -1,145 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.gnab_data import GNAB_FEATURE_TYPE - - -class GNABSearcher(object): - feature_search_valid_fields = set(['gene_name', 'value_field']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, - 'values': ['variant_classification', 'variant_type', 'sequence_source', 'num_mutations']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return GNAB_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_gnab" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("GNAB", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Mutation | Gene:EGFR, Value:variant_classification' - label = "Mutation | Gene:" + row['gene_name'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, value_field, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name=%s'\ - ' AND value_field LIKE %s' \ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = [input['gene_name'], - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = GNAB_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError as mse: - raise BackendException('database error') - diff --git a/bq_data_access/feature_search/methylation_searcher.py b/bq_data_access/feature_search/methylation_searcher.py deleted file mode 100644 index 62fc8093..00000000 --- a/bq_data_access/feature_search/methylation_searcher.py +++ /dev/null @@ -1,167 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.methylation_data import METH_FEATURE_TYPE - -class METHSearcher(object): - feature_search_valid_fields = set(['gene_name', 'probe_name', 'platform', 'relation_to_gene', 'relation_to_island']) - field_search_valid_fields = set(['gene_name', 'probe_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'probe_name', - 'label': 'CpG Probe', - 'static': False}, - {'name': 'platform', - 'label': 'Platform', - 'static': True, - 'values': ['HumanMethylation27', 'HumanMethylation450']}, - {'name': 'relation_to_gene', - 'label': 'Gene region', - 'static': True, 'values': ['Body', '5\'UTR', '1stExon', 'TSS200', 'TSS1500', '3\'UTR']}, - {'name': 'relation_to_island', - 'label': 'CpG Island region', - 'static': True, 'values': ['Island', 'N_Shelf', 'N_Shore', 'S_Shore', 'S_Shelf']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return METH_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_meth" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("METH", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Methylation | Probe:cg07311521, Gene:EGFR, Gene Region:TSS1500, Relation to CpG Island:Island, Platform:HumanMethylation450, Value:beta_value' - # If value is not present, display '-' - if row['gene_name'] is '': - row['gene_name'] = "-" - row['relation_to_gene'] = "-" - if row['relation_to_island'] is '': - row['relation_to_island'] = "-" - - label = "Methylation | Probe:" + row['probe_name'] + ", Gene:" + row['gene_name'] + \ - ", Gene Region:" + row['relation_to_gene'] + ", CpG Island Region:" + row['relation_to_island'] + \ - ", Platform:" + row['platform'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, probe_name, platform, relation_to_gene, relation_to_island, ' \ - 'value_field, internal_feature_id ' \ - 'FROM {table_name} ' \ - 'WHERE gene_name=%s ' \ - 'AND probe_name LIKE %s ' \ - 'AND platform LIKE %s ' \ - 'AND relation_to_gene LIKE %s ' \ - 'AND relation_to_island LIKE %s ' \ - 'LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - query_args = [input['gene_name'], - '%' + input['probe_name'] + '%', - '%' + input['platform'] + '%', - '%' + input['relation_to_gene'] + '%', - '%' + input['relation_to_island'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = METH_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error') \ No newline at end of file diff --git a/bq_data_access/feature_search/microrna_searcher.py b/bq_data_access/feature_search/microrna_searcher.py deleted file mode 100644 index 2834de04..00000000 --- a/bq_data_access/feature_search/microrna_searcher.py +++ /dev/null @@ -1,149 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict -import logging -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.mirna_data import MIRN_FEATURE_TYPE - - -class MIRNSearcher(object): - feature_search_valid_fields = set(['mirna_name', 'platform', 'value_field']) - field_search_valid_fields = set(['mirna_name']) - - searchable_fields = [ - {'name': 'mirna_name', - 'label': 'miRNA Name', - 'static': False}, - {'name': 'platform', - 'label': 'Platform', - 'static': True, - 'values': ['IlluminaGA', 'IlluminaHiSeq']}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, 'values': ['RPM', 'normalized_count']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return MIRN_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_mirna" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("MIRN", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'MicroRNA | miRNA Name:hsa-mir-126, Platform:IlluminaGA, Value:RPM' - label = "MicroRNA | miRNA Name:" + row['mirna_name'] + ", Platform:" + row['platform'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT mirna_name, platform, value_field, internal_feature_id ' \ - 'FROM {table_name} ' \ - 'WHERE mirna_name LIKE %s ' \ - 'AND platform LIKE %s ' \ - 'AND value_field LIKE %s ' \ - 'LIMIT %s'.format(table_name=self.get_table_name() - ) - logging.debug(query) - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - query_args = ['%' + input['mirna_name'] + '%', - '%' + input['platform'] + '%', - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = MIRN_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error') \ No newline at end of file diff --git a/bq_data_access/feature_search/mirna.py b/bq_data_access/feature_search/mirna.py deleted file mode 100644 index 51d70e05..00000000 --- a/bq_data_access/feature_search/mirna.py +++ /dev/null @@ -1,74 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException - -from bq_data_access.mirna_data import build_feature_label, MIRN_FEATURE_TYPE - - -class MIRNSearcher(object): - search_fields = set(['mirna_name', 'platform', 'value_field']) - - @classmethod - def get_table_name(cls): - return "feature_defs_mirna" - - def validate_search_field(self, keyword, field): - if field not in self.search_fields: - raise InvalidFieldException("MIRN", keyword, field) - - def search(self, keyword, field): - self.validate_search_field(keyword, field) - - query = 'SELECT mirna_name, platform, value_field, internal_feature_id ' \ - 'FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = MIRN_FEATURE_TYPE - item['label'] = build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error', keyword, field) - - - diff --git a/bq_data_access/feature_search/mutation.py b/bq_data_access/feature_search/mutation.py deleted file mode 100644 index f78e08f0..00000000 --- a/bq_data_access/feature_search/mutation.py +++ /dev/null @@ -1,139 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.maf_data import build_feature_label, GNAB_FEATURE_TYPE - -class GNABSearcher(object): - feature_search_valid_fields = set(['gene_name', 'value_field']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, - 'values': ['variant_classification', 'variant_type', 'sequence_source', 'num_mutations']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return GNAB_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_gnab" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("GNAB", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, value_field, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name LIKE %s'\ - ' AND value_field LIKE %s' \ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = ['%' + input['gene_name'] + '%', - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = GNAB_FEATURE_TYPE - item['label'] = build_feature_label(item) - - return items - - except MySQLError as mse: - raise BackendException('database error') - diff --git a/bq_data_access/feature_search/protein.py b/bq_data_access/feature_search/protein.py deleted file mode 100644 index d07f8b24..00000000 --- a/bq_data_access/feature_search/protein.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.protein_data import RPPA_FEATURE_TYPE - -class RPPASearcher(object): - feature_search_valid_fields = set(['gene_name', 'protein_name']) - field_search_valid_fields = set(['gene_name', 'probe_name', 'protein_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'protein_name', - 'label': 'Protein', - 'static': False} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return RPPA_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_rppa" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("RPPA", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Protein | Gene:EGFR, Protein:EGFR_pY1068, Value:protein_expression' - label = "Protein | Gene:" + row['gene_name'] + ", Protein:" + row['protein_name'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, protein_name, value_field, internal_feature_id ' \ - 'FROM {table_name} ' \ - 'WHERE gene_name=%s ' \ - 'AND protein_name LIKE %s ' \ - 'LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - query_args = [input['gene_name'], - '%' + input['protein_name'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = RPPA_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error') \ No newline at end of file diff --git a/bq_data_access/feature_search/util.py b/bq_data_access/feature_search/util.py deleted file mode 100755 index bd3c2ed7..00000000 --- a/bq_data_access/feature_search/util.py +++ /dev/null @@ -1,49 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from bq_data_access.feature_search.gexp_searcher import GEXPSearcher -from bq_data_access.feature_search.clinical_searcher import ClinicalSearcher -from bq_data_access.feature_search.methylation_searcher import METHSearcher -from bq_data_access.feature_search.copynumber_search import CNVRSearcher -from bq_data_access.feature_search.protein import RPPASearcher -from bq_data_access.feature_search.microrna_searcher import MIRNSearcher -from bq_data_access.feature_search.gnab_searcher import GNABSearcher - -class SearchableFieldHelper(object): - datatype_handlers = [ - GEXPSearcher, - ClinicalSearcher, - METHSearcher, - CNVRSearcher, - RPPASearcher, - MIRNSearcher, - GNABSearcher - ] - - @classmethod - def get_fields_for_all_datatypes(cls): - result = [] - for searcher in cls.datatype_handlers: - datatype = searcher.get_datatype_identifier() - searchable_fields = searcher.get_searchable_fields() - result.append({ - 'datatype': datatype, - 'fields': searchable_fields - }) - - return result diff --git a/bq_data_access/feature_value_types.py b/bq_data_access/feature_value_types.py deleted file mode 100755 index a0b1bd19..00000000 --- a/bq_data_access/feature_value_types.py +++ /dev/null @@ -1,101 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from protorpc.messages import Enum - - -def enum(**enums): - return type('Enum', (), enums) - -# TODO decouple from protorpc.messages -class ValueType(Enum): - STRING = 1 - INTEGER = 2 - FLOAT = 3 - BOOLEAN = 4 - UNKNOWN = 5 # We can get queries that return no data, which may be of an unknown type - - -def is_log_transformable(attr_type): - return isinstance(attr_type, ValueType) and (attr_type == ValueType.FLOAT or attr_type == ValueType.INTEGER) - -IdentifierTypes = enum(PATIENT=1, SAMPLE=2, ALIQUOT=3) -DataTypes = enum(CLIN=1, GEXP=2, METH=3, CNVR=4, RPPA=5, MIRN=6, GNAB=7, USER=8) - -IDENTIER_FIELDS_FOR_DATA_TYPES = { - #TODO: change clin to match new BQ clin table in tcga_data_open - DataTypes.CLIN: { - IdentifierTypes.PATIENT: 'ParticipantBarcode' - }, - #TODO: change gexp to match new BQ gexp table in tcga_data_open; not yet uploaded yet - DataTypes.GEXP: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.METH: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.CNVR: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.RPPA: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.MIRN: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.GNAB: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'Tumor_SampleBarcode', - IdentifierTypes.ALIQUOT: 'Tumor_AliquotBarcode' - }, - DataTypes.USER: { - IdentifierTypes.SAMPLE: 'sample_barcode' - } -} - -class DataPointIdentifierTools(object): - @classmethod - def get_id_field_name_for_data_type(cls, data_type, identifier_type): - return IDENTIER_FIELDS_FOR_DATA_TYPES[data_type][identifier_type] - -class BigQuerySchemaToValueTypeConverter(object): - field_to_value_types = { - 'STRING': ValueType.STRING, - 'INTEGER': ValueType.INTEGER, - 'FLOAT': ValueType.FLOAT, - 'BOOLEAN': ValueType.BOOLEAN - } - - @classmethod - def get_value_type(cls, schema_field): - return cls.field_to_value_types[schema_field] - -class StringToDataTypeConverter(object): - @classmethod - def get_datatype(cls, x): - pass \ No newline at end of file diff --git a/bq_data_access/gexp_data.py b/bq_data_access/gexp_data.py deleted file mode 100755 index 70129881..00000000 --- a/bq_data_access/gexp_data.py +++ /dev/null @@ -1,192 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.utils import DurationLogged - -import sys - -TABLES = [ - { - 'table_id': 'mRNA_UNC_GA_RSEM', - 'platform': 'Illumina GA', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_ga', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - }, - { - 'table_id': 'mRNA_UNC_HiSeq_RSEM', - 'platform': 'Illumina HiSeq', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_hiseq', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - } -] - -GEXP_FEATURE_TYPE = 'GEXP' - -GENE_LABEL_FIELD = 'HGNC_gene_symbol' - - -def get_feature_type(): - return GEXP_FEATURE_TYPE - - -class GEXPFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: GEXP:TP53:mrna_bcgsc_illumina_hiseq - regex = re_compile("^GEXP:" - # gene - "([a-zA-Z0-9\-]+):" - # table - "(" + "|".join([table['id'] for table in TABLES]) + - ")$") - - def __init__(self, gene, value_field, table_id): - self.gene = gene - self.value_field = value_field - self.table_id = table_id - - @classmethod - def get_table_info(cls, table_id): - table_info = None - for table_entry in TABLES: - if table_id == table_entry['id']: - table_info = table_entry - - return table_info - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - gene_label, table_id = feature_fields[0] - value_field = cls.get_table_info(table_id)['value_field'] - return cls(gene_label, value_field, table_id) - - -class GEXPFeatureProvider(FeatureDataProvider): - TABLES = TABLES - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(GEXPFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.GEXP - - def process_data_point(self, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = "SELECT ParticipantBarcode AS case_id, SampleBarcode AS sample_id, AliquotBarcode AS aliquot_id, {value_field} AS value " \ - "FROM [{project_name}:{dataset_name}.{table_name}] AS gexp " \ - "WHERE {gene_label_field}='{gene_symbol}' " \ - "AND SampleBarcode IN ( " \ - " SELECT sample_barcode " \ - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " \ - " WHERE cohort_id IN ({cohort_id_list}) " \ - " AND (project_id IS NULL" - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene_label_field=GENE_LABEL_FIELD, - gene_symbol=feature_def.gene, value_field=feature_def.value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GEXP: " + query) - return query - - @DurationLogged('GEXP', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the clinical data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'case_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - - # TODO refactor, duplicate code shared with GEXPFeatureDef - def get_table_info(self, table_id): - table_info = None - for table_entry in self.TABLES: - if table_id == table_entry['id']: - table_info = table_entry - - return table_info - - def parse_internal_feature_id(self, feature_id): - self.feature_def = GEXPFeatureDef.from_feature_id(feature_id) - - table_info = self.get_table_info(self.feature_def.table_id) - self.table_name = table_info['table_id'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - GEXPFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # GEXPFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/gnab_data.py b/bq_data_access/gnab_data.py deleted file mode 100755 index a3eb2088..00000000 --- a/bq_data_access/gnab_data.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -GNAB_FEATURE_TYPE = 'GNAB' -IDENTIFIER_COLUMN_NAME = 'sample_id' - - -def get_feature_type(): - return GNAB_FEATURE_TYPE - - -class GNABFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: GNAB:SMYD3:sequence_source - regex = re_compile("^GNAB:" - # gene - "([a-zA-Z0-9_.\-]+):" - # value field - "(variant_classification|variant_type|sequence_source|num_mutations)$") - - def __init__(self, gene, value_field): - self.gene = gene - self.value_field = value_field - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - gene_label, value_field = feature_fields[0] - - return cls(gene_label, value_field) - - -class GNABFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Somatic_Mutation_calls', - 'info': 'MAF', - 'id': 'maf' - } - ] - - VALUE_FIELD_NUM_MUTATIONS = 'num_mutations' - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_info = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(GNABFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - if self.feature_def.value_field == self.VALUE_FIELD_NUM_MUTATIONS: - return ValueType.FLOAT - else: - return ValueType.STRING - - def get_feature_type(self): - return DataTypes.GNAB - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = "SELECT ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " \ - "{value_field} AS value " \ - "FROM [{project_name}:{dataset_name}.{table_name}] " \ - "WHERE Hugo_Symbol='{gene}' " \ - "AND Tumor_SampleBarcode IN ( " \ - " SELECT sample_barcode " \ - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " \ - " WHERE cohort_id IN ({cohort_id_list})" \ - " AND (project_id IS NULL" - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - value_field_bqsql = self.feature_def.value_field - - if self.feature_def.value_field == self.VALUE_FIELD_NUM_MUTATIONS: - value_field_bqsql = 'count(*)' - query_template += ("GROUP BY ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - "Normal_SampleBarcode, Normal_AliquotBarcode") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=feature_def.gene, value_field=value_field_bqsql, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GNAB: " + query) - return query - - @DurationLogged('GNAB', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the MAF data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': row['f'][3]['v'], - }) - - return result - - def get_table_info(self): - return self.TABLES[0] - - def parse_internal_feature_id(self, feature_id): - self.feature_def = GNABFeatureDef.from_feature_id(feature_id) - self.table_info = self.get_table_info() - - if self.table_info is None: - raise FeatureNotFoundException(feature_id) - - self.table_name = self.table_info['name'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - GNABFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # GNABFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/maf_data.py b/bq_data_access/maf_data.py deleted file mode 100755 index f5d280a7..00000000 --- a/bq_data_access/maf_data.py +++ /dev/null @@ -1,233 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from api.api_helpers import authorize_credentials_with_Google - -from django.conf import settings - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes - -GNAB_FEATURE_TYPE = 'GNAB' -IDENTIFIER_COLUMN_NAME = 'sample_id' - -TABLES = [ - { - 'name': 'Somatic_Mutation_calls', - 'info': 'MAF', - 'id': 'maf' - } -] - -VALUE_FIELD_NUM_MUTATIONS = 'num_mutations' -VALUES = frozenset(['variant_classification', 'variant_type', 'sequence_source', VALUE_FIELD_NUM_MUTATIONS]) - -def get_feature_type(): - return GNAB_FEATURE_TYPE - -def get_table_info(): - return TABLES[0] - -def get_table_id(): - return get_table_info()['id'] - -def build_feature_label(row): - # Example: 'Mutation | Gene:EGFR, Value:variant_classification' - label = "Mutation | Gene:" + row['gene_name'] + ", Value:" + row['value_field'] - return label - -def build_internal_feature_id(gene, value_field): - return '{feature_type}:{gene}:{table}:{value}'.format( - feature_type=get_feature_type(), - gene=gene, - table=get_table_id(), - value=value_field) - -def build_internal_feature_id(gene, value_field): - return '{feature_type}:{gene}:{value}'.format( - feature_type=get_feature_type(), - gene=gene, - value=value_field - ) - -def validate_input(query_table_id): - valid_tables = set([x['id'] for x in TABLES]) - if query_table_id not in valid_tables: - raise Exception("Invalid table ID for maf") - -def build_query(project_name, dataset_name, table_name, gene, value_field, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - "Normal_SampleBarcode, Normal_AliquotBarcode, {value_field} AS value " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE Hugo_Symbol='{gene}' " - "AND Tumor_SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - if value_field == 'num_mutations': - value_field = 'count(*)' - query_template += ("GROUP BY ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - "Normal_SampleBarcode, Normal_AliquotBarcode") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=gene, value_field=value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GNAB: " + query) - return query - -def do_query(project_id, project_name, dataset_name, table_name, gene_label, value_field, cohort_dataset, cohort_table, cohort_id_array): - bigquery_service = authorize_credentials_with_Google() - - query = build_query(project_name, dataset_name, table_name, gene_label, value_field, cohort_dataset, cohort_table, cohort_id_array) - query_body = { - 'query': query - } - - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=project_id, body=query_body).execute() - - result = [] - num_result_rows = int(query_response['totalRows']) - if num_result_rows == 0: - return result - - for row in query_response['rows']: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': row['f'][5]['v'], - }) - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][3]['v'], - 'aliquot_id': row['f'][4]['v'], - 'value': row['f'][5]['v'], - }) - - return result - -def build_feature_query(): - query_template = ("SELECT Hugo_Symbol \ - FROM [{project_name}:{dataset_name}.{table_name}] \ - GROUP BY Hugo_Symbol") - - query_str = query_template.format(dataset_name=settings.BIGQUERY_DATASET, - project_name=settings.BIGQUERY_PROJECT_NAME, table_name='MAF') - - return [query_str] - -def build_feature_table_stmt(): - stmt = ("CREATE TABLE IF NOT EXISTS {table_name} ( " - "id int(11) unsigned NOT NULL AUTO_INCREMENT, " - "gene_name tinytext, " - "num_search_hits tinytext, " - "value_field tinytext, " - "internal_feature_id tinytext, " - "PRIMARY KEY (id))").format(table_name='feature_defs_gnab') - - fieldnames = ['gene_name', 'num_search_hits', 'value_field', 'internal_feature_id'] - - return fieldnames, stmt - -def insert_features_stmt(): - stmt = ("INSERT INTO {table_name} " - "(gene_name, num_search_hits, value_field, internal_feature_id) " - "VALUES (%s, %s, %s, %s)").format(table_name='feature_defs_gnab') - - return stmt - -def parse_response(row): - result = [] - - gene = row[0]['v'] - - for value in VALUES: - result.append({ - 'gene_name': gene, - 'num_search_hits': 0, - 'value_field': value, - 'internal_feature_id': build_internal_feature_id(gene, value) - }) - - return len(VALUES), result - -class GNABFeatureProvider(object): - def __init__(self, feature_id): - self.feature_type = '' - self.gene_label = '' - self.table_info = None - self.value_field = '' - self.table_name = '' - self.parse_internal_feature_id(feature_id) - - def get_value_type(self): - if self.value_field == VALUE_FIELD_NUM_MUTATIONS: - return ValueType.FLOAT - else: - return ValueType.STRING - - def get_feature_type(self): - return DataTypes.GNAB - - @classmethod - def process_data_point(cls, data_point): - return str(data_point['value']) - - def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - result = do_query(project_id, project_name, dataset_name, - self.table_name, self.gene_label, self.value_field, - cohort_dataset, cohort_table, cohort_id_array) - return result - - def get_data(self, cohort_id_array, cohort_dataset, cohort_table): - result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table) - return result - - def parse_internal_feature_id(self, feature_id): - # TODO Better input validation - feature_type, gene_label, value_field = feature_id.split(':') - if value_field not in VALUES: - raise FeatureNotFoundException(feature_id) - - self.feature_type = feature_type - self.gene_label = gene_label - self.value_field = value_field - self.table_info = get_table_info() - - if self.table_info is None: - raise FeatureNotFoundException(feature_id) - - self.table_name = self.table_info['name'] diff --git a/bq_data_access/metadata.py b/bq_data_access/metadata.py deleted file mode 100755 index dd982498..00000000 --- a/bq_data_access/metadata.py +++ /dev/null @@ -1,50 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -DISEASE_CODES = [ - 'ACC', - 'BLCA', - 'BRCA', - 'CESC', - 'COAD', - 'DLBC', - 'ESCA', - 'GBM', - 'HNSC', - 'KICH', - 'KIRC', - 'KIRP', - 'LAML', - 'LGG', - 'LIHC', - 'LUAD', - 'LUSC', - 'MESO', - 'OV', - 'PAAD', - 'PCPG', - 'PRAD', - 'READ', - 'SARC', - 'SKCM', - 'STAD', - 'THCA', - 'UCEC', - 'UCS', - 'UVM' -] diff --git a/bq_data_access/methylation_data.py b/bq_data_access/methylation_data.py deleted file mode 100755 index 5a210b52..00000000 --- a/bq_data_access/methylation_data.py +++ /dev/null @@ -1,295 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.utils import DurationLogged - -METH_FEATURE_TYPE = 'METH' -IDENTIFIER_COLUMN_NAME = 'sample_id' - -TABLES = [ - { - 'name': 'DNA_Methylation_chr1', - 'info': 'Methylation chr1', - 'id': 'methylation_chr1' - }, - { - 'name': 'DNA_Methylation_chr2', - 'info': 'Methylation chr2', - 'id': 'methylation_chr2' - }, - { - 'name': 'DNA_Methylation_chr3', - 'info': 'Methylation chr3', - 'id': 'methylation_chr3' - }, - { - 'name': 'DNA_Methylation_chr4', - 'info': 'Methylation chr4', - 'id': 'methylation_chr4' - }, - { - 'name': 'DNA_Methylation_chr5', - 'info': 'Methylation chr5', - 'id': 'methylation_chr5' - }, - { - 'name': 'DNA_Methylation_chr6', - 'info': 'Methylation chr6', - 'id': 'methylation_chr6' - }, - { - 'name': 'DNA_Methylation_chr7', - 'info': 'Methylation chr7', - 'id': 'methylation_chr7' - }, - { - 'name': 'DNA_Methylation_chr8', - 'info': 'Methylation chr8', - 'id': 'methylation_chr8' - }, - { - 'name': 'DNA_Methylation_chr9', - 'info': 'Methylation chr9', - 'id': 'methylation_chr9' - }, - { - 'name': 'DNA_Methylation_chr10', - 'info': 'Methylation chr10', - 'id': 'methylation_chr10' - }, - { - 'name': 'DNA_Methylation_chr11', - 'info': 'Methylation chr11', - 'id': 'methylation_chr11' - }, - { - 'name': 'DNA_Methylation_chr12', - 'info': 'Methylation chr12', - 'id': 'methylation_chr12' - }, - { - 'name': 'DNA_Methylation_chr13', - 'info': 'Methylation chr13', - 'id': 'methylation_chr13' - }, - { - 'name': 'DNA_Methylation_chr14', - 'info': 'Methylation chr14', - 'id': 'methylation_chr14' - }, - { - 'name': 'DNA_Methylation_chr15', - 'info': 'Methylation chr15', - 'id': 'methylation_chr15' - }, - { - 'name': 'DNA_Methylation_chr16', - 'info': 'Methylation chr16', - 'id': 'methylation_chr16' - }, - { - 'name': 'DNA_Methylation_chr17', - 'info': 'Methylation chr17', - 'id': 'methylation_chr17' - }, - { - 'name': 'DNA_Methylation_chr18', - 'info': 'Methylation chr18', - 'id': 'methylation_chr18' - }, - { - 'name': 'DNA_Methylation_chr19', - 'info': 'Methylation chr19', - 'id': 'methylation_chr19' - }, - { - 'name': 'DNA_Methylation_chr20', - 'info': 'Methylation chr20', - 'id': 'methylation_chr20' - }, - { - 'name': 'DNA_Methylation_chr21', - 'info': 'Methylation chr21', - 'id': 'methylation_chr21' - }, - { - 'name': 'DNA_Methylation_chr22', - 'info': 'Methylation chr22', - 'id': 'methylation_chr22' - }, - { - 'name': 'DNA_Methylation_chrX', - 'info': 'Methylation chrX', - 'id': 'methylation_chrX' - }, - { - 'name': 'DNA_Methylation_chrY', - 'info': 'Methylation chrY', - 'id': 'methylation_chrY' - } -] - -VALUES = ['beta_value'] - - -def get_feature_type(): - return METH_FEATURE_TYPE - - -class METHFeatureDef(object): - def __init__(self, probe, platform, chromosome): - self.probe = probe - self.platform = platform - self.chromosome = chromosome - - @classmethod - def from_feature_id(cls, feature_id): - # Example ID: METH:cg08246323:HumanMethylation450:methylation_chr16 - regex = re_compile("^METH:" - # TODO better validation for probe name - "([a-zA-Z0-9_.\-]+):" - # platform - "(HumanMethylation27|HumanMethylation450):" - # validate outside - chromosome 1-23, X, Y, M - "methylation_chr(\d|\d\d|X|Y|M)$") - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - probe, platform, chromosome = feature_fields[0] - - valid_chr_set = frozenset([str(x) for x in xrange(1, 24)] + ['X', 'Y', 'M']) - if chromosome not in valid_chr_set: - raise FeatureNotFoundException(feature_id) - - return cls(probe, platform, chromosome) - - def __str__(self): - return "METH:{probe}:{platform}:methylation_chr{chr}".format( - probe=self.probe, - platform=self.platform, - chr=self.chromosome - ) - - -class METHFeatureProvider(FeatureDataProvider): - TABLES = TABLES - - def __init__(self, feature_id, **kwargs): - self.feature_type = '' - self.cpg_probe = '' - self.feature_def = None - self.table_name = '' - self.platform = '' - self.parse_internal_feature_id(feature_id) - super(METHFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.METH - - @classmethod - def process_data_point(cls, data_point): - return data_point['beta_value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, beta_value " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE ( Probe_Id='{probe_id}' AND Platform='{platform}') " - "AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - probe_id=feature_def.probe, platform=feature_def.platform, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_METH: " + query) - return query - - @DurationLogged('METH', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the clinical data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'beta_value': float(row['f'][3]['v']) - }) - - return result - - def get_table_name_from_feature_def(self, feature_def): - for table_info in self.TABLES: - if table_info['id'].endswith(feature_def.chromosome): - return table_info['name'] - - raise Exception("Table not found for " + str(feature_def)) - - def parse_internal_feature_id(self, feature_id): - self.feature_def = METHFeatureDef.from_feature_id(feature_id) - self.table_name = self.get_table_name_from_feature_def(self.feature_def) - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - METHFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # METHFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/mirna_data.py b/bq_data_access/mirna_data.py deleted file mode 100755 index 65da0b00..00000000 --- a/bq_data_access/mirna_data.py +++ /dev/null @@ -1,228 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -TABLES = [ - { - 'name': 'miRNA_BCGSC_GA_mirna', - 'info': 'miRNA (GA, BCGSC RPM)', - 'platform': 'IlluminaGA', - 'feature_id': 'mirna_illumina_ga_rpm', - 'value_field': 'reads_per_million_miRNA_mapped' - }, - { - 'name': 'miRNA_BCGSC_HiSeq_mirna', - 'info': 'miRNA (HiSeq, BCGSC RPM)', - 'platform': 'IlluminaHiSeq', - 'feature_id': 'mirna_illumina_hiseq_rpm', - 'value_field': 'reads_per_million_miRNA_mapped' - }, - { - 'name': 'miRNA_Expression', - 'platform': 'both', - 'info': 'miRNA', - 'feature_id': 'expression', - 'value_field': 'normalized_count' - } -] - -TABLE_IDX_MIRNA_EXPRESSION = 2 - -VALUE_READS_PER_MILLION = 'RPM' -VALUE_NORMALIZED_COUNT = 'normalized_count' - -MIRN_FEATURE_TYPE = 'MIRN' -COHORT_FIELD_NAME = 'sample_id' - - -def get_feature_type(): - return MIRN_FEATURE_TYPE - - -def get_mirna_expression_table_info(): - return TABLES[TABLE_IDX_MIRNA_EXPRESSION] - - -def get_table_info(platform, value): - table_info = None - if value == VALUE_NORMALIZED_COUNT: - table_info = get_mirna_expression_table_info() - else: - for table_entry in TABLES: - if platform == table_entry['platform']: - table_info = table_entry - return table_info - - -class MIRNFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: MIRN:hsa-mir-1244-1:mirna_illumina_ga_rpm - regex = re_compile("^MIRN:" - # mirna name - "([a-zA-Z0-9._\-]+):" - # table - "(" + "|".join([table['feature_id'] for table in TABLES]) + - ")$") - - def __init__(self, mirna_name, platform, value_field, table_id): - self.mirna_name = mirna_name - self.platform = platform - self.value_field = value_field - self.table_id = table_id - - @classmethod - def get_table_info(cls, table_id): - table_info = None - for table_entry in TABLES: - if table_id == table_entry['feature_id']: - table_info = table_entry - - return table_info - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - mirna_name, table_id = feature_fields[0] - table_info = cls.get_table_info(table_id) - platform = table_info['platform'] - value_field = table_info['value_field'] - - return cls(mirna_name, platform, value_field, table_id) - - -class MIRNFeatureProvider(FeatureDataProvider): - TABLES = TABLES - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_info = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(MIRNFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.MIRN - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, {value_field} AS value, {mirna_name_field} " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE {mirna_name_field}='{mirna_name}' ") - - if table_name == get_mirna_expression_table_info()['name']: - mirna_name_field = 'mirna_id' - query_template += " AND Platform='{platform}' " - else: - mirna_name_field = 'miRNA_ID' - - query_template += \ - ("AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - mirna_name_field=mirna_name_field, mirna_name=feature_def.mirna_name, - platform=feature_def.platform, - value_field=feature_def.value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_MIRN: " + query) - return query - - @DurationLogged('MIRN', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the miRNA data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - - def get_table_info(self, table_id): - table_info = None - for table_entry in self.TABLES: - if table_id == table_entry['feature_id']: - table_info = table_entry - - return table_info - - def parse_internal_feature_id(self, feature_id): - self.feature_def = MIRNFeatureDef.from_feature_id(feature_id) - self.table_info = self.get_table_info(self.feature_def.table_id) - self.table_name = self.table_info['name'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - MIRNFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # MIRNFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/mrna_data.py b/bq_data_access/mrna_data.py deleted file mode 100755 index d609d64f..00000000 --- a/bq_data_access/mrna_data.py +++ /dev/null @@ -1,186 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from api.api_helpers import authorize_credentials_with_Google -from django.conf import settings - -import logging - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes - -TABLES = [ - { - 'table_id': 'mRNA_BCGSC_GA_RPKM', - 'platform': 'Illumina GA', - 'center': 'BCGSC', - 'id': 'mrna_bcgsc_illumina_ga', - 'value_label': 'RPKM', - 'value_field': 'RPKM' - }, - { - 'table_id': 'mRNA_BCGSC_HiSeq_RPKM', - 'platform': 'Illumina HiSeq', - 'center': 'BCGSC', - 'id': 'mrna_bcgsc_illumina_hiseq', - 'value_label': 'RPKM', - 'value_field': 'RPKM' - }, - { - 'table_id': 'mRNA_UNC_GA_RSEM', - 'platform': 'Illumina GA', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_ga', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - }, - { - 'table_id': 'mRNA_UNC_HiSeq_RSEM', - 'platform': 'Illumina HiSeq', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_hiseq', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - } -] - -GEXP_FEATURE_TYPE = 'GEXP' - -def get_feature_type(): - return GEXP_FEATURE_TYPE - -def build_feature_label(gene, info): - # print info - # Example: 'EGFR mRNA (Illumina HiSeq, UNC RSEM)' - label = gene + " mRNA (" + info['platform'] + ", " + info['center'] + " " + info['value_label'] + ")" - return label - -def build_internal_feature_id(gene, table_id): - return '{feature_type}:{gene}:{table}'.format( - feature_type=get_feature_type(), - gene=gene, - table=table_id - ) - -def get_table_info(table_id): - table_info = None - for table_entry in TABLES: - if table_id == table_entry['id']: - table_info = table_entry - - return table_info - -def build_query(project_name, dataset_name, table_name, gene_symbol, value_field, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode AS patient_id, SampleBarcode AS sample_id, AliquotBarcode AS aliquot_id, {value_field} AS value " - "FROM [{project_name}:{dataset_name}.{table_name}] AS gexp " - "WHERE original_gene_symbol='{gene_symbol}' " - "AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene_symbol=gene_symbol, value_field=value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GEXP: " + query) - return query - -def do_query(project_id, project_name, dataset_name, table_name, gene_symbol, value_field, - cohort_dataset, cohort_table, cohort_id_array): - bigquery_service = authorize_credentials_with_Google() - - query = build_query(project_name, dataset_name, table_name, gene_symbol, value_field, - cohort_dataset, cohort_table, cohort_id_array) - query_body = { - 'query': query - } - - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=project_id, body=query_body).execute() - result = [] - num_result_rows = int(query_response['totalRows']) - if num_result_rows == 0: - return result - - for row in query_response['rows']: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - -class MRNAFeatureProvider(object): - def __init__(self, feature_id): - self.feature_type = '' - self.gene_label = '' - self.table_id = '' - self.table_info = None - self.value_field = '' - self.table_name = '' - self.parse_internal_feature_id(feature_id) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.GEXP - - def process_data_point(self, data_point): - return str(data_point['value']) - - def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - result = do_query(project_id, project_name, dataset_name, self.table_name, self.gene_label, self.value_field, - cohort_dataset, cohort_table, cohort_id_array) - return result - - def get_data(self, cohort_id_array, cohort_dataset, cohort_table): - result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table) - return result - - def parse_internal_feature_id(self, feature_id): - # TODO better feature ID input validation - feature_type, gene_label, table_id = feature_id.split(':') - self.feature_type = feature_type - self.gene_label = gene_label - self.table_id = table_id - self.table_info = get_table_info(table_id) - - if self.table_info is None: - raise FeatureNotFoundException(feature_id) - - self.table_name = self.table_info['table_id'] - self.value_field = self.table_info['value_field'] - diff --git a/bq_data_access/protein_data.py b/bq_data_access/protein_data.py deleted file mode 100755 index ec76f805..00000000 --- a/bq_data_access/protein_data.py +++ /dev/null @@ -1,168 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from django.conf import settings - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -RPPA_FEATURE_TYPE = 'RPPA' - - -def get_feature_type(): - return RPPA_FEATURE_TYPE - - -class RPPAFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: RPPA:GYG1:GYG-Glycogenin1 - regex = re_compile("^RPPA:" - # gene - "([a-zA-Z0-9\-]+):" - # protein name - "([a-zA-Z0-9._\-]+)$") - - def __init__(self, gene, protein_name): - self.gene = gene - self.protein_name = protein_name - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - gene_label, protein_name = feature_fields[0] - return cls(gene_label, protein_name) - - -class RPPAFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Protein_RPPA_data', - 'info': 'Protein', - 'id': 'protein' - } - ] - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_info = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(RPPAFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.RPPA - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, protein_expression AS value " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE ( gene_name='{gene}' AND protein_name='{protein}' ) " - "AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=feature_def.gene, protein=feature_def.protein_name, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_RPPA: " + query) - return query - - @DurationLogged('RPPA', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the protein data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - - def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - result = self.do_query(project_id, project_name, dataset_name, self.table_name, self.feature_def, - cohort_dataset, cohort_table, cohort_id_array) - return result - - def get_data(self, cohort_id_array, cohort_dataset, cohort_table): - result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table) - return result - - def parse_internal_feature_id(self, feature_id): - self.feature_def = RPPAFeatureDef.from_feature_id(feature_id) - self.table_name = self.TABLES[0]['name'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - RPPAFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # RPPAFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/seqpeek/__init__.py b/bq_data_access/seqpeek/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/bq_data_access/seqpeek/seqpeek_interpro.py b/bq_data_access/seqpeek/seqpeek_interpro.py deleted file mode 100644 index 18dfb77c..00000000 --- a/bq_data_access/seqpeek/seqpeek_interpro.py +++ /dev/null @@ -1,72 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from json import loads as json_loads -import logging -from api.api_helpers import authorize_credentials_with_Google - -from django.conf import settings - - -class SeqPeekMAFWithCohorts(object): - def __init__(self, maf_vector, cohort_info): - self.maf_vector = maf_vector - self.cohort_info = cohort_info - - -class InterProDataProvider(object): - def __init__(self): - logging.debug(__name__ + ".__init__") - - def build_query(self, project_name, uniprot_id): - query_template = "SELECT * FROM [{project_name}:test.interpro_filtered] WHERE uniprot_id=\'{uniprot_id}\'" - - query = query_template.format(project_name=project_name, uniprot_id=uniprot_id) - logging.debug("INTERPRO SQL: " + query) - return query - - def do_query(self, project_id, project_name, uniprot_id): - bigquery_service = authorize_credentials_with_Google() - - query = self.build_query(project_name, uniprot_id) - query_body = { - 'query': query - } - - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=project_id, body=query_body).execute() - - num_result_rows = int(query_response['totalRows']) - if num_result_rows == 0: - return None - - row = query_response['rows'][0] - interpro_literal = row['f'][1]['v'] - interpro_literal = interpro_literal.replace('\'', '"') - interpro_literal = json_loads(interpro_literal) - - return interpro_literal - - def get_data_from_bigquery(self, uniprot_id): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - result = self.do_query(project_id, project_name, uniprot_id) - return result - - def get_data(self, uniprot_id): - return self.get_data_from_bigquery(uniprot_id) diff --git a/bq_data_access/seqpeek/seqpeek_maf_formatter.py b/bq_data_access/seqpeek/seqpeek_maf_formatter.py deleted file mode 100644 index 1159f6bc..00000000 --- a/bq_data_access/seqpeek/seqpeek_maf_formatter.py +++ /dev/null @@ -1,85 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from collections import defaultdict -from copy import deepcopy -import logging -from re import compile as re_compile - -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess - -COORDINATE_FIELD_NAME = 'uniprot_aapos' -TYPE_FIELD_NAME = 'variant_classification' - -DIGIT_FINDER_RE = re_compile('^\d+$') - - -class SeqPeekMAFWithCohorts(object): - def __init__(self, maf_vector, cohort_info, removed_row_stats): - self.maf_vector = maf_vector - self.cohort_info = cohort_info - self.removed_row_statistics = removed_row_stats - - -class SeqPeekMAFDataFormatter(object): - def annotate_vector_with_cohorts(self, cohort_id_array, result): - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - for row in result: - sample_id = row['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - row['cohort'] = cohort_set - - def remove_rows_with_no_aa_position(self, data): - result = [] - removed_stats = defaultdict(int) - - count = 0 - for row in data: - aapos = row[COORDINATE_FIELD_NAME] - if aapos is not None and len(DIGIT_FINDER_RE.findall(aapos)) == 1: - count += 1 - item = deepcopy(row) - item[COORDINATE_FIELD_NAME] = int(aapos) - result.append(item) - # Include removed row in statistics - else: - removed_stats[row[TYPE_FIELD_NAME]] += 1 - - logging.debug("SeqPeek MAF filtered rows: {0}, total: {1}".format(len(result), len(data))) - return result, removed_stats - - def get_cohort_information(self, cohort_id_array): - # Get the name, size and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - - return cohort_info_array - - def format_maf_vector_for_view(self, maf_vector, cohort_id_array): - filtered_maf_vector, removed_stats = self.remove_rows_with_no_aa_position(maf_vector) - self.annotate_vector_with_cohorts(cohort_id_array, filtered_maf_vector) - cohort_info = self.get_cohort_information(cohort_id_array) - - return SeqPeekMAFWithCohorts(filtered_maf_vector, cohort_info, removed_stats) diff --git a/bq_data_access/seqpeek/seqpeek_view.py b/bq_data_access/seqpeek/seqpeek_view.py deleted file mode 100644 index 675949af..00000000 --- a/bq_data_access/seqpeek/seqpeek_view.py +++ /dev/null @@ -1,268 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from copy import deepcopy -import logging - -from bq_data_access.seqpeek.seqpeek_interpro import InterProDataProvider - -SAMPLE_ID_FIELD_NAME = 'sample_id' -TRACK_ID_FIELD = "tumor" -COORDINATE_FIELD_NAME = 'uniprot_aapos' -PROTEIN_ID_FIELD = 'uniprot_id' - -PROTEIN_DOMAIN_DB = 'PFAM' - -SEQPEEK_VIEW_DEBUG_MODE = False - - -def build_gnab_feature_id(gene): - return "GNAB:{gene_label}:variant_classification".format(gene_label=gene) - - -def get_number_of_unique_samples(track): - sample_ids = set() - for mutation in track['mutations']: - sample_ids.add(mutation[SAMPLE_ID_FIELD_NAME]) - - return len(sample_ids) - - -def get_number_of_mutated_positions(track): - sample_locations = set() - for mutation in track['mutations']: - sample_locations.add(mutation[COORDINATE_FIELD_NAME]) - - return len(sample_locations) - - -# TODO remove if not needed -def clean_track_mutations(mutations_array): - retval = [] - for mutation in mutations_array: - cleaned = deepcopy(mutation) - cleaned[COORDINATE_FIELD_NAME] = int(mutation[COORDINATE_FIELD_NAME]) - retval.append(cleaned) - - return retval - - -def sort_track_mutations(mutations_array): - return sorted(mutations_array, key=lambda k: k[COORDINATE_FIELD_NAME]) - - -def get_track_statistics_by_track_type(track, cohort_info_map): - track_id = track[TRACK_ID_FIELD] - - result = { - 'samples': { - 'numberOf': get_number_of_unique_samples(track), - 'mutated_positions': get_number_of_mutated_positions(track) - } - } - - if track['type'] == 'tumor': - cohort_info = cohort_info_map[track_id] - result['cohort_size'] = cohort_info['size'] - else: - # Do not assign cohort size for the 'COMBINED' track. - result['cohort_size'] = None - - return result - - -def filter_protein_domains(match_array): - return [m for m in match_array if m['dbname'] == PROTEIN_DOMAIN_DB] - - -def get_table_row_id(tumor_type): - return "seqpeek_row_{0}".format(tumor_type) - - -def build_seqpeek_regions(protein_data): - return [{ - 'type': 'exon', - 'start': 0, - 'end': protein_data['length'] - }] - - -def build_summary_track(tracks): - all = [] - for track in tracks: - all.extend(track["mutations"]) - - return { - 'mutations': all, - 'label': 'COMBINED', - 'tumor': 'none-combined', - 'type': 'summary' - } - - -def get_track_label_and_cohort_information(track_id_value, cohort_info_map): - cohort_info = cohort_info_map[track_id_value] - - label = cohort_info['name'] - cohort_size = cohort_info['size'] - return label, cohort_size - - -def get_track_label(track, cohort_info_array): - # The IDs in cohort_info_array are integers, whereas the track IDs are strings. - cohort_map = {str(item['id']): item['name'] for item in cohort_info_array} - return cohort_map[track[TRACK_ID_FIELD]] - - -def get_protein_domains(uniprot_id): - protein = InterProDataProvider().get_data(uniprot_id) - return protein - - -class MAFData(object): - def __init__(self, cohort_info, data): - self.cohort_info = cohort_info - self.data = data - - @classmethod - def from_dict(cls, param): - return cls(param['cohort_set'], param['items']) - - -def build_track_data(track_id_list, all_tumor_mutations): - tracks = [] - for track_id in track_id_list: - tracks.append({ - TRACK_ID_FIELD: track_id, - 'mutations': filter(lambda m: int(track_id) in set(m['cohort']), all_tumor_mutations) - }) - - return tracks - - -def find_uniprot_id(mutations): - uniprot_id = None - for m in mutations: - if PROTEIN_ID_FIELD in m: - uniprot_id = m[PROTEIN_ID_FIELD] - break - - return uniprot_id - - -def get_genes_tumors_lists_debug(): - return { - 'symbol_list': ['EGFR', 'TP53', 'PTEN'], - 'disease_codes': ['ACC', 'BRCA', 'GBM'] - } - - -def get_genes_tumors_lists_remote(): - context = { - 'symbol_list': [], - 'track_id_list': [] - } - - return context - - -def get_genes_tumors_lists(): - if SEQPEEK_VIEW_DEBUG_MODE: - return get_genes_tumors_lists_debug() - else: - return get_genes_tumors_lists_remote() - - -def get_track_id_list(param): - return map(str, param) - - -def format_removed_row_statistics_to_list(stats_dict): - result = [] - for key, value in stats_dict.items(): - result.append({ - 'name': key, - 'num': value - }) - - return result - - -class SeqPeekViewDataBuilder(object): - def build_view_data(self, hugo_symbol, filtered_maf_vector, seqpeek_cohort_info, cohort_id_list, removed_row_statistics): - context = get_genes_tumors_lists() - - cohort_info_map = {str(item['id']): item for item in seqpeek_cohort_info} - track_id_list = get_track_id_list(cohort_id_list) - - # Since the gene (hugo_symbol) parameter is part of the GNAB feature ID, - # it will be sanity-checked in the SeqPeekMAFDataAccess instance. - uniprot_id = find_uniprot_id(filtered_maf_vector) - - logging.info("UniProt ID: " + str(uniprot_id)) - protein_data = get_protein_domains(uniprot_id) - track_data = build_track_data(track_id_list, filtered_maf_vector) - - plot_data = { - 'gene_label': hugo_symbol, - 'tracks': track_data, - 'protein': protein_data - } - - # Pre-processing - # - Sort mutations by chromosomal coordinate - for track in plot_data['tracks']: - track['mutations'] = sort_track_mutations(track['mutations']) - - # Annotations - # - Add label, possibly human readable - # - Add type that indicates whether the track is driven by data from search or - # if the track is aggregate - for track in plot_data['tracks']: - track['type'] = 'tumor' - - label, cohort_size = get_track_label_and_cohort_information(track[TRACK_ID_FIELD], cohort_info_map) - track['label'] = label - - # Display the "combined" track only if more than one cohort is visualized - if len(cohort_id_list) >= 2: - plot_data['tracks'].append(build_summary_track(plot_data['tracks'])) - - for track in plot_data['tracks']: - # Calculate statistics - track['statistics'] = get_track_statistics_by_track_type(track, cohort_info_map) - # Unique ID for each row - track['render_info'] = { - 'row_id': get_table_row_id(track[TRACK_ID_FIELD]) - } - - plot_data['regions'] = build_seqpeek_regions(plot_data['protein']) - plot_data['protein']['matches'] = filter_protein_domains(plot_data['protein']['matches']) - - tumor_list = ','.join(track_id_list) - - context.update({ - 'plot_data': plot_data, - 'hugo_symbol': hugo_symbol, - 'tumor_list': tumor_list, - 'cohort_id_list': track_id_list, - 'removed_row_statistics': format_removed_row_statistics_to_list(removed_row_statistics) - }) - - return context - diff --git a/bq_data_access/seqpeek_maf_data.py b/bq_data_access/seqpeek_maf_data.py deleted file mode 100755 index 10437638..00000000 --- a/bq_data_access/seqpeek_maf_data.py +++ /dev/null @@ -1,85 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from bq_data_access.gnab_data import GNABFeatureProvider -from bq_data_access.utils import DurationLogged - -SEQPEEK_FEATURE_TYPE = 'SEQPEEK' - - -class SeqPeekDataProvider(GNABFeatureProvider): - def __init__(self, feature_id, **kwargs): - super(SeqPeekDataProvider, self).__init__(feature_id, **kwargs) - - @classmethod - def process_data_point(cls, data_point): - return str(data_point['value']) - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - " Hugo_symbol, " - " UniProt_AApos, " - " variant_classification, " - " HGNC_UniProt_ID_Supplied_By_UniProt as uniprot_id " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE Hugo_Symbol='{gene}' " - "AND Tumor_SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=feature_def.gene, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_SEQPEEK: " + query) - return query - - @DurationLogged('SEQPEEK_GNAB', 'UNPACK') - def unpack_query_response(self, query_result_array): - result = [] - - skip_count = 0 - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'hugo_symbol': row['f'][3]['v'], - 'uniprot_aapos': row['f'][4]['v'], - 'variant_classification': row['f'][5]['v'], - 'uniprot_id': row['f'][6]['v'], - }) - - logging.debug("Query result is {qrows} rows, skipped {skipped} rows".format(qrows=len(query_result_array), - skipped=skip_count)) - return result - diff --git a/bq_data_access/user_data.py b/bq_data_access/user_data.py deleted file mode 100644 index c1607150..00000000 --- a/bq_data_access/user_data.py +++ /dev/null @@ -1,456 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from api.api_helpers import sql_connection - -import MySQLdb - -from django.conf import settings - -from bq_data_access.feature_value_types import ValueType - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -USER_FEATURE_TYPE = 'USER' - - -class InvalidUserFeatureIDException(Exception): - def __init__(self, feature_id, reason): - self.feature_id = feature_id - self.reason = reason - - def __str__(self): - return "Invalid feature ID '{feature_id}', reason '{reason}'".format( - feature_id=self.feature_id, - reason=self.reason - ) - - -class UserFeatureDef(object): - def __init__(self, bq_table, column_name, project_id, is_numeric, filters): - logging.debug(str([bq_table, column_name, project_id, is_numeric, filters])) - self.bq_table = bq_table - self.column_name = column_name - self.project_id = project_id - self.filters = filters - self.bq_row_id = None - self.type = "STRING" if is_numeric else "FLOAT" - - def get_value_type(self): - if self.type == "STRING": - return ValueType.STRING - else: - return ValueType.FLOAT - - @classmethod - def get_table_and_field(cls, bq_id): - split = bq_id.split(':') - # First pieces are the project:dataset:table:Data_type:symbol:column_name - bq_table = split[0] + ':' + split[1] + '.' + split[2] - # Last piece is the column name - column_name = bq_id.split(':')[-1] - - # Is only symbol at the moment - symbol = None - if len(split) > 5: - symbol = split[4] - - return bq_table, column_name, symbol - - @classmethod - def from_user_feature_id(cls, feature_id): - logging.debug("UserFeatureDef.from_user_feature_id {0}".format(str([feature_id]))) - # ID breakdown: project ID:Feature ID - # Example ID: USER:1:6 - regex = re_compile("^USER:" - # project ID - "([0-9]+):" - # Feature ID - "([0-9]+)$" - ) - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - project_id, user_feature_id = feature_fields[0] - bq_id = None - shared_id = None - is_numeric = False - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(""" - SELECT feature_name, bq_map_id, shared_map_id, is_numeric - FROM projects_user_feature_definitions - WHERE id = %s - """, (user_feature_id,)) - for row in cursor.fetchall(): - if row['shared_map_id']: - shared_id = row['shared_map_id'] - bq_id = row["bq_map_id"] - is_numeric = row['is_numeric'] == 1 - - cursor.close() - db.close() - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - if shared_id is not None: - return cls.from_feature_id(bq_id, project_id) - - if bq_id is None: - raise FeatureNotFoundException(feature_id) - - # Else we're querying a very specific feature from a specific project - bq_table, column_name, symbol = cls.get_table_and_field(bq_id) - if bq_table is None or column_name is None: - raise FeatureNotFoundException(feature_id) - - logging.debug("{0} {1} {2}".format(bq_table, column_name, symbol)) - - filters = None - if symbol is not None: - filters = { - 'Symbol': symbol - } - - return [cls(bq_table, column_name, project_id, is_numeric, filters)] - - @classmethod - def from_feature_id(cls, feature_id, project_id=None): - logging.debug("UserFeatureDef.from_feature_id: {0}".format(str([feature_id, project_id]))) - if feature_id is None: - raise FeatureNotFoundException(feature_id) - # ID breakdown: project ID:Feature ID - # Example ID: USER:1:6 - regex = re_compile("^USER:" - # project ID - "([0-9]+):" - # Feature ID - "([0-9]+)$" - ) - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - project_id, user_feature_id = feature_fields[0] - - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(""" - SELECT bq_map_id, project_id, is_numeric - FROM projects_user_feature_definitions - WHERE id = %s - """, (user_feature_id,)) - - results = [] - for row in cursor.fetchall(): - bq_table, column_name, symbol = cls.get_table_and_field(row['bq_map_id']) - filters = None - if symbol is not None: - filters = { - 'Symbol': symbol - } - results.append(cls(bq_table, column_name, row['project_id'], row['is_numeric'] == 1, filters)) - - cursor.close() - db.close() - - return results - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - def unpack_value_from_bigquery_row(self, bq_row): - return bq_row['f'][self.bq_row_id + 2]['v'] - - def build_query(self, cohort_table, cohort_ids, project_id_array): - - cohort_str = ",".join([str(cohort_id) for cohort_id in cohort_ids]) - project_id_stmt = '' - if project_id_array is not None and len(project_id_array): - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = "SELECT {fdef_id} AS fdef_id, t.sample_barcode, t.{column_name} FROM [{table_name}] AS t " \ - "JOIN [{cohort_table}] AS c ON c.sample_barcode = t.sample_barcode " \ - "WHERE c.cohort_id IN ({cohort_list}) AND (c.project_id IS NULL" - - - query_template += (" OR c.project_id IN ({project_id_list}))" if project_id_array is not None and len(project_id_array) else ")") - query = query_template.format(fdef_id=self.bq_row_id, - column_name=self.column_name, - table_name=self.bq_table, - cohort_table=cohort_table, - cohort_list=cohort_str, - project_id_list=project_id_stmt) - - if self.filters is not None: - for key, val in self.filters.items(): - query += ' AND t.{filter_key} = "{value}" '.format(filter_key=key, value=val) - - query += " GROUP BY t.sample_barcode, t.{column_name} ".format(column_name=self.column_name) # To prevent duplicates from multiple cohorts - return query - - -class UserFeatureProvider(FeatureDataProvider): - """ - Feature data provider for user data. - """ - def __init__(self, feature_id, user_feature_id=None, **kwargs): - self.feature_defs = None - self.parse_internal_feature_id(feature_id, user_feature_id=user_feature_id) - self._project_ids = None - super(UserFeatureProvider, self).__init__(**kwargs) - - def get_feature_type(self): - return DataTypes.USER - - @classmethod - def convert_user_feature_id(cls, feature_id): - bq_id = None - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(""" - SELECT feature_name, bq_map_id, shared_map_id - FROM projects_user_feature_definitions - WHERE id = %s - """, (int(feature_id.split(':')[-1]),)) - for row in cursor.fetchall(): - bq_id = row["shared_map_id"] - - cursor.close() - db.close() - - logging.debug("UserFeatureProvider.convert_user_feature_id {0} -> {1}".format(feature_id, bq_id)) - return bq_id - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def get_value_type(self): - return self.feature_defs[0].get_value_type() - - def build_query(self, project_ids, cohort_id_array, cohort_dataset, cohort_table, project_id_array): - """ - Builds the BigQuery query string for USER data. The query string is constructed from one or more data sources - (queries), such that each associated UserFeatureDef instance constructs one data source. Each data source - selects one column from a user data table, and therefore maps to one column in the query result. - - The query result table contains a column ("fdef_id") that identifies which data source (UserFeatureDef instance) - that produced each row. - - When unpacking the query result table, the decoding of each row is delegated to the a UserFeatureDef instance - identified by the "fdef_id" column value in that row. - - Example of a query result table: - - |-------+--------------+--------+--------+--------| - |fdef_id|sample_barcode|column_0|column_1|column_n| - |-------+--------------+--------+--------+--------| - |0 |barcode_1 | |null |null | - |0 |barcode_2 | |null |null | - |0 |... | |null |null | - |0 |barcode_m | |null |null | - |-------+--------------+--------+--------+--------| - |1 |barcode_1 |null | |null | - |1 |barcode_2 |null | |null | - |1 |... |null | |null | - |1 |barcode_m |null | |null | - |-------+--------------+--------+--------+--------| - |n |barcode_1 |null |null | | - |n |barcode_2 |null |null | | - |n |... |null |null | | - |n |barcode_m |null |null | | - |-------+--------------+--------+--------+--------| - - Returns: BigQuery query string. - - """ - queries = [] - cohort_table_full = settings.BIGQUERY_PROJECT_NAME + ':' + cohort_dataset + '.' + cohort_table - # TODO: this is a hack to append project_ids to the tcga project id list. project_id_array is actually empty. - project_id_array += project_ids - for feature_def in self.feature_defs: - if int(feature_def.project_id) in project_ids: - # Build our query - queries.append(feature_def.build_query(cohort_table_full, cohort_id_array, project_id_array)) - - # Create a combination query using the UNION ALL operator. Each data source defined above (query1, query2, ...) - # will be combined as follows: - # - # (query 1) - # , - # (query 2) - # , - # ... - # , - # (query n) - # - query = ' , '.join(['(' + query + ')' for query in queries]) - logging.info("BQ_QUERY_USER: " + query) - return query - - def unpack_value_from_row_with_feature_def(self, row): - """ - Decodes the value from a query result. - - Delegates the selection and decoding of the correct column in the row to the UserFeatureDef instance - identified by the "fdef_id" column on the row. - - Args: - row: BigQuery query result row. - - Returns: The value for the row. - - """ - feature_def_index = int(row['f'][0]['v']) # fdef_id - feature_def = self.feature_defs[feature_def_index] - - return feature_def.unpack_value_from_bigquery_row(row) - - @DurationLogged('USER', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': None, - 'sample_id': row['f'][1]['v'], - 'aliquot_id': None, - 'value': self.unpack_value_from_row_with_feature_def(row) - }) - - return result - - def get_project_ids(self, cohort_id_array): - """ - Returns: The user project identifiers associated with the samples in all given cohorts. - """ - if self._project_ids is not None: - return self._project_ids - - project_ids = () - for cohort_id in cohort_id_array: - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - cursor.execute("SELECT project_id FROM cohorts_samples WHERE cohort_id = %s GROUP BY project_id", (cohort_id,)) - for row in cursor.fetchall(): - if row['project_id'] is not None: - project_ids += (row['project_id'],) - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - self._project_ids = project_ids - return self._project_ids - - def parse_internal_feature_id(self, feature_id, user_feature_id=None): - if user_feature_id is None or user_feature_id is '': - logging.debug("UserFeatureProvider.parse_internal_feature_id - feature_id: {0}".format(feature_id)) - self.feature_defs = UserFeatureDef.from_feature_id(feature_id) - else: - logging.debug("UserFeatureProvider.parse_internal_feature_id - user_feature_id: {0}".format(user_feature_id)) - self.feature_defs = UserFeatureDef.from_user_feature_id(user_feature_id) - - # Assign a unique identifier to all feature defs. - # TODO document - for index, feature_def in enumerate(self.feature_defs): - feature_def.bq_row_id = index - - def _submit_query_and_get_job_ref(self, project_id, project_name, dataset_name, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - project_ids = self.get_project_ids(cohort_id_array) - - bigquery_service = self.get_bq_service() - - query_body = self.build_query(project_ids, cohort_id_array, cohort_dataset, cohort_table, project_id_array) - query_job = self.submit_bigquery_job(bigquery_service, project_id, query_body) - - self.job_reference = query_job['jobReference'] - - return self.job_reference - - def is_queryable(self, cohort_id_array): - """ - Answers if this instance would submit a BigQuery job if the submit_query_and_get_job_ref member function - was called. - """ - project_ids = self.get_project_ids(cohort_id_array) - queryable = False - - for feature_def in self.feature_defs: - if int(feature_def.project_id) in project_ids: - queryable = True - break - - return queryable - - def get_data_job_reference(self, cohort_id_array, cohort_dataset, cohort_table, project_id_array): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - - result = self._submit_query_and_get_job_ref(project_id, project_name, dataset_name, - cohort_dataset, cohort_table, cohort_id_array, project_id_array) - return result - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - UserFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # UserFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/utils.py b/bq_data_access/utils.py deleted file mode 100755 index 3c094a01..00000000 --- a/bq_data_access/utils.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from collections import OrderedDict -import logging -from time import time - - -def vector_to_dict(vector, id_key, value_key): - result = {} - for item in vector: - item_id = item[id_key] - value = item[value_key] - result[item_id] = value - - return result - - -def find_all_id_keys(*vectors, **kwargs): - value_key = kwargs['value_key'] - id_key = kwargs['id_key'] - identifiers = set() - for index, vector in enumerate(vectors): - for item in vector: - item_id = item[id_key] - - identifiers.add(item_id) - - return identifiers - - -class VectorMergeSupport(object): - def __init__(self, missing_value, sample_id_key, case_id_key, row_ids=[]): - self.missing_value = missing_value - self.sample_id_key = sample_id_key - self.case_id_key = case_id_key - - # Sparse data goes here - self.data = OrderedDict() - self.sample_ids = OrderedDict() - self.case_ids = {} - self.current_sample_index = 0 - - for row_id in row_ids: - self.data[row_id] = [] - - def _get_index_for_sample_id(self, sample_id): - if sample_id not in self.sample_ids: - self.sample_ids[sample_id] = self.current_sample_index - self.current_sample_index += 1 - - return self.sample_ids[sample_id] - - def _add_data_point(self, row_id, sample_id, value): - if row_id not in self.data: - self.data[row_id] = [] - - self.data[row_id].append((self._get_index_for_sample_id(sample_id), value)) - - def add_dict_array(self, vector, vector_id, value_key): - for item in vector: - sample_id = item[self.sample_id_key] - if item[self.sample_id_key] not in self.case_ids: - self.case_ids[sample_id] = item[self.case_id_key] - value = item[value_key] - self._add_data_point(vector_id, sample_id, value) - - def get_merged_dict(self): - num_samples = self.current_sample_index - result = [{} for _ in xrange(num_samples)] - sample_id_keys = self.sample_ids.keys() - - for row_id, row_samples in self.data.items(): - row_values = [self.missing_value] * len(self.sample_ids) - - for sample_index, value in row_samples: - row_values[sample_index] = value - - counter = 0 - for index, value in enumerate(row_values): - counter += 1 - d = result[index] - d[self.sample_id_key] = sample_id_keys[index] - d[row_id] = value - d[self.case_id_key] = self.case_ids[sample_id_keys[index]] - - return result - - -class DurationLogged(object): - """ - Decorator for logging duration of a function. By default, the messages are logged by calling - "logging.info". The messages have the following format: - " TIME " - - For example, if datatype is CLIN and operation is BQ_QUERY and duration was 1.5 seconds, the message would be - "CLIN BQ_QUERY TIME 1.5". - """ - def __init__(self, datatype, operation): - """ - Constructor. - - Args: - datatype: "Datatype" field for the logged message - operation: "Operation" field for the logged message - """ - self.datatype = datatype - self.operation = operation - - def log(self, msg): - logging.info(msg) - - def __call__(self, f): - def wrapped_function(*args, **kwargs): - time_start = time() - result = f(*args, **kwargs) - time_end = time() - time_elapsed = time_end - time_start - - self.log("{dt} {op} TIME {t}".format(dt=self.datatype, - op=self.operation, - t=str(time_elapsed))) - return result - - return wrapped_function - diff --git a/cgc_api.py b/cgc_api.py deleted file mode 100644 index bb05e37e..00000000 --- a/cgc_api.py +++ /dev/null @@ -1,82 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api.cohorts_delete import CohortsDeleteAPI -from api_3.isb_cgc_api.cohorts_get import CohortsGetAPI -from api_3.isb_cgc_api.cohorts_list import CohortsListAPI -from api_3.isb_cgc_api.cohorts_cloudstoragefilepaths import CohortsCloudStorageFilePathsAPI -from api_3.isb_cgc_api.files_get_file_paths import FilesGetPath -from api_3.isb_cgc_api.cohort_file_manifest import CohortFileManifestAPI - -from api_3.isb_cgc_api_TCGA.cohorts_preview import TCGACohortsPreviewAPI -from api_3.isb_cgc_api_TCGA.cohorts_create import TCGACohortsCreateAPI -from api_3.isb_cgc_api_TCGA.patients_get import TCGACasesGetAPI -from api_3.isb_cgc_api_TCGA.patients_annotations import TCGACasesAnnotationAPI -from api_3.isb_cgc_api_TCGA.samples_get import TCGASamplesGetAPI -from api_3.isb_cgc_api_TCGA.samples_cloudstoragefilepaths import TCGASamplesCloudStorageFilePathsAPI -from api_3.isb_cgc_api_TCGA.samples_annotations import TCGASamplesAnnotationAPI -from api_3.isb_cgc_api_TCGA.aliquots_annotations import TCGAAliquotsAnnotationAPI -from api_3.isb_cgc_api_TCGA.users_get import TCGAUserGetAPI - -from api_3.isb_cgc_api_TARGET.cohorts_preview import TARGETCohortsPreviewAPI -from api_3.isb_cgc_api_TARGET.cohorts_create import TARGETCohortsCreateAPI -from api_3.isb_cgc_api_TARGET.patients_get import TARGETCasesGetAPI -from api_3.isb_cgc_api_TARGET.samples_get import TARGETSamplesGetAPI -from api_3.isb_cgc_api_TARGET.samples_cloudstoragefilepaths import TARGETSamplesCloudStorageFilePathsAPI -from api_3.isb_cgc_api_TARGET.users_get import TARGETUserGetAPI - -from api_3.isb_cgc_api_CCLE.cohorts_preview import CCLECohortsPreviewAPI -from api_3.isb_cgc_api_CCLE.cohorts_create import CCLECohortsCreateAPI -from api_3.isb_cgc_api_CCLE.patients_get import CCLECasesGetAPI -from api_3.isb_cgc_api_CCLE.samples_get import CCLESamplesGetAPI -from api_3.isb_cgc_api_CCLE.samples_cloudstoragefilepaths import CCLESamplesCloudStorageFilePathsAPI - -package = 'isb-cgc-api' - -APPLICATION = endpoints.api_server([ - CohortsDeleteAPI, - CohortsGetAPI, - CohortsListAPI, - CohortsCloudStorageFilePathsAPI, - CohortFileManifestAPI, - FilesGetPath, - - TCGACohortsPreviewAPI, - TCGACohortsCreateAPI, - TCGACasesGetAPI, - TCGACasesAnnotationAPI, - TCGASamplesGetAPI, - TCGASamplesCloudStorageFilePathsAPI, - TCGASamplesAnnotationAPI, - TCGAAliquotsAnnotationAPI, - TCGAUserGetAPI, - - TARGETCohortsPreviewAPI, - TARGETCohortsCreateAPI, - TARGETCasesGetAPI, - TARGETSamplesGetAPI, - TARGETSamplesCloudStorageFilePathsAPI, - TARGETUserGetAPI, - - CCLECohortsPreviewAPI, - CCLECohortsCreateAPI, - CCLECasesGetAPI, - CCLESamplesGetAPI, - CCLESamplesCloudStorageFilePathsAPI -]) diff --git a/isb_cgc_apiv3_openapiv2.json b/isb_cgc_apiv3_openapiv2.json deleted file mode 100644 index 2ddbe4e0..00000000 --- a/isb_cgc_apiv3_openapiv2.json +++ /dev/null @@ -1,500 +0,0 @@ -{ - "basePath": "/_ah/api", - "consumes": [ - "application/json" - ], - "definitions": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "required": [ - "case_barcode", - "disease_code", - "file_path", - "program" - ], - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - } - }, - "host": "api-dot-isb-cgc.appspot.com", - "info": { - "description": "Get information about cohorts for ISB-CGC. List, get, delete cohorts, and retrieve or export their file manifests.", - "title": "isb_cgc_api", - "version": "v3" - }, - "paths": { - "/isb_cgc_api/v3/cohorts": { - "get": { - "operationId": "CohortsListAPI_list", - "parameters": [], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCohortGetListHelperCohortDetailsList" - } - } - } - } - }, - "/isb_cgc_api/v3/cohorts/{cohort_id}": { - "delete": { - "operationId": "CohortsDeleteAPI_delete", - "parameters": [ - { - "format": "int64", - "in": "path", - "name": "cohort_id", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCohortsDeleteReturnJSON" - } - } - } - }, - "get": { - "operationId": "CohortsGetAPI_get", - "parameters": [ - { - "format": "int64", - "in": "path", - "name": "cohort_id", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCohortGetListHelperCohortDetails" - } - } - } - } - }, - "/isb_cgc_api/v3/cohorts/{cohort_id}/cloud_storage_file_paths": { - "get": { - "operationId": "CohortsCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "format": "int64", - "in": "path", - "name": "cohort_id", - "required": true, - "type": "string" - }, - { - "format": "int64", - "in": "query", - "name": "limit", - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "genomic_build", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - }, - "/isb_cgc_api/v3/cohorts/{cohort_id}/file_manifest": { - "get": { - "operationId": "CohortFileManifestAPI_fileManifest", - "parameters": [ - { - "format": "int64", - "in": "path", - "name": "cohort_id", - "required": true, - "type": "string" - }, - { - "format": "int64", - "in": "query", - "name": "fetch_count", - "type": "string" - }, - { - "format": "int64", - "in": "query", - "name": "offset", - "type": "string" - }, - { - "in": "query", - "name": "genomic_build", - "type": "string" - }, - { - "in": "query", - "name": "do_filter_count", - "type": "boolean" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCohortFileManifestFileManifest" - } - } - } - }, - "post": { - "operationId": "CohortFileManifestAPI_fileManifestFiltered", - "parameters": [ - { - "format": "int64", - "in": "path", - "name": "cohort_id", - "required": true, - "type": "string" - }, - { - "format": "int64", - "in": "query", - "name": "fetch_count", - "type": "string" - }, - { - "format": "int64", - "in": "query", - "name": "offset", - "type": "string" - }, - { - "in": "query", - "name": "genomic_build", - "type": "string" - }, - { - "in": "query", - "name": "do_filter_count", - "type": "boolean" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCohortFileManifestFileManifest" - } - } - } - } - }, - "/isb_cgc_api/v3/file_paths": { - "get": { - "operationId": "FilesGetPath_get", - "parameters": [ - { - "collectionFormat": "multi", - "in": "query", - "items": { - "type": "string" - }, - "name": "file_uuids", - "type": "array" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiFilesGetFilePathsFilePaths" - } - } - } - } - } - }, - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "securityDefinitions": { - "google_id_token": { - "authorizationUrl": "", - "flow": "implicit", - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs" - } - }, - "swagger": "2.0" -} \ No newline at end of file diff --git a/isb_cgc_apiv3_openapiv3.json b/isb_cgc_apiv3_openapiv3.json deleted file mode 100644 index 3a32f189..00000000 --- a/isb_cgc_apiv3_openapiv3.json +++ /dev/null @@ -1,571 +0,0 @@ -{ - "openapi": "3.0.0", - "servers": [ - { - "url": "https://api-dot-isb-cgc.appspot.com/_ah/api" - } - ], - "info": { - "description": "Get information about cohorts for ISB-CGC. List, get, delete cohorts, and retrieve or export their file manifests.", - "title": "ISB-CGC API: Cohorts", - "version": "v3" - }, - "paths": { - "/isb_cgc_api/v3/cohorts": { - "get": { - "operationId": "CohortsListAPI_list", - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortGetListHelperCohortDetailsList" - } - } - } - } - } - } - }, - "/isb_cgc_api/v3/cohorts/{cohort_id}": { - "delete": { - "operationId": "CohortsDeleteAPI_delete", - "parameters": [ - { - "in": "path", - "name": "cohort_id", - "required": true, - "schema": { - "type": "string", - "format": "int64" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortsDeleteReturnJSON" - } - } - } - } - } - }, - "get": { - "operationId": "CohortsGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "cohort_id", - "required": true, - "schema": { - "type": "string", - "format": "int64" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortGetListHelperCohortDetails" - } - } - } - } - } - } - }, - "/isb_cgc_api/v3/cohorts/{cohort_id}/cloud_storage_file_paths": { - "get": { - "operationId": "CohortsCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "cohort_id", - "required": true, - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "limit", - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "genomic_build", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - } - } - }, - "/isb_cgc_api/v3/cohorts/{cohort_id}/file_manifest": { - "get": { - "operationId": "CohortFileManifestAPI_fileManifest", - "parameters": [ - { - "in": "path", - "name": "cohort_id", - "required": true, - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "fetch_count", - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "offset", - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "genomic_build", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "do_filter_count", - "schema": { - "type": "boolean" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortFileManifestFileManifest" - } - } - } - } - } - }, - "post": { - "operationId": "CohortFileManifestAPI_fileManifestFiltered", - "parameters": [ - { - "in": "path", - "name": "cohort_id", - "required": true, - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "fetch_count", - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "offset", - "schema": { - "type": "string", - "format": "int64" - } - }, - { - "in": "query", - "name": "genomic_build", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "do_filter_count", - "schema": { - "type": "boolean" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortFileManifestFileManifest" - } - } - } - } - } - } - }, - "/isb_cgc_api/v3/file_paths": { - "get": { - "operationId": "FilesGetPath_get", - "parameters": [ - { - "in": "query", - "name": "file_uuids", - "explode": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiFilesGetFilePathsFilePaths" - } - } - } - } - } - } - } - }, - "components": { - "schemas": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "required": [ - "case_barcode", - "disease_code", - "file_path", - "program" - ], - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - } - }, - "securitySchemes": { - "google_id_token": { - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs", - "flows": { - "implicit": { - "authorizationUrl": "/", - "scopes": {} - } - } - } - } - } -} \ No newline at end of file diff --git a/isb_cgc_ccle_apiv3_openapiv2.json b/isb_cgc_ccle_apiv3_openapiv2.json deleted file mode 100644 index 1483b86e..00000000 --- a/isb_cgc_ccle_apiv3_openapiv2.json +++ /dev/null @@ -1,4083 +0,0 @@ -{ - "basePath": "/_ah/api", - "consumes": [ - "application/json" - ], - "definitions": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCohortCasesSamplesList": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCreatedCohort": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesClinicalMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "hist_subtype": { - "type": "string" - }, - "histology": { - "type": "string" - }, - "site_primary": { - "type": "string" - }, - "source": { - "type": "string" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesClinicalMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hist_subtype": { - "items": { - "type": "string" - }, - "type": "array" - }, - "histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "site_primary": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source": { - "items": { - "type": "string" - }, - "type": "array" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLESamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/definitions/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "tumor_code": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_code": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "event_free_survival": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "protocol": { - "type": "string" - }, - "race": { - "type": "string" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "type": "string" - }, - "wbc_at_diagnosis": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "event_free_survival": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "event_free_survival_gte": { - "format": "int32", - "type": "integer" - }, - "event_free_survival_lte": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "protocol": { - "items": { - "type": "string" - }, - "type": "array" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "wbc_at_diagnosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "wbc_at_diagnosis_gte": { - "format": "double", - "type": "number" - }, - "wbc_at_diagnosis_lte": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_last_follow_up_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETSamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/definitions/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem": { - "properties": { - "aliquot_barcode": { - "type": "string" - }, - "annotation_gdc_id": { - "type": "string" - }, - "annotation_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "category": { - "type": "string" - }, - "classification": { - "type": "string" - }, - "entity_barcode": { - "type": "string" - }, - "entity_gdc_id": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "status": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_collection": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "num_portions": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "type": "string" - }, - "preservation_method": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_collection": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_collection_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_collection_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_sample_procurement_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement_lte": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "num_portions": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_portions_gte": { - "format": "int32", - "type": "integer" - }, - "num_portions_lte": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_slides_gte": { - "format": "int32", - "type": "integer" - }, - "num_slides_lte": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "items": { - "type": "string" - }, - "type": "array" - }, - "preservation_method": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "type": "string" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "bmi": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "clinical_M": { - "type": "string" - }, - "clinical_N": { - "type": "string" - }, - "clinical_T": { - "type": "string" - }, - "clinical_stage": { - "type": "string" - }, - "colorectal_cancer": { - "type": "string" - }, - "country": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "gleason_score_combined": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "type": "string" - }, - "height": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "type": "string" - }, - "history_of_colon_polyps": { - "type": "string" - }, - "history_of_neoadjuvant_treatment": { - "type": "string" - }, - "hpv_calls": { - "type": "string" - }, - "hpv_status": { - "type": "string" - }, - "icd_10": { - "type": "string" - }, - "icd_o_3_histology": { - "type": "string" - }, - "icd_o_3_site": { - "type": "string" - }, - "lymphatic_invasion": { - "type": "string" - }, - "lymphnodes_examined": { - "type": "string" - }, - "lymphovascular_invasion_present": { - "type": "string" - }, - "menopause_status": { - "type": "string" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "type": "string" - }, - "neoplasm_histologic_grade": { - "type": "string" - }, - "new_tumor_event_after_initial_treatment": { - "type": "string" - }, - "number_of_lymphnodes_examined": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "type": "string" - }, - "other_malignancy_anatomic_site": { - "type": "string" - }, - "other_malignancy_histological_type": { - "type": "string" - }, - "other_malignancy_type": { - "type": "string" - }, - "pathologic_M": { - "type": "string" - }, - "pathologic_N": { - "type": "string" - }, - "pathologic_T": { - "type": "string" - }, - "pathologic_stage": { - "type": "string" - }, - "person_neoplasm_cancer_status": { - "type": "string" - }, - "pregnancies": { - "type": "string" - }, - "primary_neoplasm_melanoma_dx": { - "type": "string" - }, - "primary_therapy_outcome_success": { - "type": "string" - }, - "psa_value": { - "format": "double", - "type": "number" - }, - "race": { - "type": "string" - }, - "residual_tumor": { - "type": "string" - }, - "stopped_smoking_year": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "type": "string" - }, - "tss_code": { - "type": "string" - }, - "tumor_tissue_site": { - "type": "string" - }, - "tumor_type": { - "type": "string" - }, - "venous_invasion": { - "type": "string" - }, - "vital_status": { - "type": "string" - }, - "weight": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_began_smoking_in_years_gte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years_lte": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "items": { - "type": "string" - }, - "type": "array" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "bmi": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "bmi_gte": { - "format": "double", - "type": "number" - }, - "bmi_lte": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "colorectal_cancer": { - "items": { - "type": "string" - }, - "type": "array" - }, - "country": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_initial_pathologic_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_submitted_specimen_dx_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gleason_score_combined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "gleason_score_combined_gte": { - "format": "int32", - "type": "integer" - }, - "gleason_score_combined_lte": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "items": { - "type": "string" - }, - "type": "array" - }, - "height": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "height_gte": { - "format": "int32", - "type": "integer" - }, - "height_lte": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_colon_polyps": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_neoadjuvant_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_calls": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_10": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphatic_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphnodes_examined": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphovascular_invasion_present": { - "items": { - "type": "string" - }, - "type": "array" - }, - "menopause_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "neoplasm_histologic_grade": { - "items": { - "type": "string" - }, - "type": "array" - }, - "new_tumor_event_after_initial_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "number_of_lymphnodes_examined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_examined_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_examined_lte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_positive_by_he_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he_lte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_pack_years_smoked_gte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked_lte": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_anatomic_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "person_neoplasm_cancer_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pregnancies": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_neoplasm_melanoma_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_therapy_outcome_success": { - "items": { - "type": "string" - }, - "type": "array" - }, - "psa_value": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "psa_value_gte": { - "format": "double", - "type": "number" - }, - "psa_value_lte": { - "format": "double", - "type": "number" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "residual_tumor": { - "items": { - "type": "string" - }, - "type": "array" - }, - "stopped_smoking_year": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "stopped_smoking_year_gte": { - "format": "int32", - "type": "integer" - }, - "stopped_smoking_year_lte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tss_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_tissue_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "venous_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "weight": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "weight_gte": { - "format": "int32", - "type": "integer" - }, - "weight_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_tobacco_smoking_onset_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGASamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/definitions/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3SamplesGetHelperDataDetails": { - "properties": { - "access": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_name_key": { - "type": "string" - }, - "file_size": { - "format": "int64", - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3UsersGetCommonUserGetAPIReturnJSON": { - "properties": { - "dbGaP_allowed": { - "type": "boolean" - }, - "dbGaP_authorized": { - "type": "boolean" - }, - "message": { - "type": "string" - } - }, - "type": "object" - } - }, - "host": "api-dot-isb-cgc.appspot.com", - "info": { - "description": "Get information about cohorts, cases, and samples for CCLE. Create cohorts.", - "title": "isb_cgc_ccle_api", - "version": "v3" - }, - "paths": { - "/isb_cgc_ccle_api/v3/ccle/cases/{case_barcode}": { - "get": { - "operationId": "CCLECasesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEPatientsGetCaseDetails" - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/ccle/cohorts/create": { - "post": { - "operationId": "CCLECohortsCreateAPI_create", - "parameters": [ - { - "in": "body", - "name": "body", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem" - } - }, - { - "in": "query", - "name": "name", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperCreatedCohort" - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/ccle/samples/{sample_barcode}": { - "get": { - "operationId": "CCLESamplesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "endpoint_type", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCCLESamplesGetSampleDetails" - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/ccle/samples/{sample_barcode}/cloud_storage_file_paths": { - "get": { - "operationId": "CCLESamplesCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "genomic_build", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/tcga/cohorts/preview": { - "post": { - "operationId": "CCLECohortsPreviewAPI_preview", - "parameters": [ - { - "in": "body", - "name": "body", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperCohortCasesSamplesList" - } - } - } - } - } - }, - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "securityDefinitions": { - "google_id_token": { - "authorizationUrl": "", - "flow": "implicit", - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs" - } - }, - "swagger": "2.0" -} \ No newline at end of file diff --git a/isb_cgc_ccle_apiv3_openapiv3.json b/isb_cgc_ccle_apiv3_openapiv3.json deleted file mode 100644 index 429cc535..00000000 --- a/isb_cgc_ccle_apiv3_openapiv3.json +++ /dev/null @@ -1,4140 +0,0 @@ -{ - "openapi": "3.0.0", - "servers": [ - { - "url": "https://api-dot-isb-cgc.appspot.com/_ah/api" - } - ], - "info": { - "description": "Get information about cases and samples for CCLE; preview and create cohorts containing CCLE cases and samples.", - "title": "ISB-CGC API: CCLE", - "version": "v3" - }, - "paths": { - "/isb_cgc_ccle_api/v3/ccle/cases/{case_barcode}": { - "get": { - "operationId": "CCLECasesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEPatientsGetCaseDetails" - } - } - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/ccle/cohorts/create": { - "post": { - "operationId": "CCLECohortsCreateAPI_create", - "parameters": [ - { - "in": "query", - "name": "name", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperCreatedCohort" - } - } - } - } - }, - "requestBody": { - "$ref": "#/components/requestBodies/Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem" - } - } - }, - "/isb_cgc_ccle_api/v3/ccle/samples/{sample_barcode}": { - "get": { - "operationId": "CCLESamplesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "endpoint_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLESamplesGetSampleDetails" - } - } - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/ccle/samples/{sample_barcode}/cloud_storage_file_paths": { - "get": { - "operationId": "CCLESamplesCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "genomic_build", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - } - } - }, - "/isb_cgc_ccle_api/v3/tcga/cohorts/preview": { - "post": { - "operationId": "CCLECohortsPreviewAPI_preview", - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperCohortCasesSamplesList" - } - } - } - } - }, - "requestBody": { - "$ref": "#/components/requestBodies/Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem" - } - } - } - }, - "components": { - "schemas": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCohortCasesSamplesList": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCreatedCohort": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesClinicalMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "hist_subtype": { - "type": "string" - }, - "histology": { - "type": "string" - }, - "site_primary": { - "type": "string" - }, - "source": { - "type": "string" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesClinicalMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hist_subtype": { - "items": { - "type": "string" - }, - "type": "array" - }, - "histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "site_primary": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source": { - "items": { - "type": "string" - }, - "type": "array" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesDataHG19MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLEPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCCLESamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/components/schemas/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "tumor_code": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_code": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "event_free_survival": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "protocol": { - "type": "string" - }, - "race": { - "type": "string" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "type": "string" - }, - "wbc_at_diagnosis": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "event_free_survival": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "event_free_survival_gte": { - "format": "int32", - "type": "integer" - }, - "event_free_survival_lte": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "protocol": { - "items": { - "type": "string" - }, - "type": "array" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "wbc_at_diagnosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "wbc_at_diagnosis_gte": { - "format": "double", - "type": "number" - }, - "wbc_at_diagnosis_lte": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_last_follow_up_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETSamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/components/schemas/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem": { - "properties": { - "aliquot_barcode": { - "type": "string" - }, - "annotation_gdc_id": { - "type": "string" - }, - "annotation_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "category": { - "type": "string" - }, - "classification": { - "type": "string" - }, - "entity_barcode": { - "type": "string" - }, - "entity_gdc_id": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "status": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_collection": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "num_portions": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "type": "string" - }, - "preservation_method": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_collection": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_collection_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_collection_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_sample_procurement_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement_lte": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "num_portions": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_portions_gte": { - "format": "int32", - "type": "integer" - }, - "num_portions_lte": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_slides_gte": { - "format": "int32", - "type": "integer" - }, - "num_slides_lte": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "items": { - "type": "string" - }, - "type": "array" - }, - "preservation_method": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "type": "string" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "bmi": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "clinical_M": { - "type": "string" - }, - "clinical_N": { - "type": "string" - }, - "clinical_T": { - "type": "string" - }, - "clinical_stage": { - "type": "string" - }, - "colorectal_cancer": { - "type": "string" - }, - "country": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "gleason_score_combined": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "type": "string" - }, - "height": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "type": "string" - }, - "history_of_colon_polyps": { - "type": "string" - }, - "history_of_neoadjuvant_treatment": { - "type": "string" - }, - "hpv_calls": { - "type": "string" - }, - "hpv_status": { - "type": "string" - }, - "icd_10": { - "type": "string" - }, - "icd_o_3_histology": { - "type": "string" - }, - "icd_o_3_site": { - "type": "string" - }, - "lymphatic_invasion": { - "type": "string" - }, - "lymphnodes_examined": { - "type": "string" - }, - "lymphovascular_invasion_present": { - "type": "string" - }, - "menopause_status": { - "type": "string" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "type": "string" - }, - "neoplasm_histologic_grade": { - "type": "string" - }, - "new_tumor_event_after_initial_treatment": { - "type": "string" - }, - "number_of_lymphnodes_examined": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "type": "string" - }, - "other_malignancy_anatomic_site": { - "type": "string" - }, - "other_malignancy_histological_type": { - "type": "string" - }, - "other_malignancy_type": { - "type": "string" - }, - "pathologic_M": { - "type": "string" - }, - "pathologic_N": { - "type": "string" - }, - "pathologic_T": { - "type": "string" - }, - "pathologic_stage": { - "type": "string" - }, - "person_neoplasm_cancer_status": { - "type": "string" - }, - "pregnancies": { - "type": "string" - }, - "primary_neoplasm_melanoma_dx": { - "type": "string" - }, - "primary_therapy_outcome_success": { - "type": "string" - }, - "psa_value": { - "format": "double", - "type": "number" - }, - "race": { - "type": "string" - }, - "residual_tumor": { - "type": "string" - }, - "stopped_smoking_year": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "type": "string" - }, - "tss_code": { - "type": "string" - }, - "tumor_tissue_site": { - "type": "string" - }, - "tumor_type": { - "type": "string" - }, - "venous_invasion": { - "type": "string" - }, - "vital_status": { - "type": "string" - }, - "weight": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_began_smoking_in_years_gte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years_lte": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "items": { - "type": "string" - }, - "type": "array" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "bmi": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "bmi_gte": { - "format": "double", - "type": "number" - }, - "bmi_lte": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "colorectal_cancer": { - "items": { - "type": "string" - }, - "type": "array" - }, - "country": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_initial_pathologic_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_submitted_specimen_dx_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gleason_score_combined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "gleason_score_combined_gte": { - "format": "int32", - "type": "integer" - }, - "gleason_score_combined_lte": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "items": { - "type": "string" - }, - "type": "array" - }, - "height": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "height_gte": { - "format": "int32", - "type": "integer" - }, - "height_lte": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_colon_polyps": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_neoadjuvant_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_calls": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_10": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphatic_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphnodes_examined": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphovascular_invasion_present": { - "items": { - "type": "string" - }, - "type": "array" - }, - "menopause_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "neoplasm_histologic_grade": { - "items": { - "type": "string" - }, - "type": "array" - }, - "new_tumor_event_after_initial_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "number_of_lymphnodes_examined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_examined_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_examined_lte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_positive_by_he_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he_lte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_pack_years_smoked_gte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked_lte": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_anatomic_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "person_neoplasm_cancer_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pregnancies": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_neoplasm_melanoma_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_therapy_outcome_success": { - "items": { - "type": "string" - }, - "type": "array" - }, - "psa_value": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "psa_value_gte": { - "format": "double", - "type": "number" - }, - "psa_value_lte": { - "format": "double", - "type": "number" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "residual_tumor": { - "items": { - "type": "string" - }, - "type": "array" - }, - "stopped_smoking_year": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "stopped_smoking_year_gte": { - "format": "int32", - "type": "integer" - }, - "stopped_smoking_year_lte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tss_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_tissue_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "venous_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "weight": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "weight_gte": { - "format": "int32", - "type": "integer" - }, - "weight_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_tobacco_smoking_onset_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGASamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/components/schemas/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3SamplesGetHelperDataDetails": { - "properties": { - "access": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_name_key": { - "type": "string" - }, - "file_size": { - "format": "int64", - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3UsersGetCommonUserGetAPIReturnJSON": { - "properties": { - "dbGaP_allowed": { - "type": "boolean" - }, - "dbGaP_authorized": { - "type": "boolean" - }, - "message": { - "type": "string" - } - }, - "type": "object" - } - }, - "requestBodies": { - "Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiCCLEMessageClassesMetadataRangesItem" - } - } - } - } - }, - "securitySchemes": { - "google_id_token": { - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs", - "flows": { - "implicit": { - "authorizationUrl": "/", - "scopes": {} - } - } - } - } - } -} \ No newline at end of file diff --git a/isb_cgc_target_apiv3_openapiv2.json b/isb_cgc_target_apiv3_openapiv2.json deleted file mode 100644 index 76779529..00000000 --- a/isb_cgc_target_apiv3_openapiv2.json +++ /dev/null @@ -1,3612 +0,0 @@ -{ - "basePath": "/_ah/api", - "consumes": [ - "application/json" - ], - "definitions": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCohortCasesSamplesList": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCreatedCohort": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "tumor_code": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_code": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "event_free_survival": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "protocol": { - "type": "string" - }, - "race": { - "type": "string" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "type": "string" - }, - "wbc_at_diagnosis": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "event_free_survival": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "event_free_survival_gte": { - "format": "int32", - "type": "integer" - }, - "event_free_survival_lte": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "protocol": { - "items": { - "type": "string" - }, - "type": "array" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "wbc_at_diagnosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "wbc_at_diagnosis_gte": { - "format": "double", - "type": "number" - }, - "wbc_at_diagnosis_lte": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_last_follow_up_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETSamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/definitions/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem": { - "properties": { - "aliquot_barcode": { - "type": "string" - }, - "annotation_gdc_id": { - "type": "string" - }, - "annotation_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "category": { - "type": "string" - }, - "classification": { - "type": "string" - }, - "entity_barcode": { - "type": "string" - }, - "entity_gdc_id": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "status": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_collection": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "num_portions": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "type": "string" - }, - "preservation_method": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_collection": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_collection_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_collection_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_sample_procurement_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement_lte": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "num_portions": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_portions_gte": { - "format": "int32", - "type": "integer" - }, - "num_portions_lte": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_slides_gte": { - "format": "int32", - "type": "integer" - }, - "num_slides_lte": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "items": { - "type": "string" - }, - "type": "array" - }, - "preservation_method": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "type": "string" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "bmi": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "clinical_M": { - "type": "string" - }, - "clinical_N": { - "type": "string" - }, - "clinical_T": { - "type": "string" - }, - "clinical_stage": { - "type": "string" - }, - "colorectal_cancer": { - "type": "string" - }, - "country": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "gleason_score_combined": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "type": "string" - }, - "height": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "type": "string" - }, - "history_of_colon_polyps": { - "type": "string" - }, - "history_of_neoadjuvant_treatment": { - "type": "string" - }, - "hpv_calls": { - "type": "string" - }, - "hpv_status": { - "type": "string" - }, - "icd_10": { - "type": "string" - }, - "icd_o_3_histology": { - "type": "string" - }, - "icd_o_3_site": { - "type": "string" - }, - "lymphatic_invasion": { - "type": "string" - }, - "lymphnodes_examined": { - "type": "string" - }, - "lymphovascular_invasion_present": { - "type": "string" - }, - "menopause_status": { - "type": "string" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "type": "string" - }, - "neoplasm_histologic_grade": { - "type": "string" - }, - "new_tumor_event_after_initial_treatment": { - "type": "string" - }, - "number_of_lymphnodes_examined": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "type": "string" - }, - "other_malignancy_anatomic_site": { - "type": "string" - }, - "other_malignancy_histological_type": { - "type": "string" - }, - "other_malignancy_type": { - "type": "string" - }, - "pathologic_M": { - "type": "string" - }, - "pathologic_N": { - "type": "string" - }, - "pathologic_T": { - "type": "string" - }, - "pathologic_stage": { - "type": "string" - }, - "person_neoplasm_cancer_status": { - "type": "string" - }, - "pregnancies": { - "type": "string" - }, - "primary_neoplasm_melanoma_dx": { - "type": "string" - }, - "primary_therapy_outcome_success": { - "type": "string" - }, - "psa_value": { - "format": "double", - "type": "number" - }, - "race": { - "type": "string" - }, - "residual_tumor": { - "type": "string" - }, - "stopped_smoking_year": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "type": "string" - }, - "tss_code": { - "type": "string" - }, - "tumor_tissue_site": { - "type": "string" - }, - "tumor_type": { - "type": "string" - }, - "venous_invasion": { - "type": "string" - }, - "vital_status": { - "type": "string" - }, - "weight": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_began_smoking_in_years_gte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years_lte": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "items": { - "type": "string" - }, - "type": "array" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "bmi": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "bmi_gte": { - "format": "double", - "type": "number" - }, - "bmi_lte": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "colorectal_cancer": { - "items": { - "type": "string" - }, - "type": "array" - }, - "country": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_initial_pathologic_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_submitted_specimen_dx_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gleason_score_combined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "gleason_score_combined_gte": { - "format": "int32", - "type": "integer" - }, - "gleason_score_combined_lte": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "items": { - "type": "string" - }, - "type": "array" - }, - "height": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "height_gte": { - "format": "int32", - "type": "integer" - }, - "height_lte": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_colon_polyps": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_neoadjuvant_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_calls": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_10": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphatic_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphnodes_examined": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphovascular_invasion_present": { - "items": { - "type": "string" - }, - "type": "array" - }, - "menopause_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "neoplasm_histologic_grade": { - "items": { - "type": "string" - }, - "type": "array" - }, - "new_tumor_event_after_initial_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "number_of_lymphnodes_examined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_examined_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_examined_lte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_positive_by_he_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he_lte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_pack_years_smoked_gte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked_lte": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_anatomic_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "person_neoplasm_cancer_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pregnancies": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_neoplasm_melanoma_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_therapy_outcome_success": { - "items": { - "type": "string" - }, - "type": "array" - }, - "psa_value": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "psa_value_gte": { - "format": "double", - "type": "number" - }, - "psa_value_lte": { - "format": "double", - "type": "number" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "residual_tumor": { - "items": { - "type": "string" - }, - "type": "array" - }, - "stopped_smoking_year": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "stopped_smoking_year_gte": { - "format": "int32", - "type": "integer" - }, - "stopped_smoking_year_lte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tss_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_tissue_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "venous_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "weight": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "weight_gte": { - "format": "int32", - "type": "integer" - }, - "weight_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_tobacco_smoking_onset_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGASamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/definitions/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3SamplesGetHelperDataDetails": { - "properties": { - "access": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_name_key": { - "type": "string" - }, - "file_size": { - "format": "int64", - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3UsersGetCommonUserGetAPIReturnJSON": { - "properties": { - "dbGaP_allowed": { - "type": "boolean" - }, - "dbGaP_authorized": { - "type": "boolean" - }, - "message": { - "type": "string" - } - }, - "type": "object" - } - }, - "host": "api-dot-isb-cgc.appspot.com", - "info": { - "description": "Get information about cohorts, cases, and samples for TARGET. Create cohorts.", - "title": "isb_cgc_target_api", - "version": "v3" - }, - "paths": { - "/isb_cgc_target_api/v3/target/cases/{case_barcode}": { - "get": { - "operationId": "TARGETCasesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETPatientsGetCaseDetails" - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/cohorts/create": { - "post": { - "operationId": "TARGETCohortsCreateAPI_create", - "parameters": [ - { - "in": "body", - "name": "body", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem" - } - }, - { - "in": "query", - "name": "name", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperCreatedCohort" - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/samples/{sample_barcode}": { - "get": { - "operationId": "TARGETSamplesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "endpoint_type", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETSamplesGetSampleDetails" - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/samples/{sample_barcode}/cloud_storage_file_paths": { - "get": { - "operationId": "TARGETSamplesCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "genomic_build", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/users": { - "get": { - "operationId": "TARGETUserGetAPI_get", - "parameters": [], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3UsersGetCommonUserGetAPIReturnJSON" - } - } - } - } - }, - "/isb_cgc_target_api/v3/tcga/cohorts/preview": { - "post": { - "operationId": "TARGETCohortsPreviewAPI_preview", - "parameters": [ - { - "in": "body", - "name": "body", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperCohortCasesSamplesList" - } - } - } - } - } - }, - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "securityDefinitions": { - "google_id_token": { - "authorizationUrl": "", - "flow": "implicit", - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs" - } - }, - "swagger": "2.0" -} \ No newline at end of file diff --git a/isb_cgc_target_apiv3_openapiv3.json b/isb_cgc_target_apiv3_openapiv3.json deleted file mode 100644 index cbb2cb16..00000000 --- a/isb_cgc_target_apiv3_openapiv3.json +++ /dev/null @@ -1,3672 +0,0 @@ -{ - "openapi": "3.0.0", - "servers": [ - { - "url": "https://api-dot-isb-cgc.appspot.com/_ah/api" - } - ], - "info": { - "description": "Get information about cases and samples for TARGET; preview and create cohorts containing TARGET cases and samples.", - "title": "ISB-CGC APi: TARGET", - "version": "v3" - }, - "paths": { - "/isb_cgc_target_api/v3/target/cases/{case_barcode}": { - "get": { - "operationId": "TARGETCasesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETPatientsGetCaseDetails" - } - } - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/cohorts/create": { - "post": { - "operationId": "TARGETCohortsCreateAPI_create", - "parameters": [ - { - "in": "query", - "name": "name", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperCreatedCohort" - } - } - } - } - }, - "requestBody": { - "$ref": "#/components/requestBodies/Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem" - } - } - }, - "/isb_cgc_target_api/v3/target/samples/{sample_barcode}": { - "get": { - "operationId": "TARGETSamplesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "endpoint_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETSamplesGetSampleDetails" - } - } - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/samples/{sample_barcode}/cloud_storage_file_paths": { - "get": { - "operationId": "TARGETSamplesCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "genomic_build", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - } - } - }, - "/isb_cgc_target_api/v3/target/users": { - "get": { - "operationId": "TARGETUserGetAPI_get", - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3UsersGetCommonUserGetAPIReturnJSON" - } - } - } - } - } - } - }, - "/isb_cgc_target_api/v3/tcga/cohorts/preview": { - "post": { - "operationId": "TARGETCohortsPreviewAPI_preview", - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperCohortCasesSamplesList" - } - } - } - } - }, - "requestBody": { - "$ref": "#/components/requestBodies/Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem" - } - } - } - }, - "components": { - "schemas": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCohortCasesSamplesList": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCreatedCohort": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem": { - "properties": { - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "tumor_code": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_code": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "event_free_survival": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "protocol": { - "type": "string" - }, - "race": { - "type": "string" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "type": "string" - }, - "wbc_at_diagnosis": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "event_free_survival": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "event_free_survival_gte": { - "format": "int32", - "type": "integer" - }, - "event_free_survival_lte": { - "format": "int32", - "type": "integer" - }, - "first_event": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "protocol": { - "items": { - "type": "string" - }, - "type": "array" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "wbc_at_diagnosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "wbc_at_diagnosis_gte": { - "format": "double", - "type": "number" - }, - "wbc_at_diagnosis_lte": { - "format": "double", - "type": "number" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_last_follow_up_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_last_follow_up_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTARGETSamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/components/schemas/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem": { - "properties": { - "aliquot_barcode": { - "type": "string" - }, - "annotation_gdc_id": { - "type": "string" - }, - "annotation_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "category": { - "type": "string" - }, - "classification": { - "type": "string" - }, - "entity_barcode": { - "type": "string" - }, - "entity_gdc_id": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "status": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_collection": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "num_portions": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "type": "string" - }, - "preservation_method": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_collection": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_collection_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_collection_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_sample_procurement_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement_lte": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "num_portions": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_portions_gte": { - "format": "int32", - "type": "integer" - }, - "num_portions_lte": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_slides_gte": { - "format": "int32", - "type": "integer" - }, - "num_slides_lte": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "items": { - "type": "string" - }, - "type": "array" - }, - "preservation_method": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "type": "string" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "bmi": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "clinical_M": { - "type": "string" - }, - "clinical_N": { - "type": "string" - }, - "clinical_T": { - "type": "string" - }, - "clinical_stage": { - "type": "string" - }, - "colorectal_cancer": { - "type": "string" - }, - "country": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "gleason_score_combined": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "type": "string" - }, - "height": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "type": "string" - }, - "history_of_colon_polyps": { - "type": "string" - }, - "history_of_neoadjuvant_treatment": { - "type": "string" - }, - "hpv_calls": { - "type": "string" - }, - "hpv_status": { - "type": "string" - }, - "icd_10": { - "type": "string" - }, - "icd_o_3_histology": { - "type": "string" - }, - "icd_o_3_site": { - "type": "string" - }, - "lymphatic_invasion": { - "type": "string" - }, - "lymphnodes_examined": { - "type": "string" - }, - "lymphovascular_invasion_present": { - "type": "string" - }, - "menopause_status": { - "type": "string" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "type": "string" - }, - "neoplasm_histologic_grade": { - "type": "string" - }, - "new_tumor_event_after_initial_treatment": { - "type": "string" - }, - "number_of_lymphnodes_examined": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "type": "string" - }, - "other_malignancy_anatomic_site": { - "type": "string" - }, - "other_malignancy_histological_type": { - "type": "string" - }, - "other_malignancy_type": { - "type": "string" - }, - "pathologic_M": { - "type": "string" - }, - "pathologic_N": { - "type": "string" - }, - "pathologic_T": { - "type": "string" - }, - "pathologic_stage": { - "type": "string" - }, - "person_neoplasm_cancer_status": { - "type": "string" - }, - "pregnancies": { - "type": "string" - }, - "primary_neoplasm_melanoma_dx": { - "type": "string" - }, - "primary_therapy_outcome_success": { - "type": "string" - }, - "psa_value": { - "format": "double", - "type": "number" - }, - "race": { - "type": "string" - }, - "residual_tumor": { - "type": "string" - }, - "stopped_smoking_year": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "type": "string" - }, - "tss_code": { - "type": "string" - }, - "tumor_tissue_site": { - "type": "string" - }, - "tumor_type": { - "type": "string" - }, - "venous_invasion": { - "type": "string" - }, - "vital_status": { - "type": "string" - }, - "weight": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_began_smoking_in_years_gte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years_lte": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "items": { - "type": "string" - }, - "type": "array" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "bmi": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "bmi_gte": { - "format": "double", - "type": "number" - }, - "bmi_lte": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "colorectal_cancer": { - "items": { - "type": "string" - }, - "type": "array" - }, - "country": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_initial_pathologic_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_submitted_specimen_dx_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gleason_score_combined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "gleason_score_combined_gte": { - "format": "int32", - "type": "integer" - }, - "gleason_score_combined_lte": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "items": { - "type": "string" - }, - "type": "array" - }, - "height": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "height_gte": { - "format": "int32", - "type": "integer" - }, - "height_lte": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_colon_polyps": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_neoadjuvant_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_calls": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_10": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphatic_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphnodes_examined": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphovascular_invasion_present": { - "items": { - "type": "string" - }, - "type": "array" - }, - "menopause_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "neoplasm_histologic_grade": { - "items": { - "type": "string" - }, - "type": "array" - }, - "new_tumor_event_after_initial_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "number_of_lymphnodes_examined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_examined_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_examined_lte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_positive_by_he_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he_lte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_pack_years_smoked_gte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked_lte": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_anatomic_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "person_neoplasm_cancer_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pregnancies": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_neoplasm_melanoma_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_therapy_outcome_success": { - "items": { - "type": "string" - }, - "type": "array" - }, - "psa_value": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "psa_value_gte": { - "format": "double", - "type": "number" - }, - "psa_value_lte": { - "format": "double", - "type": "number" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "residual_tumor": { - "items": { - "type": "string" - }, - "type": "array" - }, - "stopped_smoking_year": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "stopped_smoking_year_gte": { - "format": "int32", - "type": "integer" - }, - "stopped_smoking_year_lte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tss_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_tissue_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "venous_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "weight": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "weight_gte": { - "format": "int32", - "type": "integer" - }, - "weight_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_tobacco_smoking_onset_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGASamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/components/schemas/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3SamplesGetHelperDataDetails": { - "properties": { - "access": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_name_key": { - "type": "string" - }, - "file_size": { - "format": "int64", - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3UsersGetCommonUserGetAPIReturnJSON": { - "properties": { - "dbGaP_allowed": { - "type": "boolean" - }, - "dbGaP_authorized": { - "type": "boolean" - }, - "message": { - "type": "string" - } - }, - "type": "object" - } - }, - "requestBodies": { - "Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTARGETMessageClassesMetadataRangesItem" - } - } - } - } - }, - "securitySchemes": { - "google_id_token": { - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs", - "flows": { - "implicit": { - "authorizationUrl": "/", - "scopes": {} - } - } - } - } - } -} \ No newline at end of file diff --git a/isb_cgc_tcga_apiv3_openapiv2.json b/isb_cgc_tcga_apiv3_openapiv2.json deleted file mode 100644 index 7be9694c..00000000 --- a/isb_cgc_tcga_apiv3_openapiv2.json +++ /dev/null @@ -1,2819 +0,0 @@ -{ - "basePath": "/_ah/api", - "consumes": [ - "application/json" - ], - "definitions": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCohortCasesSamplesList": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCreatedCohort": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/definitions/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem": { - "properties": { - "aliquot_barcode": { - "type": "string" - }, - "annotation_gdc_id": { - "type": "string" - }, - "annotation_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "category": { - "type": "string" - }, - "classification": { - "type": "string" - }, - "entity_barcode": { - "type": "string" - }, - "entity_gdc_id": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "status": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_collection": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "num_portions": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "type": "string" - }, - "preservation_method": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_collection": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_collection_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_collection_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_sample_procurement_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement_lte": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "num_portions": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_portions_gte": { - "format": "int32", - "type": "integer" - }, - "num_portions_lte": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_slides_gte": { - "format": "int32", - "type": "integer" - }, - "num_slides_lte": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "items": { - "type": "string" - }, - "type": "array" - }, - "preservation_method": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "type": "string" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "bmi": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "clinical_M": { - "type": "string" - }, - "clinical_N": { - "type": "string" - }, - "clinical_T": { - "type": "string" - }, - "clinical_stage": { - "type": "string" - }, - "colorectal_cancer": { - "type": "string" - }, - "country": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "gleason_score_combined": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "type": "string" - }, - "height": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "type": "string" - }, - "history_of_colon_polyps": { - "type": "string" - }, - "history_of_neoadjuvant_treatment": { - "type": "string" - }, - "hpv_calls": { - "type": "string" - }, - "hpv_status": { - "type": "string" - }, - "icd_10": { - "type": "string" - }, - "icd_o_3_histology": { - "type": "string" - }, - "icd_o_3_site": { - "type": "string" - }, - "lymphatic_invasion": { - "type": "string" - }, - "lymphnodes_examined": { - "type": "string" - }, - "lymphovascular_invasion_present": { - "type": "string" - }, - "menopause_status": { - "type": "string" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "type": "string" - }, - "neoplasm_histologic_grade": { - "type": "string" - }, - "new_tumor_event_after_initial_treatment": { - "type": "string" - }, - "number_of_lymphnodes_examined": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "type": "string" - }, - "other_malignancy_anatomic_site": { - "type": "string" - }, - "other_malignancy_histological_type": { - "type": "string" - }, - "other_malignancy_type": { - "type": "string" - }, - "pathologic_M": { - "type": "string" - }, - "pathologic_N": { - "type": "string" - }, - "pathologic_T": { - "type": "string" - }, - "pathologic_stage": { - "type": "string" - }, - "person_neoplasm_cancer_status": { - "type": "string" - }, - "pregnancies": { - "type": "string" - }, - "primary_neoplasm_melanoma_dx": { - "type": "string" - }, - "primary_therapy_outcome_success": { - "type": "string" - }, - "psa_value": { - "format": "double", - "type": "number" - }, - "race": { - "type": "string" - }, - "residual_tumor": { - "type": "string" - }, - "stopped_smoking_year": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "type": "string" - }, - "tss_code": { - "type": "string" - }, - "tumor_tissue_site": { - "type": "string" - }, - "tumor_type": { - "type": "string" - }, - "venous_invasion": { - "type": "string" - }, - "vital_status": { - "type": "string" - }, - "weight": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_began_smoking_in_years_gte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years_lte": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "items": { - "type": "string" - }, - "type": "array" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "bmi": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "bmi_gte": { - "format": "double", - "type": "number" - }, - "bmi_lte": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "colorectal_cancer": { - "items": { - "type": "string" - }, - "type": "array" - }, - "country": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_initial_pathologic_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_submitted_specimen_dx_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gleason_score_combined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "gleason_score_combined_gte": { - "format": "int32", - "type": "integer" - }, - "gleason_score_combined_lte": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "items": { - "type": "string" - }, - "type": "array" - }, - "height": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "height_gte": { - "format": "int32", - "type": "integer" - }, - "height_lte": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_colon_polyps": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_neoadjuvant_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_calls": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_10": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphatic_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphnodes_examined": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphovascular_invasion_present": { - "items": { - "type": "string" - }, - "type": "array" - }, - "menopause_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "neoplasm_histologic_grade": { - "items": { - "type": "string" - }, - "type": "array" - }, - "new_tumor_event_after_initial_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "number_of_lymphnodes_examined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_examined_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_examined_lte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_positive_by_he_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he_lte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_pack_years_smoked_gte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked_lte": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_anatomic_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "person_neoplasm_cancer_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pregnancies": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_neoplasm_melanoma_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_therapy_outcome_success": { - "items": { - "type": "string" - }, - "type": "array" - }, - "psa_value": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "psa_value_gte": { - "format": "double", - "type": "number" - }, - "psa_value_lte": { - "format": "double", - "type": "number" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "residual_tumor": { - "items": { - "type": "string" - }, - "type": "array" - }, - "stopped_smoking_year": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "stopped_smoking_year_gte": { - "format": "int32", - "type": "integer" - }, - "stopped_smoking_year_lte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tss_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_tissue_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "venous_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "weight": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "weight_gte": { - "format": "int32", - "type": "integer" - }, - "weight_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_tobacco_smoking_onset_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGASamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/definitions/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3SamplesGetHelperDataDetails": { - "properties": { - "access": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_name_key": { - "type": "string" - }, - "file_size": { - "format": "int64", - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3UsersGetCommonUserGetAPIReturnJSON": { - "properties": { - "dbGaP_allowed": { - "type": "boolean" - }, - "dbGaP_authorized": { - "type": "boolean" - }, - "message": { - "type": "string" - } - }, - "type": "object" - } - }, - "host": "api-dot-isb-cgc.appspot.com", - "info": { - "description": "Get information about cohorts, cases, and samples for TCGA. Create cohorts.", - "title": "isb_cgc_tcga_api", - "version": "v3" - }, - "paths": { - "/isb_cgc_tcga_api/v3/aliquots/{aliquot_barcode}/annotations": { - "get": { - "operationId": "TCGAAliquotsAnnotationAPI_annotations", - "parameters": [ - { - "in": "path", - "name": "aliquot_barcode", - "required": true, - "type": "string" - }, - { - "collectionFormat": "multi", - "in": "query", - "items": { - "type": "string" - }, - "name": "entity_type", - "type": "array" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/cases/{case_barcode}/annotations": { - "get": { - "operationId": "TCGACasesAnnotationAPI_annotations", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "type": "string" - }, - { - "collectionFormat": "multi", - "in": "query", - "items": { - "type": "string" - }, - "name": "entity_type", - "type": "array" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/samples/{sample_barcode}/annotations": { - "get": { - "operationId": "TCGASamplesAnnotationAPI_annotations", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "collectionFormat": "multi", - "in": "query", - "items": { - "type": "string" - }, - "name": "entity_type", - "type": "array" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/cases/{case_barcode}": { - "get": { - "operationId": "TCGACasesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAPatientsGetCaseDetails" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/cohorts/create": { - "post": { - "operationId": "TCGACohortsCreateAPI_create", - "parameters": [ - { - "in": "body", - "name": "body", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem" - } - }, - { - "in": "query", - "name": "name", - "required": true, - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperCreatedCohort" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/cohorts/preview": { - "post": { - "operationId": "TCGACohortsPreviewAPI_preview", - "parameters": [ - { - "in": "body", - "name": "body", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem" - } - }, - { - "in": "query", - "name": "fields", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CohortCreatePreviewHelperCohortCasesSamplesList" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/samples/{sample_barcode}": { - "get": { - "operationId": "TCGASamplesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "endpoint_type", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3IsbCgcApiTCGASamplesGetSampleDetails" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/samples/{sample_barcode}/cloud_storage_file_paths": { - "get": { - "operationId": "TCGASamplesCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "type": "string" - }, - { - "in": "query", - "name": "data_type", - "type": "string" - }, - { - "in": "query", - "name": "data_category", - "type": "string" - }, - { - "in": "query", - "name": "experimental_strategy", - "type": "string" - }, - { - "in": "query", - "name": "data_format", - "type": "string" - }, - { - "in": "query", - "name": "platform", - "type": "string" - }, - { - "in": "query", - "name": "genomic_build", - "type": "string" - }, - { - "in": "query", - "name": "analysis_workflow_type", - "type": "string" - } - ], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/users": { - "get": { - "operationId": "TCGAUserGetAPI_get", - "parameters": [], - "responses": { - "200": { - "description": "A successful response", - "schema": { - "$ref": "#/definitions/Api3UsersGetCommonUserGetAPIReturnJSON" - } - } - } - } - } - }, - "produces": [ - "application/json" - ], - "schemes": [ - "https" - ], - "securityDefinitions": { - "google_id_token": { - "authorizationUrl": "", - "flow": "implicit", - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs" - } - }, - "swagger": "2.0" -} \ No newline at end of file diff --git a/isb_cgc_tcga_apiv3_openapiv3.json b/isb_cgc_tcga_apiv3_openapiv3.json deleted file mode 100644 index c5aea028..00000000 --- a/isb_cgc_tcga_apiv3_openapiv3.json +++ /dev/null @@ -1,2907 +0,0 @@ -{ - "openapi": "3.0.0", - "servers": [ - { - "url": "https://api-dot-isb-cgc.appspot.com/_ah/api" - } - ], - "info": { - "description": "Get information about cases and samples for TCGA; preview and create cohorts containing TCGA cases and samples.", - "title": "ISB-CGC API: TCGA", - "version": "v3" - }, - "paths": { - "/isb_cgc_tcga_api/v3/aliquots/{aliquot_barcode}/annotations": { - "get": { - "operationId": "TCGAAliquotsAnnotationAPI_annotations", - "parameters": [ - { - "in": "path", - "name": "aliquot_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "entity_type", - "explode": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList" - } - } - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/cases/{case_barcode}/annotations": { - "get": { - "operationId": "TCGACasesAnnotationAPI_annotations", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "entity_type", - "explode": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList" - } - } - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/samples/{sample_barcode}/annotations": { - "get": { - "operationId": "TCGASamplesAnnotationAPI_annotations", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "entity_type", - "explode": true, - "schema": { - "type": "array", - "items": { - "type": "string" - } - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList" - } - } - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/cases/{case_barcode}": { - "get": { - "operationId": "TCGACasesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "case_barcode", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAPatientsGetCaseDetails" - } - } - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/cohorts/create": { - "post": { - "operationId": "TCGACohortsCreateAPI_create", - "parameters": [ - { - "in": "query", - "name": "name", - "required": true, - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperCreatedCohort" - } - } - } - } - }, - "requestBody": { - "$ref": "#/components/requestBodies/Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem" - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/cohorts/preview": { - "post": { - "operationId": "TCGACohortsPreviewAPI_preview", - "parameters": [ - { - "in": "query", - "name": "fields", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperCohortCasesSamplesList" - } - } - } - } - }, - "requestBody": { - "$ref": "#/components/requestBodies/Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem" - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/samples/{sample_barcode}": { - "get": { - "operationId": "TCGASamplesGetAPI_get", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "endpoint_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGASamplesGetSampleDetails" - } - } - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/samples/{sample_barcode}/cloud_storage_file_paths": { - "get": { - "operationId": "TCGASamplesCloudStorageFilePathsAPI_cloudStorageFilePaths", - "parameters": [ - { - "in": "path", - "name": "sample_barcode", - "required": true, - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_type", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_category", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "experimental_strategy", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "data_format", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "platform", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "genomic_build", - "schema": { - "type": "string" - } - }, - { - "in": "query", - "name": "analysis_workflow_type", - "schema": { - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3CloudstoragefilepathsHelperGCSFilePathList" - } - } - } - } - } - } - }, - "/isb_cgc_tcga_api/v3/tcga/users": { - "get": { - "operationId": "TCGAUserGetAPI_get", - "responses": { - "200": { - "description": "A successful response", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3UsersGetCommonUserGetAPIReturnJSON" - } - } - } - } - } - } - } - }, - "components": { - "schemas": { - "Api3CloudstoragefilepathsHelperGCSFilePathList": { - "properties": { - "cloud_storage_file_paths": { - "items": { - "type": "string" - }, - "type": "array" - }, - "count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCohortCasesSamplesList": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperCreatedCohort": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortCreatePreviewHelperFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3CohortCreatePreviewHelperFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3CohortEndpointHelpersFilterDetails": { - "properties": { - "name": { - "type": "string" - }, - "value": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileDetail": { - "properties": { - "access": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_uuid": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_uuid": { - "type": "string" - }, - "file_path": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortFileManifestFileManifest": { - "properties": { - "files": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortFileManifestFileDetail" - }, - "type": "array" - }, - "files_retrieved": { - "format": "int32", - "type": "integer" - }, - "total_file_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "cases": { - "items": { - "type": "string" - }, - "type": "array" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortDetailsList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiCohortGetListHelperCohortListDetails" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortGetListHelperCohortListDetails": { - "properties": { - "case_count": { - "format": "int32", - "type": "integer" - }, - "comments": { - "type": "string" - }, - "email": { - "type": "string" - }, - "filters": { - "items": { - "$ref": "#/components/schemas/Api3CohortEndpointHelpersFilterDetails" - }, - "type": "array" - }, - "id": { - "type": "string" - }, - "last_date_saved": { - "type": "string" - }, - "name": { - "type": "string" - }, - "parent_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "permission": { - "type": "string" - }, - "sample_count": { - "format": "int32", - "type": "integer" - }, - "source_notes": { - "type": "string" - }, - "source_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiCohortsDeleteReturnJSON": { - "properties": { - "message": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiFilesGetFilePathsFilePaths": { - "properties": { - "paths": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAAnnotationsApiMetadataAnnotationList": { - "properties": { - "count": { - "format": "int32", - "type": "integer" - }, - "items": { - "items": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesAnnotationMetadataItem": { - "properties": { - "aliquot_barcode": { - "type": "string" - }, - "annotation_gdc_id": { - "type": "string" - }, - "annotation_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "category": { - "type": "string" - }, - "classification": { - "type": "string" - }, - "entity_barcode": { - "type": "string" - }, - "entity_gdc_id": { - "type": "string" - }, - "entity_type": { - "type": "string" - }, - "notes": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "status": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "days_to_collection": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "format": "double", - "type": "number" - }, - "num_portions": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "type": "string" - }, - "preservation_method": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem": { - "properties": { - "avg_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "avg_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "avg_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_collection": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_collection_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_collection_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_sample_procurement_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_sample_procurement_lte": { - "format": "int32", - "type": "integer" - }, - "max_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "max_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "max_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "max_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "max_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_lymphocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_lymphocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_monocyte_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_monocyte_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_necrosis_gte": { - "format": "double", - "type": "number" - }, - "min_percent_necrosis_lte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_neutrophil_infiltration_gte": { - "format": "double", - "type": "number" - }, - "min_percent_neutrophil_infiltration_lte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_normal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_normal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_stromal_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_stromal_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_cells_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_cells_lte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "min_percent_tumor_nuclei_gte": { - "format": "double", - "type": "number" - }, - "min_percent_tumor_nuclei_lte": { - "format": "double", - "type": "number" - }, - "num_portions": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_portions_gte": { - "format": "int32", - "type": "integer" - }, - "num_portions_lte": { - "format": "int32", - "type": "integer" - }, - "num_slides": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "num_slides_gte": { - "format": "int32", - "type": "integer" - }, - "num_slides_lte": { - "format": "int32", - "type": "integer" - }, - "pathology_report_uuid": { - "items": { - "type": "string" - }, - "type": "array" - }, - "preservation_method": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem": { - "properties": { - "age_at_diagnosis": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "type": "string" - }, - "batch_number": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "type": "string" - }, - "bmi": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "clinical_M": { - "type": "string" - }, - "clinical_N": { - "type": "string" - }, - "clinical_T": { - "type": "string" - }, - "clinical_stage": { - "type": "string" - }, - "colorectal_cancer": { - "type": "string" - }, - "country": { - "type": "string" - }, - "days_to_birth": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "type": "string" - }, - "gender": { - "type": "string" - }, - "gleason_score_combined": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "type": "string" - }, - "height": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "type": "string" - }, - "history_of_colon_polyps": { - "type": "string" - }, - "history_of_neoadjuvant_treatment": { - "type": "string" - }, - "hpv_calls": { - "type": "string" - }, - "hpv_status": { - "type": "string" - }, - "icd_10": { - "type": "string" - }, - "icd_o_3_histology": { - "type": "string" - }, - "icd_o_3_site": { - "type": "string" - }, - "lymphatic_invasion": { - "type": "string" - }, - "lymphnodes_examined": { - "type": "string" - }, - "lymphovascular_invasion_present": { - "type": "string" - }, - "menopause_status": { - "type": "string" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "type": "string" - }, - "neoplasm_histologic_grade": { - "type": "string" - }, - "new_tumor_event_after_initial_treatment": { - "type": "string" - }, - "number_of_lymphnodes_examined": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "type": "string" - }, - "other_malignancy_anatomic_site": { - "type": "string" - }, - "other_malignancy_histological_type": { - "type": "string" - }, - "other_malignancy_type": { - "type": "string" - }, - "pathologic_M": { - "type": "string" - }, - "pathologic_N": { - "type": "string" - }, - "pathologic_T": { - "type": "string" - }, - "pathologic_stage": { - "type": "string" - }, - "person_neoplasm_cancer_status": { - "type": "string" - }, - "pregnancies": { - "type": "string" - }, - "primary_neoplasm_melanoma_dx": { - "type": "string" - }, - "primary_therapy_outcome_success": { - "type": "string" - }, - "psa_value": { - "format": "double", - "type": "number" - }, - "race": { - "type": "string" - }, - "residual_tumor": { - "type": "string" - }, - "stopped_smoking_year": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "type": "string" - }, - "tss_code": { - "type": "string" - }, - "tumor_tissue_site": { - "type": "string" - }, - "tumor_type": { - "type": "string" - }, - "venous_invasion": { - "type": "string" - }, - "vital_status": { - "type": "string" - }, - "weight": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem": { - "properties": { - "age_at_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_at_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "age_at_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "age_began_smoking_in_years_gte": { - "format": "int32", - "type": "integer" - }, - "age_began_smoking_in_years_lte": { - "format": "int32", - "type": "integer" - }, - "anatomic_neoplasm_subdivision": { - "items": { - "type": "string" - }, - "type": "array" - }, - "batch_number": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "batch_number_gte": { - "format": "int32", - "type": "integer" - }, - "batch_number_lte": { - "format": "int32", - "type": "integer" - }, - "bcr": { - "items": { - "type": "string" - }, - "type": "array" - }, - "bmi": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "bmi_gte": { - "format": "double", - "type": "number" - }, - "bmi_lte": { - "format": "double", - "type": "number" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "colorectal_cancer": { - "items": { - "type": "string" - }, - "type": "array" - }, - "country": { - "items": { - "type": "string" - }, - "type": "array" - }, - "days_to_birth": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_birth_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_birth_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_death": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_death_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_death_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_initial_pathologic_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_initial_pathologic_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_followup_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_followup_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_last_known_alive_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_last_known_alive_lte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "days_to_submitted_specimen_dx_gte": { - "format": "int32", - "type": "integer" - }, - "days_to_submitted_specimen_dx_lte": { - "format": "int32", - "type": "integer" - }, - "ethnicity": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gender": { - "items": { - "type": "string" - }, - "type": "array" - }, - "gleason_score_combined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "gleason_score_combined_gte": { - "format": "int32", - "type": "integer" - }, - "gleason_score_combined_lte": { - "format": "int32", - "type": "integer" - }, - "h_pylori_infection": { - "items": { - "type": "string" - }, - "type": "array" - }, - "height": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "height_gte": { - "format": "int32", - "type": "integer" - }, - "height_lte": { - "format": "int32", - "type": "integer" - }, - "histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_colon_polyps": { - "items": { - "type": "string" - }, - "type": "array" - }, - "history_of_neoadjuvant_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_calls": { - "items": { - "type": "string" - }, - "type": "array" - }, - "hpv_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_10": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_histology": { - "items": { - "type": "string" - }, - "type": "array" - }, - "icd_o_3_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphatic_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphnodes_examined": { - "items": { - "type": "string" - }, - "type": "array" - }, - "lymphovascular_invasion_present": { - "items": { - "type": "string" - }, - "type": "array" - }, - "menopause_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "neoplasm_histologic_grade": { - "items": { - "type": "string" - }, - "type": "array" - }, - "new_tumor_event_after_initial_treatment": { - "items": { - "type": "string" - }, - "type": "array" - }, - "number_of_lymphnodes_examined": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_examined_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_examined_lte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_of_lymphnodes_positive_by_he_gte": { - "format": "int32", - "type": "integer" - }, - "number_of_lymphnodes_positive_by_he_lte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "number_pack_years_smoked_gte": { - "format": "int32", - "type": "integer" - }, - "number_pack_years_smoked_lte": { - "format": "int32", - "type": "integer" - }, - "other_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_anatomic_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_histological_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "other_malignancy_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_M": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_N": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_T": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pathologic_stage": { - "items": { - "type": "string" - }, - "type": "array" - }, - "person_neoplasm_cancer_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "pregnancies": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_neoplasm_melanoma_dx": { - "items": { - "type": "string" - }, - "type": "array" - }, - "primary_therapy_outcome_success": { - "items": { - "type": "string" - }, - "type": "array" - }, - "psa_value": { - "items": { - "format": "double", - "type": "number" - }, - "type": "array" - }, - "psa_value_gte": { - "format": "double", - "type": "number" - }, - "psa_value_lte": { - "format": "double", - "type": "number" - }, - "race": { - "items": { - "type": "string" - }, - "type": "array" - }, - "residual_tumor": { - "items": { - "type": "string" - }, - "type": "array" - }, - "stopped_smoking_year": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "stopped_smoking_year_gte": { - "format": "int32", - "type": "integer" - }, - "stopped_smoking_year_lte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "summary_file_count_gte": { - "format": "int32", - "type": "integer" - }, - "summary_file_count_lte": { - "format": "int32", - "type": "integer" - }, - "tobacco_smoking_history": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tss_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_tissue_site": { - "items": { - "type": "string" - }, - "type": "array" - }, - "tumor_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "venous_invasion": { - "items": { - "type": "string" - }, - "type": "array" - }, - "vital_status": { - "items": { - "type": "string" - }, - "type": "array" - }, - "weight": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "weight_gte": { - "format": "int32", - "type": "integer" - }, - "weight_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_diagnosis_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_diagnosis_lte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset": { - "items": { - "format": "int32", - "type": "integer" - }, - "type": "array" - }, - "year_of_tobacco_smoking_onset_gte": { - "format": "int32", - "type": "integer" - }, - "year_of_tobacco_smoking_onset_lte": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem": { - "properties": { - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem": { - "properties": { - "disease_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "endpoint_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "program_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "project_short_name": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem": { - "properties": { - "access": { - "type": "string" - }, - "aliquot_barcode": { - "type": "string" - }, - "aliquot_gdc_id": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "archive_file_name": { - "type": "string" - }, - "archive_submitter_id": { - "type": "string" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "center_code": { - "type": "string" - }, - "center_name": { - "type": "string" - }, - "center_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_state": { - "type": "string" - }, - "file_uploaded": { - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - }, - "species": { - "type": "string" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem": { - "properties": { - "access": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "aliquot_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "analysis_workflow_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "archive_submitter_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "case_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_code": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "center_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_category": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_format": { - "items": { - "type": "string" - }, - "type": "array" - }, - "data_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "experimental_strategy": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_state": { - "items": { - "type": "string" - }, - "type": "array" - }, - "file_uploaded": { - "items": { - "type": "string" - }, - "type": "array" - }, - "index_file_name": { - "items": { - "type": "string" - }, - "type": "array" - }, - "platform": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_barcode": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_gdc_id": { - "items": { - "type": "string" - }, - "type": "array" - }, - "sample_type": { - "items": { - "type": "string" - }, - "type": "array" - }, - "species": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesCommonMetadataItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "properties": { - "Biospecimen": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesBiospecimenMetadataRangesItem" - }, - "Clinical": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesClinicalMetadataRangesItem" - }, - "Common": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesCommonMetadataRangesItem" - }, - "Data_HG19": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG19MetadataRangesItem" - }, - "Data_HG38": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesDataHG38MetadataRangesItem" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGAPatientsGetCaseDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "clinical_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "samples": { - "items": { - "type": "string" - }, - "type": "array" - } - }, - "type": "object" - }, - "Api3IsbCgcApiTCGASamplesGetSampleDetails": { - "properties": { - "aliquots": { - "items": { - "type": "string" - }, - "type": "array" - }, - "biospecimen_data": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataItem" - }, - "case_barcode": { - "type": "string" - }, - "case_gdc_id": { - "type": "string" - }, - "data_details": { - "items": { - "$ref": "#/components/schemas/Api3SamplesGetHelperDataDetails" - }, - "type": "array" - }, - "data_details_count": { - "format": "int32", - "type": "integer" - } - }, - "type": "object" - }, - "Api3SamplesGetHelperDataDetails": { - "properties": { - "access": { - "type": "string" - }, - "analysis_workflow_type": { - "type": "string" - }, - "data_category": { - "type": "string" - }, - "data_format": { - "type": "string" - }, - "data_type": { - "type": "string" - }, - "disease_code": { - "type": "string" - }, - "endpoint_type": { - "type": "string" - }, - "experimental_strategy": { - "type": "string" - }, - "file_gdc_id": { - "type": "string" - }, - "file_name": { - "type": "string" - }, - "file_name_key": { - "type": "string" - }, - "file_size": { - "format": "int64", - "type": "string" - }, - "index_file_name": { - "type": "string" - }, - "platform": { - "type": "string" - }, - "program_name": { - "type": "string" - }, - "project_short_name": { - "type": "string" - }, - "sample_barcode": { - "type": "string" - }, - "sample_gdc_id": { - "type": "string" - }, - "sample_type": { - "type": "string" - } - }, - "type": "object" - }, - "Api3UsersGetCommonUserGetAPIReturnJSON": { - "properties": { - "dbGaP_allowed": { - "type": "boolean" - }, - "dbGaP_authorized": { - "type": "boolean" - }, - "message": { - "type": "string" - } - }, - "type": "object" - } - }, - "requestBodies": { - "Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Api3IsbCgcApiTCGAMessageClassesMetadataRangesItem" - } - } - } - } - }, - "securitySchemes": { - "google_id_token": { - "type": "oauth2", - "x-google-issuer": "https://accounts.google.com", - "x-google-jwks_uri": "https://www.googleapis.com/oauth2/v3/certs", - "flows": { - "implicit": { - "authorizationUrl": "/", - "scopes": {} - } - } - } - } - } -} \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 00000000..fcb8a4a8 --- /dev/null +++ b/main.py @@ -0,0 +1,77 @@ +""" + +Copyright 2017, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import base64 +import json +import logging + +from flask import Flask, jsonify, request +from flask_cors import cross_origin + +app = Flask(__name__) + + +def _base64_decode(encoded_str): + # Add paddings manually if necessary. + num_missed_paddings = 4 - len(encoded_str) % 4 + if num_missed_paddings != 4: + encoded_str += b'=' * num_missed_paddings + return base64.b64decode(encoded_str).decode('utf-8') + + + +# [START endpoints_auth_info_backend] +def auth_info(): + """Retrieves the authenication information from Google Cloud Endpoints.""" + encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None) + + if encoded_info: + info_json = _base64_decode(encoded_info) + user_info = json.loads(info_json) + else: + user_info = {'id': 'anonymous'} + + return jsonify(user_info) +# [START endpoints_auth_info_backend] + + +@app.route('/auth/info/googlejwt', methods=['GET']) +def auth_info_google_jwt(): + """Auth info with Google signed JWT.""" + return auth_info() + + +@app.route('/auth/info/googleidtoken', methods=['GET']) +def auth_info_google_id_token(): + """Auth info with Google ID token.""" + return auth_info() + + +@app.errorhandler(http_client.INTERNAL_SERVER_ERROR) +def unexpected_error(e): + """Handle exceptions by returning swagger-compliant json.""" + logging.exception('An error occured while processing the request.') + response = jsonify({ + 'code': http_client.INTERNAL_SERVER_ERROR, + 'message': 'Exception: {}'.format(e)}) + response.status_code = http_client.INTERNAL_SERVER_ERROR + return response + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8080, debug=True) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0e83fdf5..35dce3ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +django==1.11.18 +mysqlclient==1.3.13 requests==2.20.0 ecdsa==0.13 google-api-python-client==1.6.1 @@ -16,4 +18,3 @@ django-finalware==1.0.0 django-allauth==0.35.0 jsonschema==2.6.0 enum34==1.1.6 -google-endpoints==4.0.0 \ No newline at end of file From 68187d11bccae6e37aa3362572b780fbdf72aba7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 4 Mar 2019 14:43:48 -0800 Subject: [PATCH 003/323] -> First pass --- auth.py | 48 ++++++++++++++++ cohorts/__init__.py | 0 cohorts/routes.py | 112 ++++++++++++++++++++++++++++++++++++ cohorts/views.py | 136 ++++++++++++++++++++++++++++++++++++++++++++ main.py | 56 ++++++------------ 5 files changed, 312 insertions(+), 40 deletions(-) create mode 100644 auth.py create mode 100644 cohorts/__init__.py create mode 100644 cohorts/routes.py create mode 100644 cohorts/views.py diff --git a/auth.py b/auth.py new file mode 100644 index 00000000..4dc7e7a2 --- /dev/null +++ b/auth.py @@ -0,0 +1,48 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import base64 +import json + +from flask import request + +logger = logging.getLogger(__name__) + + +# BEGIN METHODS +def _base64_decode(encoded_str): + # Add paddings manually if necessary. + num_missed_paddings = 4 - len(encoded_str) % 4 + if num_missed_paddings != 4: + encoded_str += b'=' * num_missed_paddings + return base64.b64decode(encoded_str).decode('utf-8') + + +def auth_info(): + """Retrieves the authenication information from Google Cloud Endpoints.""" + encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None) + + if encoded_info: + info_json = _base64_decode(encoded_info) + user_info = json.loads(info_json) + else: + user_info = {'id': 'anonymous'} + + return jsonify(user_info) +# END METHODS diff --git a/cohorts/__init__.py b/cohorts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cohorts/routes.py b/cohorts/routes.py new file mode 100644 index 00000000..c24390fd --- /dev/null +++ b/cohorts/routes.py @@ -0,0 +1,112 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +from flask import jsonify, request + +from . import views + + +@app.route('/cohorts/', methods=['GET']) +def cohort(): + """Retrieve extended information for a specific cohort""" + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + response = None + + if not user: + response = jsonify({ + 'code': 403, + 'message': "User {} does not have access to cohort ID {}".format(user_info['email'],str(cohort_id))}) + response.status_code = 403 + + else: + cohort_info = get_cohort_info(cohort_id) + if cohort_info: + response = jsonify({ + 'code': 200, + 'data': jsonify(cohort_info) + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Cohort ID {} was not found.".format(str(cohort_id))}) + response.status_code = 404 + + return response + + +@app.route('/cohorts', methods=['GET']) +def cohorts(): + """Retrieve a user's list of cohorts""" + user_info = auth_info() + + cohort_list = get_cohorts(user_info['email']) + + response = None + + if not cohort_list: + response = jsonify({ + 'code': 404, + 'message': "No cohorts were found for user {}".format(user_info['email'])}) + response.status_code = 404 + + else: + cohort_info = get_cohort_info(cohort_id) + if cohort_info: + response = jsonify({ + 'code': 200, + 'data': jsonify(cohort_info) + }) + response.status_code = 200 + + return response + + +@app.route('/cohorts//file_manifest', methods=['POST', 'GET']) +def cohort_file_manifest(): + """Retrieve a cohort's file manifest""" + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + response = None + + if not user: + response = jsonify({ + 'code': 403, + 'message': "User {} does not have access to cohort ID {}".format(user_info['email'],str(cohort_id))}) + response.status_code = 403 + + else: + file_manifest = get_file_manifest(cohort_id, user) + if file_manifest: + response = jsonify({ + 'code': 200, + 'data': file_manifest + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id))}) + response.status_code = 500 + + return response diff --git a/cohorts/views.py b/cohorts/views.py new file mode 100644 index 00000000..7442f4ab --- /dev/null +++ b/cohorts/views.py @@ -0,0 +1,136 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +import django + +from flask import request + +from django.core.signals import request_finished +from django.contrib.auth.models import User as Django_User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.conf import settings + +from cohorts.models import Cohort_Perms, Cohort, Filters +from accounts.sa_utils import auth_dataset_whitelists_for_user +from cohorts.file_helpers import cohort_files + +import auth + + +logger = logging.getLogger(__name__) + + +def validate_user(user_email, cohort_id): + user = None + django.setup() + try: + user = Django_User.objects.get(email=user_email) + Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) + except ObjectDoesNotExist as e: + err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) + if 'Cohort_Perms' in e.message: + err_msg = "User {} does not have permissions on cohort {}.".format(user_email, cohort_id) + logger.warn(err_msg) + except Exception as e: + logger.exception(e) + + return user + + +def get_file_manifest(cohort_id, user): + file_manifest = None + inc_filters = {} + + try: + has_access = auth_dataset_whitelists_for_user(user.id) + + params = { + 'limit': settings.MAX_FILE_LIST_REQUEST, + 'build': 'HG19', + 'access': has_access + } + + request_data = request.get_json() + + if request_data: + if request_data['offset']: + params['offset'] = request.get_assigned_value('offset') + + if request_data['fetch_count']: + params['limit'] = request.get_assigned_value('fetch_count') + + if request_data['genomic_build']: + params['build'] = request.get_assigned_value('genomic_build').upper() + + inc_filters = { + filter: request_data[filter] + for filter in request_data.keys() + if filter not in ['cohort_id', 'fetch_count', 'offset', 'genomic_build'] + } + + response = cohort_files(cohort_id, user=user, inc_filters=inc_filters, **params) + + file_manifest = response['file_list'] if response and response['file_list'] else None + + except Exception as e: + logger.exception(e) + + return file_manifest + + +def get_cohort_info(cohort_id): + cohort = None + try: + cohort_obj = Cohorts.objects.get(id=cohort_id) + cohort = { + 'id': cohort_obj.id, + 'name': cohort_obj.name, + 'case_count': cohort_obj.case_size(), + 'sample_count': cohort_obj.sample_size(), + 'creation_filters': cohort_obj.get_current_filters(), + 'programs': cohort_obj.get_program_names() + } + except ObjectDoesNotExist as e: + logger.warn("Cohort with ID {} was not found!".format(str(cohort_id))) + except Exception as e: + logger.exception(e) + + return cohort + + +def get_cohorts(user_email): + + cohort_list = None + + try: + user = Django_User.objects.get(email=user_email) + cohort_perms = Cohort_Perms.objects.filter(user_id=user.id) + cohort_list = [] + for cohort_perm in cohort_perms: + cohort_list.append({ + 'id': cohort_perm.cohort.id, + 'name': cohort_perm.cohort.name, + 'permission': cohort_perm.perm + }) + + except ObjectDoesNotExist as e: + logger.info("No cohorts found for user {}".format(user_email)) + + return cohort_list diff --git a/main.py b/main.py index fcb8a4a8..b6c7f666 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,6 @@ """ -Copyright 2017, Institute for Systems Biology +Copyright 2019, Institute for Systems Biology Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -16,62 +16,38 @@ """ -import base64 -import json import logging from flask import Flask, jsonify, request from flask_cors import cross_origin +import cohorts.views app = Flask(__name__) -def _base64_decode(encoded_str): - # Add paddings manually if necessary. - num_missed_paddings = 4 - len(encoded_str) % 4 - if num_missed_paddings != 4: - encoded_str += b'=' * num_missed_paddings - return base64.b64decode(encoded_str).decode('utf-8') +@app.route('/', methods['GET', 'POST']) +def base(): + """Base response""" + response = jsonify({ + 'code': 200, + 'message': 'Welcome to the ISB-CGC API, Version 4.' + }) + response.status_code = 200 + return response -# [START endpoints_auth_info_backend] -def auth_info(): - """Retrieves the authenication information from Google Cloud Endpoints.""" - encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None) - - if encoded_info: - info_json = _base64_decode(encoded_info) - user_info = json.loads(info_json) - else: - user_info = {'id': 'anonymous'} - - return jsonify(user_info) -# [START endpoints_auth_info_backend] - - -@app.route('/auth/info/googlejwt', methods=['GET']) -def auth_info_google_jwt(): - """Auth info with Google signed JWT.""" - return auth_info() - - -@app.route('/auth/info/googleidtoken', methods=['GET']) -def auth_info_google_id_token(): - """Auth info with Google ID token.""" - return auth_info() - - -@app.errorhandler(http_client.INTERNAL_SERVER_ERROR) +# Error handlers +@app.errorhandler(500) def unexpected_error(e): """Handle exceptions by returning swagger-compliant json.""" logging.exception('An error occured while processing the request.') response = jsonify({ - 'code': http_client.INTERNAL_SERVER_ERROR, + 'code': 500, 'message': 'Exception: {}'.format(e)}) - response.status_code = http_client.INTERNAL_SERVER_ERROR + response.status_code = 500 return response if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) \ No newline at end of file + app.run(host='127.0.0.1', port=8080, debug=True) From 0a0a0e41ef6af1fe8e9d2f95f6e9cc95aac6c120 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 4 Mar 2019 14:47:49 -0800 Subject: [PATCH 004/323] -> Imports --- cohorts/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cohorts/views.py b/cohorts/views.py index 7442f4ab..b1dc2c5e 100644 --- a/cohorts/views.py +++ b/cohorts/views.py @@ -31,7 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -import auth +from .. import auth logger = logging.getLogger(__name__) From 133af7d11ae4891c5947e4db3e67964ffd122071 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 4 Mar 2019 14:55:09 -0800 Subject: [PATCH 005/323] -> Reqs, Vagrantfile --- Vagrantfile | 19 ++----------------- requirements.txt | 11 +++++++++-- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 4989f824..3f8edac6 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -1,22 +1,8 @@ # -*- mode: ruby -*- # vi: set ft=ruby : Vagrant.configure(2) do |config| - required_plugins = %w( vagrant-vbguest vagrant-disksize ) - _retry = false - required_plugins.each do |plugin| - unless Vagrant.has_plugin? plugin - system "vagrant plugin install #{plugin}" - _retry=true - end - end - - if (_retry) - exec "vagrant " + ARGV.join(' ') - end - - config.vm.box_url = "https://app.vagrantup.com/debian/boxes/stretch64" - config.vm.box = "debian/stretch64" - config.disksize.size = "40GB" + config.vm.box_url = "https://app.vagrantup.com/ubuntu/boxes/xenial64" + config.vm.box = "ubuntu/xenial64" # API ports config.vm.network "forwarded_port", guest: 8090, host: 8090 @@ -27,7 +13,6 @@ Vagrant.configure(2) do |config| # Map Common and lib for API config.vm.synced_folder "../ISB-CGC-Common", "/home/vagrant/API/ISB-CGC-Common" - config.vm.synced_folder "../secure_files", "/home/vagrant/API/secure_files" config.vm.provision "shell", path: 'shell/install-deps.sh' config.vm.provision "shell", path: 'shell/vagrant-start-server.sh' diff --git a/requirements.txt b/requirements.txt index 35dce3ab..66e68881 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -django==1.11.18 +django==1.11.20 mysqlclient==1.3.13 requests==2.20.0 ecdsa==0.13 @@ -10,7 +10,8 @@ pyasn1==0.4.1 requests-oauthlib==0.7.0 rsa==3.2 simplejson==3.8.1 -six==1.11.0 +PyYAML==4.2b1 +six==1.12.0 uritemplate==3.0.0 GoogleAppEngineCloudStorageClient==1.9.22.1 django-dotenv==1.4.2 @@ -18,3 +19,9 @@ django-finalware==1.0.0 django-allauth==0.35.0 jsonschema==2.6.0 enum34==1.1.6 +Flask==1.0.2 +flask-cors==3.0.7 +gunicorn==19.9.0 +pyyaml==3.13 +pyjwt==1.6.1 +cryptography==2.4.2 From 2208863db2b699d79d19d1baf4774ab9b8b83906 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:09:56 -0700 Subject: [PATCH 006/323] -> Shift to using copied .env, remove vars from app.yaml -> Update settings.py -> Use stretch --- .circleci/config.yml | 2 +- settings.py | 327 ++++++++++++++++++----------- shell/gcloud-pull-staging-files.sh | 18 +- shell/install-deps.sh | 26 ++- 4 files changed, 239 insertions(+), 134 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index b1ade0d5..0e487858 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: name: Building Deployment Container working_directory: ~/ISB-CGC-API docker: - - image: circleci/python:2.7.14 + - image: circleci/python:2.7.15-stretch - image: circleci/mysql:5.7 environment: MYSQL_ROOT_HOST: "%" diff --git a/settings.py b/settings.py index f92bf96f..a49a9b8c 100644 --- a/settings.py +++ b/settings.py @@ -1,9 +1,12 @@ """ -Copyright 2017, Institute for Systems Biology +Copyright 2019, Institute for Systems Biology + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,68 +15,97 @@ """ import os +from os.path import join, dirname import sys +import dotenv +from socket import gethostname, gethostbyname -BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY') -DEBUG = os.environ.get('DEBUG') -TEMPLATE_DEBUG = DEBUG +env_path = '../' +if os.environ.get('SECURE_LOCAL_PATH', None): + env_path += os.environ.get('SECURE_LOCAL_PATH') -ALLOWED_HOSTS = [ - os.environ.get('ALLOWED_HOST') -] +dotenv.read_dotenv(join(dirname(__file__), env_path+'.env')) -### Check what we're running in APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' -IS_DEV = (os.environ.get('IS_DEV', 'False') == 'True') -IS_APP_ENGINE_FLEX = os.getenv('GAE_INSTANCE', '').startswith(APP_ENGINE_FLEX) -IS_APP_ENGINE = os.getenv('SERVER_SOFTWARE', '').startswith(APP_ENGINE) -IS_DEV = bool(os.environ.get('IS_DEV', False)) -ADMINS = () -MANAGERS = ADMINS +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep -PROJECT_ID = os.environ.get('PROJECT_ID') -BQ_PROJECT_ID = os.environ.get('BQ_PROJECT_ID') -MAX_BQ_INSERT = int(os.environ.get('MAX_BQ_INSERT', '500')) -BQ_MAX_ATTEMPTS = int(os.environ.get('MAX_BQ_ATTEMPTS', '10')) +SHARED_SOURCE_DIRECTORIES = [ + 'ISB-CGC-Common', +] -USER_DATA_ON = bool(os.environ.get('USER_DATA_ON', 'False') == 'True') +# Add the shared Django application subdirectory to the Python module search path +for directory_name in SHARED_SOURCE_DIRECTORIES: + sys.path.append(os.path.join(BASE_DIR, directory_name)) -MAX_FILE_LIST_REQUEST = int(os.environ.get('MAX_FILE_LIST_REQUEST', '50000')) -MAX_FILES_IGV = int(os.environ.get('MAX_FILES_IGV', '5')) +DEBUG = (os.environ.get('DEBUG', 'False') == 'True') -BASE_URL = os.environ.get('CLOUD_BASE_URL') -BASE_API_URL = os.environ.get('CLOUD_API_URL') +print >> sys.stdout, "[STATUS] DEBUG mode is "+str(DEBUG) -# Compute services -PAIRWISE_SERVICE_URL = os.environ.get('PAIRWISE_SERVICE_URL') +ALLOWED_HOSTS = list(set(os.environ.get('ALLOWED_HOST', 'localhost').split(',') + ['localhost', '127.0.0.1', '[::1]', gethostname(), gethostbyname(gethostname()),])) +# Testing health checks problem +# ALLOWED_HOSTS = ['*'] -# Data Buckets -OPEN_DATA_BUCKET = os.environ.get('OPEN_DATA_BUCKET') -DCC_CONTROLLED_DATA_BUCKET = os.environ.get('DCC_CONTROLLED_DATA_BUCKET') -CGHUB_CONTROLLED_DATA_BUCKET = os.environ.get('CGHUB_CONTROLLED_DATA_BUCKET') +SSL_DIR = os.path.abspath(os.path.dirname(__file__))+os.sep -GCLOUD_BUCKET = os.environ.get('GCLOUD_BUCKET') +ADMINS = () +MANAGERS = ADMINS + +GCLOUD_PROJECT_ID = os.environ.get('GCLOUD_PROJECT_ID', '') +GCLOUD_PROJECT_NUMBER = os.environ.get('GCLOUD_PROJECT_NUMBER', '') +BIGQUERY_PROJECT_ID = os.environ.get('BIGQUERY_PROJECT_ID', GCLOUD_PROJECT_ID) +BIGQUERY_DATASET_V1 = os.environ.get('BIGQUERY_DATASET_V1', '') +BIGQUERY_DATA_PROJECT_ID = os.environ.get('BIGQUERY_DATA_PROJECT_ID', GCLOUD_PROJECT_ID) + +# Deployment module +CRON_MODULE = os.environ.get('CRON_MODULE') + +# Log Names +SERVICE_ACCOUNT_LOG_NAME = os.environ.get('SERVICE_ACCOUNT_LOG_NAME', 'local_dev_logging') +WEBAPP_LOGIN_LOG_NAME = os.environ.get('WEBAPP_LOGIN_LOG_NAME', 'local_dev_logging') +GCP_ACTIVITY_LOG_NAME = os.environ.get('GCP_ACTIVITY_LOG_NAME', 'local_dev_logging') + +BASE_URL = os.environ.get('BASE_URL', 'https://isb-cgc.appspot.com') +BASE_API_URL = os.environ.get('BASE_API_URL', 'https://api-dot-isb-cgc.appspot.com') + +# Compute services - Should not be necessary in webapp +PAIRWISE_SERVICE_URL = os.environ.get('PAIRWISE_SERVICE_URL', None) + +# Data Buckets +OPEN_DATA_BUCKET = os.environ.get('OPEN_DATA_BUCKET', '') +DCC_CONTROLLED_DATA_BUCKET = os.environ.get('DCC_CONTROLLED_DATA_BUCKET', '') +CGHUB_CONTROLLED_DATA_BUCKET = os.environ.get('CGHUB_CONTROLLED_DATA_BUCKET', '') +GCLOUD_BUCKET = os.environ.get('GOOGLE_STORAGE_BUCKET') # BigQuery cohort storage settings -COHORT_DATASET_ID = os.environ.get('COHORT_DATASET_ID') -DEVELOPER_COHORT_TABLE_ID = os.environ.get('DEVELOPER_COHORT_TABLE_ID') +BIGQUERY_COHORT_DATASET_ID = os.environ.get('BIGQUERY_COHORT_DATASET_ID', 'cohort_dataset') +BIGQUERY_COHORT_TABLE_ID = os.environ.get('BIGQUERY_COHORT_TABLE_ID', 'developer_cohorts') +BIGQUERY_COSMIC_DATASET_ID = os.environ.get('BIGQUERY_COSMIC_DATASET_ID', '') +BIGQUERY_CGC_TABLE_ID = os.environ.get('BIGQUERY_CGC_TABLE_ID', '') +MAX_BQ_INSERT = int(os.environ.get('MAX_BQ_INSERT', '500')) -NIH_AUTH_ON = os.environ.get('NIH_AUTH_ON', False) +USER_DATA_ON = bool(os.environ.get('USER_DATA_ON', False)) DATABASES = { 'default': { 'ENGINE': os.environ.get('DATABASE_ENGINE', 'django.db.backends.mysql'), 'HOST': os.environ.get('DATABASE_HOST', '127.0.0.1'), - 'NAME': os.environ.get('DATABASE_NAME', 'test'), + 'NAME': os.environ.get('DATABASE_NAME', ''), 'USER': os.environ.get('DATABASE_USER'), 'PASSWORD': os.environ.get('DATABASE_PASSWORD') } } -if os.environ.has_key('DB_SSL_CERT'): +DB_SOCKET = DATABASES['default']['HOST'] if 'cloudsql' in DATABASES['default']['HOST'] else None + +IS_DEV = (os.environ.get('IS_DEV', 'False') == 'True') +IS_APP_ENGINE_FLEX = os.getenv('GAE_INSTANCE', '').startswith(APP_ENGINE_FLEX) +IS_APP_ENGINE = os.getenv('SERVER_SOFTWARE', '').startswith(APP_ENGINE) + +# If this is a GAE deployment, we don't need to specify SSL; the proxy will take +# care of that for us +if os.environ.has_key('DB_SSL_CERT') and not (IS_APP_ENGINE_FLEX or IS_APP_ENGINE): DATABASES['default']['OPTIONS'] = { 'ssl': { 'ca': os.environ.get('DB_SSL_CA'), @@ -82,8 +114,6 @@ } } -DB_SOCKET = DATABASES['default']['HOST'] if 'cloudsql' in DATABASES['default']['HOST'] else None - # Default to localhost for the site ID SITE_ID = 3 @@ -91,37 +121,48 @@ print >> sys.stdout, "[STATUS] AppEngine detected." SITE_ID = 4 -# For running local unit tests for models -if 'test' in sys.argv: - DATABASES = os.environ.get('TEST_DATABASE') - def get_project_identifier(): - return BQ_PROJECT_ID - -BIGQUERY_DATASET = os.environ.get('BIGQUERY_DATASET') - -def get_bigquery_dataset(): - return BIGQUERY_DATASET - -PROJECT_NAME = os.environ.get('PROJECT_NAME') -BIGQUERY_PROJECT_NAME = os.environ.get('BIGQUERY_PROJECT_NAME') - -def get_bigquery_project_name(): - return BIGQUERY_PROJECT_NAME + return BIGQUERY_PROJECT_ID # Set cohort table here -if DEVELOPER_COHORT_TABLE_ID is None: +if BIGQUERY_COHORT_TABLE_ID is None: raise Exception("Developer-specific cohort table ID is not set.") +BQ_MAX_ATTEMPTS = int(os.environ.get('BQ_MAX_ATTEMPTS', '10')) + + +# TODO Remove duplicate class. +# +# This class is retained here, as it is required by bq_data_access/v1. +# bq_data_access/v2 uses the class from the bq_data_access/bigquery_cohorts module. class BigQueryCohortStorageSettings(object): def __init__(self, dataset_id, table_id): self.dataset_id = dataset_id self.table_id = table_id + def GET_BQ_COHORT_SETTINGS(): - return BigQueryCohortStorageSettings(COHORT_DATASET_ID, DEVELOPER_COHORT_TABLE_ID) + return BigQueryCohortStorageSettings(BIGQUERY_COHORT_DATASET_ID, BIGQUERY_COHORT_TABLE_ID) + +USE_CLOUD_STORAGE = os.environ.get('USE_CLOUD_STORAGE', False) + +PROCESSING_ENABLED = os.environ.get('PROCESSING_ENABLED', False) +PROCESSING_JENKINS_URL = os.environ.get('PROCESSING_JENKINS_URL', 'http://localhost/jenkins') +PROCESSING_JENKINS_PROJECT = os.environ.get('PROCESSING_JENKINS_PROJECT', 'cgc-processing') +PROCESSING_JENKINS_USER = os.environ.get('PROCESSING_JENKINS_USER', 'user') +PROCESSING_JENKINS_PASSWORD = os.environ.get('PROCESSING_JENKINS_PASSWORD', '') + +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + +CSRF_COOKIE_SECURE = bool(os.environ.get('CSRF_COOKIE_SECURE', False)) +SESSION_COOKIE_SECURE = bool(os.environ.get('SESSION_COOKIE_SECURE', False)) +SECURE_SSL_REDIRECT = bool(os.environ.get('SECURE_SSL_REDIRECT', False)) + +SECURE_REDIRECT_EXEMPT = [] -USE_CLOUD_STORAGE = os.environ.get('USE_CLOUD_STORAGE') +if SECURE_SSL_REDIRECT: + # Exempt the health check so it can go through + SECURE_REDIRECT_EXEMPT = [r'^_ah/(vm_)?health$', ] # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name @@ -147,6 +188,17 @@ def GET_BQ_COHORT_SETTINGS(): # If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = True +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_FOLDER = os.environ.get('MEDIA_FOLDER', 'uploads/') +MEDIA_ROOT = os.path.join(os.path.dirname(__file__), '..', '..', MEDIA_FOLDER) +MEDIA_ROOT = os.path.normpath(MEDIA_ROOT) + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '' + # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. @@ -157,6 +209,8 @@ def GET_BQ_COHORT_SETTINGS(): # Example: "http://media.lawrence.com/static/" STATIC_URL = os.environ.get('STATIC_URL', '/static/') +GCS_STORAGE_URI = os.environ.get('GCS_STORAGE_URI', 'https://storage.googleapis.com/') + # Additional locations of static files STATICFILES_DIRS = ( # Put strings here, like "/home/html/static" or "C:/www/django/static". @@ -173,22 +227,24 @@ def GET_BQ_COHORT_SETTINGS(): ) # Make this unique, and don't share it with anybody. -# SECRET_KEY = os.environ.get('SECRET_KEY') +SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY', '') + +SECURE_HSTS_INCLUDE_SUBDOMAINS = (os.environ.get('SECURE_HSTS_INCLUDE_SUBDOMAINS','True') == 'True') +SECURE_HSTS_PRELOAD = (os.environ.get('SECURE_HSTS_PRELOAD','True') == 'True') +SECURE_HSTS_SECONDS = int(os.environ.get('SECURE_HSTS_SECONDS','3600')) -MIDDLEWARE_CLASSES = ( - # For using NDB with Django - # documentation: https://cloud.google.com/appengine/docs/python/ndb/#integration - 'google.appengine.ext.ndb.django_middleware.NdbDjangoMiddleware', - 'google.appengine.ext.appstats.recording.AppStatsDjangoMiddleware', +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', + 'GenespotRE.checkreqsize_middleware.CheckReqSize', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'adminrestrict.middleware.AdminPagesRestrictMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', # Uncomment the next line for simple clickjacking protection: - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - -) + 'django.middleware.clickjacking.XFrameOptionsMiddleware' +] ROOT_URLCONF = 'GenespotRE.urls' @@ -202,18 +258,10 @@ def GET_BQ_COHORT_SETTINGS(): 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', - # 'django.contrib.admin', - # 'django.contrib.admindocs', - # 'GenespotRE', - # 'visualizations', - # 'seqpeek', 'sharing', 'cohorts', 'projects', - # 'genes', - # 'variables', - # 'workbooks', - 'data_upload' + 'data_upload', ) ############################# @@ -221,13 +269,13 @@ def GET_BQ_COHORT_SETTINGS(): ############################# # testing "session security works at the moment" commit -# INSTALLED_APPS += ('session_security',) -# SESSION_SECURITY_WARN_AFTER = 540 -# SESSION_SECURITY_EXPIRE_AFTER = 600 +INSTALLED_APPS += ('session_security',) +SESSION_SECURITY_WARN_AFTER = 540 +SESSION_SECURITY_EXPIRE_AFTER = 600 SESSION_EXPIRE_AT_BROWSER_CLOSE = True -MIDDLEWARE_CLASSES += ( +MIDDLEWARE.append( # for django-session-security -- must go *after* AuthenticationMiddleware - # 'session_security.middleware.SessionSecurityMiddleware', + 'session_security.middleware.SessionSecurityMiddleware', ) ############################### @@ -290,14 +338,29 @@ def GET_BQ_COHORT_SETTINGS(): 'level': 'DEBUG', 'propagate': True, }, - } + 'allauth': { + 'handlers': ['console_dev', 'console_prod'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'google_helpers': { + 'handlers': ['console_dev', 'console_prod'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'data_upload': { + 'handlers': ['console_dev', 'console_prod'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, } ########################## # Start django-allauth # ########################## -LOGIN_REDIRECT_URL = '/dashboard/' +LOGIN_REDIRECT_URL = '/extended_login/' INSTALLED_APPS += ( 'accounts', @@ -319,7 +382,6 @@ def GET_BQ_COHORT_SETTINGS(): 'OPTIONS': { # add any context processors here 'context_processors': ( - 'allauth.socialaccount.context_processors.socialaccount', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', @@ -352,16 +414,23 @@ def GET_BQ_COHORT_SETTINGS(): } } +# Trying to force allauth to only use https +ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'https' +# ...but not if this is a local dev build +if IS_DEV: + ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'http' + ########################## # End django-allauth # ########################## -GOOGLE_APPLICATION_CREDENTIALS = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') -CLIENT_SECRETS = os.environ.get('CLIENT_SECRETS') -CLIENT_EMAIL = os.environ.get('CLIENT_EMAIL') -WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID') -INSTALLED_APP_CLIENT_ID = os.environ.get('INSTALLED_APP_CLIENT_ID') +GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(os.path.dirname(__file__)), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' # Path to privatekey.json +CLIENT_SECRETS = os.path.join(os.path.dirname(os.path.dirname(__file__)), os.environ.get('CLIENT_SECRETS')) if os.environ.get('CLIENT_SECRETS') else '' +WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID', '') # Client ID from client_secrets.json +IGV_WEB_CLIENT_ID = os.environ.get('IGV_WEB_CLIENT_ID', WEB_CLIENT_ID) +INSTALLED_APP_CLIENT_ID = os.environ.get('INSTALLED_APP_CLIENT_ID', '') # Native Client ID +GCP_REG_CLIENT_EMAIL = os.environ.get('CLIENT_EMAIL','') ################################# # For NIH/eRA Commons login # @@ -371,34 +440,6 @@ def GET_BQ_COHORT_SETTINGS(): OPEN_ACL_GOOGLE_GROUP = os.environ.get('OPEN_ACL_GOOGLE_GROUP', '') GOOGLE_GROUP_ADMIN = os.environ.get('GOOGLE_GROUP_ADMIN', '') SUPERADMIN_FOR_REPORTS = os.environ.get('SUPERADMIN_FOR_REPORTS', '') -ERA_LOGIN_URL = os.environ.get('ERA_LOGIN_URL', '') -SAML_FOLDER = os.environ.get('SAML_FOLDER', '') - -###################################### -# For directory, reports services # -###################################### -GOOGLE_GROUP_ADMIN = os.environ.get('GOOGLE_GROUP_ADMIN', '') -SUPERADMIN_FOR_REPORTS = os.environ.get('SUPERADMIN_FOR_REPORTS', '') - -############################## -# Start django-finalware # -############################## - -INSTALLED_APPS += ( - 'finalware',) - -SITE_SUPERUSER_USERNAME = os.environ.get('SU_USER') -SITE_SUPERUSER_EMAIL = '' -SITE_SUPERUSER_PASSWORD = os.environ.get('SU_PASS') - -############################ -# End django-finalware # -############################ - -CONN_MAX_AGE = 0 - -# Deployment module -CRON_MODULE = os.environ.get('CRON_MODULE') # TaskQueue used when users go through the ERA flow LOGOUT_WORKER_TASKQUEUE = os.environ.get('LOGOUT_WORKER_TASKQUEUE', '') @@ -416,9 +457,6 @@ def GET_BQ_COHORT_SETTINGS(): # Log name for ERA login views LOG_NAME_ERA_LOGIN_VIEW = os.environ.get('LOG_NAME_ERA_LOGIN_VIEW', '') -# Log Names -SERVICE_ACCOUNT_LOG_NAME = os.environ.get('SERVICE_ACCOUNT_LOG_NAME', 'local_dev_logging') - # Service account blacklist file path SERVICE_ACCOUNT_BLACKLIST_PATH = os.environ.get('SERVICE_ACCOUNT_BLACKLIST_PATH', '') @@ -431,9 +469,15 @@ def GET_BQ_COHORT_SETTINGS(): # Dataset configuration file path DATASET_CONFIGURATION_PATH = os.environ.get('DATASET_CONFIGURATION_PATH', '') +# DCF Phase I enable flag +DCF_TEST = bool(os.environ.get('DCF_TEST', 'False') == 'True') + # SA via DCF SA_VIA_DCF = bool(os.environ.get('SA_VIA_DCF', 'False') == 'True') +# DCF Monitoring SA +DCF_MONITORING_SA = os.environ.get('DCF_MONITORING_SA', '') + ################################# # For DCF login # ################################# @@ -452,4 +496,47 @@ def GET_BQ_COHORT_SETTINGS(): DCF_GOOGLE_SA_MONITOR_URL = os.environ.get('DCF_GOOGLE_SA_MONITOR_URL', '') DCF_GOOGLE_SA_URL = os.environ.get('DCF_GOOGLE_SA_URL', '') DCF_TOKEN_REFRESH_WINDOW_SECONDS = int(os.environ.get('DCF_TOKEN_REFRESH_WINDOW_SECONDS', 86400)) -DCF_LOGIN_EXPIRATION_SECONDS = int(os.environ.get('DCF_LOGIN_EXPIRATION_SECONDS', 86400)) \ No newline at end of file +DCF_LOGIN_EXPIRATION_SECONDS = int(os.environ.get('DCF_LOGIN_EXPIRATION_SECONDS', 86400)) + +############################## +# Start django-finalware # +############################## + +INSTALLED_APPS += ( + 'finalware',) + +SITE_SUPERUSER_USERNAME = os.environ.get('SUPERUSER_USERNAME', '') +SITE_SUPERUSER_EMAIL = '' +SITE_SUPERUSER_PASSWORD = os.environ.get('SUPERUSER_PASSWORD', '') + +############################ +# End django-finalware # +############################ + +CONN_MAX_AGE = 60 + +############################################################## +# MAXes to prevent size-limited events from causing errors +############################################################## + +# Google App Engine has a response size limit of 32M. ~65k entries from the cohort_filelist view will +# equal just under the 32M limit. If each individual listing is ever lengthened or shortened this +# number should be adjusted +MAX_FILE_LIST_REQUEST = 65000 + +# IGV limit to prevent users from trying ot open dozens of files +MAX_FILES_IGV = 5 + +# Rough max file size to allow for eg. barcode list upload, to revent triggering RequestDataTooBig +FILE_SIZE_UPLOAD_MAX = 1950000 + +############################################################## +# MailGun Email Settings +############################################################## + +EMAIL_SERVICE_API_URL = os.environ.get('EMAIL_SERVICE_API_URL', '') +EMAIL_SERVICE_API_KEY = os.environ.get('EMAIL_SERVICE_API_KEY', '') +NOTIFICATION_EMAIL_FROM_ADDRESS = os.environ.get('NOTIFICATOON_EMAIL_FROM_ADDRESS', '') + +# Explicitly check for known items +BLACKLIST_RE = ur'((?i)|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i))' diff --git a/shell/gcloud-pull-staging-files.sh b/shell/gcloud-pull-staging-files.sh index bc44cd4c..8f2af4a7 100644 --- a/shell/gcloud-pull-staging-files.sh +++ b/shell/gcloud-pull-staging-files.sh @@ -1,16 +1,18 @@ mkdir ./json mkdir ./txt -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${DEV_API_APP_YAML}" ./app.yaml -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${DEV_SECRETS_FILE}" ./client_secrets.json -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${DEV_JSON_FILE}" ./privatekey.json - -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${SERVICE_ACCOUNT_BLACKLIST_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${GOOGLE_ORG_WHITELIST_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${MANAGED_SERVICE_ACCOUNTS_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${DEV_GCLOUD_BUCKET}/${DEV_DATASET_JSON_FILE}" ./ +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_ENV_FILE}" ./.env +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_SECRETS_FILE}" ./client_secrets.json +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_JSON_FILE}" ./privatekey.json +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_USER_GCP_KEY}" ./ +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_DCF_SECRETS_FILE}" ./dcf_secrets.txt +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${SERVICE_ACCOUNT_BLACKLIST_JSON_FILE}" ./ +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${GOOGLE_ORG_WHITELIST_JSON_FILE}" ./ +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${MANAGED_SERVICE_ACCOUNTS_JSON_FILE}" ./ +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_DATASET_JSON_FILE}" ./ # Pack staged files for caching +echo "Packing JSON and text files for caching into deployment..." cp --verbose *.json ./json cp --verbose *.txt ./txt diff --git a/shell/install-deps.sh b/shell/install-deps.sh index c015be47..a4bd5d67 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -3,6 +3,10 @@ if [ -n "$CI" ]; then export HOMEROOT=/home/circleci/${CIRCLE_PROJECT_REPONAME} # Clone dependencies git clone -b master https://github.com/isb-cgc/ISB-CGC-Common.git + + # Remove .pyc files; these can sometimes stick around and if a + # model has changed names it will cause various load failures + find . -type f -name '*.pyc' -delete else export $(cat /home/vagrant/API/.env | grep -v ^# | xargs) 2> /dev/null export HOME=/home/vagrant @@ -14,16 +18,28 @@ export DEBIAN_FRONTEND=noninteractive # Install and update apt-get info echo "Preparing System..." apt-get -y install software-properties-common -wget https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb -apt-get install -y lsb-release -dpkg -i mysql-apt-config_0.8.9-1_all.deb + +if [ -n "$CI" ]; then + # Use these next 4 lines to update mysql public build key + echo 'download mysql public build key' + wget -O - -q 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x8C718D3B5072E1F5' | grep -v '>' | grep -v '<' | grep -v '{' > mysql_pubkey.asc + apt-key add mysql_pubkey.asc || exit 1 + echo 'mysql build key update done.' + wget https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb + apt-get install -y lsb-release + dpkg -i mysql-apt-config_0.8.9-1_all.deb +fi apt-get update -qq # Install apt-get dependencies echo "Installing Dependencies..." -apt-get install -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl dos2unix python3.5 -apt-get install -y --force-yes mysql-client +if [ -n "$CI" ]; then + apt-get install -qq -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python2.7-dev git ruby g++ dos2unix curl + apt-get install -y mysql-client +else + apt-get install -qq -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev mysql-client-5.7 python2.7 python-dev git ruby g++ dos2unix curl +fi echo "Dependencies Installed" # If this is local development, clean out lib for a re-structuring From 856d31449fc6a0913a650a809a66d2d94438cbbb Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:10:43 -0700 Subject: [PATCH 007/323] -> Start storing API app.yaml --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index cb8ba822..d500e575 100644 --- a/.gitignore +++ b/.gitignore @@ -85,7 +85,6 @@ target/ .env *.env scripts/metadata_featdef_tables.sql -app.yaml # dummy datasets *.tsv From 3eebc570419fa0873617e05accf37dd56baa6589 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:11:46 -0700 Subject: [PATCH 008/323] -> Start storing API app.yaml --- app.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 app.yaml diff --git a/app.yaml b/app.yaml new file mode 100644 index 00000000..d4cdb679 --- /dev/null +++ b/app.yaml @@ -0,0 +1,27 @@ +runtime: python +service: api-v4 +env: flex +entrypoint: gunicorn -b :$PORT main:app + +runtime_config: + python_version: 3 + +# [START configuration] +endpoints_api_service: + # The following values are to be replaced by information from the output of + # 'gcloud endpoints services deploy openapi-appengine.yaml' command. + name: api-v4-dot-isb-cgc.appspot.com + rollout_strategy: managed + # [END configuration] + +skip_files: +- ^google_appengine/.*$ +- ^google-cloud-sdk/.*$ +- ^google_appengine\.zip$ +- ^(.*/)?.*\.sql$ + +beta_settings: + cloud_sql_instances: isb-cgc:us-central1:dev-v3 + +network: + instance_tag: isb-cgc-api-v4 From ade132512a4532d95cfac0ff585205503c6b25d2 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:13:42 -0700 Subject: [PATCH 009/323] -> CircleCI config fixes --- .circleci/config.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0e487858..cde44d9d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -51,7 +51,6 @@ jobs: - ./bq_data_access - ./shell - ./ISB-CGC-Common - - ./saml - ./lib - ./appengine_config.py - ./cgc_api.py @@ -65,9 +64,9 @@ jobs: deploy_job: environment: TZ: "/usr/share/zoneinfo/America/Los_Angeles" - name: Deploy to AppEngine Standard + name: Deploy to AppEngine Flex docker: - - image: circleci/python:2.7.14 + - image: circleci/python:2.7.15-stretch working_directory: ~/ISB-CGC-API steps: - restore_cache: @@ -91,4 +90,4 @@ workflows: - build_job filters: branches: - only: master + only: api-v4 From bb4f446f17a3879047c7bc79fdd731035f9d8611 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:16:38 -0700 Subject: [PATCH 010/323] -> Not using Google Endpoints anymore --- appengine_config.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/appengine_config.py b/appengine_config.py index 4c2433ed..5b57f2d1 100644 --- a/appengine_config.py +++ b/appengine_config.py @@ -2,20 +2,9 @@ import sys from google.appengine.ext import vendor -# Per https://github.com/GoogleCloudPlatform/google-cloud-python/issues/1705#issuecomment-209721632 we have to unload -# some GAE-installed libs to make sure our newer versions are used -def unload_module(module_name): - target_modules = [m for m in sys.modules if m.startswith(module_name)] - for m in target_modules: - if m in sys.modules: - del sys.modules[m] - # Add any libraries installed in the "lib" folder. vendor.add('lib') -# The default endpoints/GAE oauth2 is way too old. -unload_module('oauth2client') - BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep SHARED_SOURCE_DIRECTORIES = [ From e381fc41c00ceac31b8518023eb88f0780ad4ea3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:23:38 -0700 Subject: [PATCH 011/323] -> On Flex, need to supply AppEngine lib -> Duplicate (old, busted) PyYAML entry --- app.yaml | 8 +++++++- requirements.txt | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app.yaml b/app.yaml index d4cdb679..6ebfc56f 100644 --- a/app.yaml +++ b/app.yaml @@ -15,10 +15,16 @@ endpoints_api_service: # [END configuration] skip_files: -- ^google_appengine/.*$ +- python-su.sh +- ^(.*/)?#.*#$ +- ^(.*/)?.*~$ +- ^(.*/)?.*\.py[co]$ +- ^(.*/)?.*/RCS/.*$ +- ^(.*/)?\.(?!env).*$ - ^google-cloud-sdk/.*$ - ^google_appengine\.zip$ - ^(.*/)?.*\.sql$ +- ^venv/.*$ beta_settings: cloud_sql_instances: isb-cgc:us-central1:dev-v3 diff --git a/requirements.txt b/requirements.txt index 66e68881..af97c72b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,5 @@ enum34==1.1.6 Flask==1.0.2 flask-cors==3.0.7 gunicorn==19.9.0 -pyyaml==3.13 pyjwt==1.6.1 cryptography==2.4.2 From a46a9ad1be76a780adedd726b58892c740f60a0e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:28:53 -0700 Subject: [PATCH 012/323] -> Actually install Python 3... --- shell/install-deps.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shell/install-deps.sh b/shell/install-deps.sh index a4bd5d67..f56592e4 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -35,10 +35,10 @@ apt-get update -qq # Install apt-get dependencies echo "Installing Dependencies..." if [ -n "$CI" ]; then - apt-get install -qq -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python2.7-dev git ruby g++ dos2unix curl - apt-get install -y mysql-client +apt-get install -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl dos2unix python3.5 +apt-get install -y --force-yes mysql-client else - apt-get install -qq -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev mysql-client-5.7 python2.7 python-dev git ruby g++ dos2unix curl + apt-get install -qq -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl dos2unix python3.5 mysql-client-5.7 fi echo "Dependencies Installed" From 9b2c81b6547d02b9664537827b4ad9b1b470c0b8 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:41:30 -0700 Subject: [PATCH 013/323] -> Force (?) new checksum --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index af97c72b..bfebfbd6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,3 +24,5 @@ flask-cors==3.0.7 gunicorn==19.9.0 pyjwt==1.6.1 cryptography==2.4.2 + + From 2eb442937c4826db27d2d64f9a806502e5332081 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 16:52:13 -0700 Subject: [PATCH 014/323] -> Update caching --- .circleci/config.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cde44d9d..4b0d03da 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,16 +46,13 @@ jobs: - save_cache: key: isb-cgc-api-{{ epoch }} paths: - - ./api - - ./api_3 - - ./bq_data_access + - ./cohorts - ./shell - ./ISB-CGC-Common - ./lib - ./appengine_config.py - - ./cgc_api.py + - ./main.py - ./app.yaml - - ./manage.py - ./settings.py - ./txt - ./json From 20e06579a2be23789879cd3c92ad3eeab66f500e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 17:01:37 -0700 Subject: [PATCH 015/323] -> Left out DJANGO_SETTINGS_MODULE... --- app.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app.yaml b/app.yaml index 6ebfc56f..db1d8555 100644 --- a/app.yaml +++ b/app.yaml @@ -14,6 +14,9 @@ endpoints_api_service: rollout_strategy: managed # [END configuration] +env_variables: + DJANGO_SETTINGS_MODULE: 'GenespotRE.settings' + skip_files: - python-su.sh - ^(.*/)?#.*#$ From 87227527d2ca44cf91c7ae5c84dc7e18e02531c3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 17:02:19 -0700 Subject: [PATCH 016/323] -> ...name it right too... --- app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.yaml b/app.yaml index db1d8555..7e686eac 100644 --- a/app.yaml +++ b/app.yaml @@ -15,7 +15,7 @@ endpoints_api_service: # [END configuration] env_variables: - DJANGO_SETTINGS_MODULE: 'GenespotRE.settings' + DJANGO_SETTINGS_MODULE: 'settings' skip_files: - python-su.sh From f92b3548ca5086c2ee1f33bf037595f67bdafb6a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 17:12:53 -0700 Subject: [PATCH 017/323] -> Regex fix --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index a49a9b8c..2d353678 100644 --- a/settings.py +++ b/settings.py @@ -539,4 +539,4 @@ def GET_BQ_COHORT_SETTINGS(): NOTIFICATION_EMAIL_FROM_ADDRESS = os.environ.get('NOTIFICATOON_EMAIL_FROM_ADDRESS', '') # Explicitly check for known items -BLACKLIST_RE = ur'((?i)|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i))' +BLACKLIST_RE = r'((?i)|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i))' From ea4f1c1794f8c61d583565e82950d9911fc0918f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 17:32:12 -0700 Subject: [PATCH 018/323] -> Swap to Common py3 branch --- shell/install-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/install-deps.sh b/shell/install-deps.sh index f56592e4..20610958 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -2,7 +2,7 @@ if [ -n "$CI" ]; then export HOME=/home/circleci/${CIRCLE_PROJECT_REPONAME} export HOMEROOT=/home/circleci/${CIRCLE_PROJECT_REPONAME} # Clone dependencies - git clone -b master https://github.com/isb-cgc/ISB-CGC-Common.git + git clone -b py-3-testing https://github.com/isb-cgc/ISB-CGC-Common.git # Remove .pyc files; these can sometimes stick around and if a # model has changed names it will cause various load failures From f2344163c38f2afaf7094864424ada58f68245b6 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 17:55:05 -0700 Subject: [PATCH 019/323] -> ...need py3 print --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 2d353678..1bbddabe 100644 --- a/settings.py +++ b/settings.py @@ -41,7 +41,7 @@ DEBUG = (os.environ.get('DEBUG', 'False') == 'True') -print >> sys.stdout, "[STATUS] DEBUG mode is "+str(DEBUG) +print("[STATUS] DEBUG mode is "+str(DEBUG)) ALLOWED_HOSTS = list(set(os.environ.get('ALLOWED_HOST', 'localhost').split(',') + ['localhost', '127.0.0.1', '[::1]', gethostname(), gethostbyname(gethostname()),])) # Testing health checks problem From 89681ab77a81a3756c1629b9645d5dda6bb7a37e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 17:55:48 -0700 Subject: [PATCH 020/323] -> ...need py3 print --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 1bbddabe..5a644763 100644 --- a/settings.py +++ b/settings.py @@ -118,7 +118,7 @@ SITE_ID = 3 if IS_APP_ENGINE_FLEX or IS_APP_ENGINE: - print >> sys.stdout, "[STATUS] AppEngine detected." + print("[STATUS] AppEngine detected.") SITE_ID = 4 def get_project_identifier(): From c7d59354cb0f35f1991f447335a736724b69cc9e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 18:15:12 -0700 Subject: [PATCH 021/323] -> No more SSL cert uplaods, we use the proxy --- settings.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/settings.py b/settings.py index 5a644763..842a9c4a 100644 --- a/settings.py +++ b/settings.py @@ -103,17 +103,6 @@ IS_APP_ENGINE_FLEX = os.getenv('GAE_INSTANCE', '').startswith(APP_ENGINE_FLEX) IS_APP_ENGINE = os.getenv('SERVER_SOFTWARE', '').startswith(APP_ENGINE) -# If this is a GAE deployment, we don't need to specify SSL; the proxy will take -# care of that for us -if os.environ.has_key('DB_SSL_CERT') and not (IS_APP_ENGINE_FLEX or IS_APP_ENGINE): - DATABASES['default']['OPTIONS'] = { - 'ssl': { - 'ca': os.environ.get('DB_SSL_CA'), - 'cert': os.environ.get('DB_SSL_CERT'), - 'key': os.environ.get('DB_SSL_KEY') - } - } - # Default to localhost for the site ID SITE_ID = 3 @@ -121,9 +110,11 @@ print("[STATUS] AppEngine detected.") SITE_ID = 4 + def get_project_identifier(): return BIGQUERY_PROJECT_ID + # Set cohort table here if BIGQUERY_COHORT_TABLE_ID is None: raise Exception("Developer-specific cohort table ID is not set.") From 8deee2c00c7eb4ec2cb9dc93d4f0394d8427b67d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 18:25:46 -0700 Subject: [PATCH 022/323] -> Unlike the webapp, env is in root relative to settings --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 842a9c4a..ba336af2 100644 --- a/settings.py +++ b/settings.py @@ -20,7 +20,7 @@ import dotenv from socket import gethostname, gethostbyname -env_path = '../' +env_path = './' if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') From 3766f2dc976d3fee58e5f1a0990e21afb2a1c0fc Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 18:39:11 -0700 Subject: [PATCH 023/323] -> Unlike the webapp, env is in root relative to settings --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index ba336af2..d7af9eb5 100644 --- a/settings.py +++ b/settings.py @@ -20,7 +20,7 @@ import dotenv from socket import gethostname, gethostbyname -env_path = './' +env_path = '' if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') From 7bb2fe1df13897b48d24cf3d121e0e1c29c73449 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 18:58:07 -0700 Subject: [PATCH 024/323] -> Unlike the webapp, env is in root relative to settings --- app.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/app.yaml b/app.yaml index 7e686eac..eb90710d 100644 --- a/app.yaml +++ b/app.yaml @@ -23,7 +23,6 @@ skip_files: - ^(.*/)?.*~$ - ^(.*/)?.*\.py[co]$ - ^(.*/)?.*/RCS/.*$ -- ^(.*/)?\.(?!env).*$ - ^google-cloud-sdk/.*$ - ^google_appengine\.zip$ - ^(.*/)?.*\.sql$ From a7b00e7c75fd3292a5ede36f55ac0192c0fd525a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 19:16:29 -0700 Subject: [PATCH 025/323] -> Checking dir contents -> Don't need shell directory --- app.yaml | 2 +- settings.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app.yaml b/app.yaml index eb90710d..52b9bf71 100644 --- a/app.yaml +++ b/app.yaml @@ -18,7 +18,7 @@ env_variables: DJANGO_SETTINGS_MODULE: 'settings' skip_files: -- python-su.sh +- ^shell/.*$ - ^(.*/)?#.*#$ - ^(.*/)?.*~$ - ^(.*/)?.*\.py[co]$ diff --git a/settings.py b/settings.py index d7af9eb5..cea6b752 100644 --- a/settings.py +++ b/settings.py @@ -20,6 +20,10 @@ import dotenv from socket import gethostname, gethostbyname + +print("Directory contents: ") +listdir() + env_path = '' if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') From 4665b84d837d81c239054b1ff9f888eee9d8a1c4 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 19:25:12 -0700 Subject: [PATCH 026/323] -> Checking dir contents -> Don't need shell directory --- app.yaml | 1 + settings.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/app.yaml b/app.yaml index 52b9bf71..d6f9da04 100644 --- a/app.yaml +++ b/app.yaml @@ -23,6 +23,7 @@ skip_files: - ^(.*/)?.*~$ - ^(.*/)?.*\.py[co]$ - ^(.*/)?.*/RCS/.*$ +- ^(.*/)?\.(?!env).*$ - ^google-cloud-sdk/.*$ - ^google_appengine\.zip$ - ^(.*/)?.*\.sql$ diff --git a/settings.py b/settings.py index cea6b752..a06b33d1 100644 --- a/settings.py +++ b/settings.py @@ -22,7 +22,7 @@ print("Directory contents: ") -listdir() +os.listdir() env_path = '' if os.environ.get('SECURE_LOCAL_PATH', None): From 2ad70c633bac3fdb555f7e83a19cd77c24a16a26 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 21:46:32 -0700 Subject: [PATCH 027/323] -> Weird issues with .env --- settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/settings.py b/settings.py index a06b33d1..f2930103 100644 --- a/settings.py +++ b/settings.py @@ -22,7 +22,8 @@ print("Directory contents: ") -os.listdir() +os.listdir('./') +os.listdir('/home/vmagent/app/') env_path = '' if os.environ.get('SECURE_LOCAL_PATH', None): From 5bf3edb294a47268ad3851e616131971101de3a5 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 22:05:14 -0700 Subject: [PATCH 028/323] -> Weird issues with .env --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index b6c7f666..b2a89d6c 100644 --- a/main.py +++ b/main.py @@ -20,7 +20,7 @@ from flask import Flask, jsonify, request from flask_cors import cross_origin -import cohorts.views +# import cohorts.views app = Flask(__name__) From b475c3f1026d354607a0a2c0648bda9a067ce4ec Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 22:24:21 -0700 Subject: [PATCH 029/323] -> Weird issues with .env --- appengine_config.py | 3 ++- main.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/appengine_config.py b/appengine_config.py index 5b57f2d1..8a18f042 100644 --- a/appengine_config.py +++ b/appengine_config.py @@ -15,6 +15,7 @@ for path in SHARED_SOURCE_DIRECTORIES: sys.path.append(path) -# Initialize Django (when running ISB-CGC-API as standalone using dev_appserver.py) +print("Checking dir listing in appengine_config:") +os.listdir() import django django.setup() diff --git a/main.py b/main.py index b2a89d6c..941f7712 100644 --- a/main.py +++ b/main.py @@ -25,7 +25,7 @@ app = Flask(__name__) -@app.route('/', methods['GET', 'POST']) +@app.route('/', methods=['GET', 'POST']) def base(): """Base response""" From 9a2e83c991c8caade3ae4219ef41dfc439c82bff Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 23:31:54 -0700 Subject: [PATCH 030/323] -> Add in openapi.yaml --- .circleci/config.yml | 2 + openapi-appengine.yaml | 93 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 openapi-appengine.yaml diff --git a/.circleci/config.yml b/.circleci/config.yml index 4b0d03da..e584ad09 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -53,6 +53,7 @@ jobs: - ./appengine_config.py - ./main.py - ./app.yaml + - ./openapi-appengine.yaml - ./settings.py - ./txt - ./json @@ -76,6 +77,7 @@ jobs: command: | sudo -E /bin/bash ./shell/gcloud_authenticate.sh sudo -E /bin/bash ./shell/unpack_for_deployment.sh + sudo -E ./google-cloud-sdk/bin/gcloud endpoints services deploy openapi-appengine.yaml sudo -E ./google-cloud-sdk/bin/gcloud app deploy --verbosity=debug ./app.yaml --quiet workflows: version: 2 diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml new file mode 100644 index 00000000..68201fa2 --- /dev/null +++ b/openapi-appengine.yaml @@ -0,0 +1,93 @@ +# [START swagger] +swagger: "2.0" +info: + description: "ISB-CGC API Endpoint, version 4" + title: "ISB-CGC API v4" + version: "4.0.0" +host: "api-v4-dot-isb-cgc.appspot.com" +# [END swagger] +consumes: +- "application/json" +produces: +- "application/json" +schemes: +- "https" +paths: + "/": + post: + description: "Basic Response." + operationId: "root" + produces: + - "application/json" + responses: + 200: + description: "Basic Response" + schema: + $ref: "#/definitions/responseMessage" + +definitions: + responseMessage: + type: "object" + properties: + message: + type: "string" + code: + type: "int" +## [START securityDef] +#securityDefinitions: +# # This section configures basic authentication with an API key. +# api_key: +# type: "apiKey" +# name: "key" +# in: "query" +## [END securityDef] +# # This section configures authentication using Google API Service Accounts +# # to sign a json web token. This is mostly used for server-to-server +# # communication. +# google_jwt: +# authorizationUrl: "" +# flow: "implicit" +# type: "oauth2" +# # This must match the 'iss' field in the JWT. +# x-google-issuer: "jwt-client.endpoints.sample.google.com" +# # Update this with your service account's email address. +# x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/jwk/YOUR-SERVICE-ACCOUNT-EMAIL" +# # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. +# x-google-audiences: "echo.endpoints.sample.google.com" +# # This section configures authentication using Google App Engine default +# # service account to sign a json web token. This is mostly used for +# # server-to-server communication. +# gae_default_service_account: +# authorizationUrl: "" +# flow: "implicit" +# type: "oauth2" +# # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. +# x-google-issuer: "YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" +# # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. +# x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" +# # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. +# x-google-audiences: "echo.endpoints.sample.google.com" +# # This section configures authentication using a service account +# # to sign a json web token. This is mostly used for server-to-server +# # communication. +# google_service_account: +# authorizationUrl: "" +# flow: "implicit" +# type: "oauth2" +# # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. +# x-google-issuer: "YOUR-SERVICE-ACCOUNT-EMAIL" +# # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. +# x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-SERVICE-ACCOUNT-EMAIL" +# # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. +# x-google-audiences: "echo.endpoints.sample.google.com" +# # This section configures authentication using Google OAuth2 ID Tokens. +# # ID Tokens can be obtained using OAuth2 clients, and can be used to access +# # your API on behalf of a particular user. +# google_id_token: +# authorizationUrl: "" +# flow: "implicit" +# type: "oauth2" +# x-google-issuer: "https://accounts.google.com" +# x-google-jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" +# # Your OAuth2 client's Client ID must be added here. You can add multiple client IDs to accept tokens form multiple clients. +# x-google-audiences: "YOUR-CLIENT-ID" From df06b09ae2036c8e241c94729a4f53f06f4b1478 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 23:34:55 -0700 Subject: [PATCH 031/323] -> integer, not int --- openapi-appengine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index 68201fa2..e980d0f9 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -32,7 +32,7 @@ definitions: message: type: "string" code: - type: "int" + type: "integer" ## [START securityDef] #securityDefinitions: # # This section configures basic authentication with an API key. From ed86093e90cec591b4b1d99f5822087cb75b9b1f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 13 Mar 2019 23:59:35 -0700 Subject: [PATCH 032/323] -> add get --- main.py | 3 ++- openapi-appengine.yaml | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 941f7712..0d3bd2ca 100644 --- a/main.py +++ b/main.py @@ -17,6 +17,7 @@ """ import logging +import os from flask import Flask, jsonify, request from flask_cors import cross_origin @@ -28,7 +29,7 @@ @app.route('/', methods=['GET', 'POST']) def base(): """Base response""" - + os.listdir('./') response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index e980d0f9..875f1ce5 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -24,6 +24,16 @@ paths: description: "Basic Response" schema: $ref: "#/definitions/responseMessage" + get: + description: "Basic Response." + operationId: "root" + produces: + - "application/json" + responses: + 200: + description: "Basic Response" + schema: + $ref: "#/definitions/responseMessage" definitions: responseMessage: From 33e7f2bbf6800673c2f99400162026620a545e7d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 00:17:30 -0700 Subject: [PATCH 033/323] -> opID collision --- openapi-appengine.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index 875f1ce5..8e1e7dd6 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -16,7 +16,7 @@ paths: "/": post: description: "Basic Response." - operationId: "root" + operationId: "root-post" produces: - "application/json" responses: @@ -26,7 +26,7 @@ paths: $ref: "#/definitions/responseMessage" get: description: "Basic Response." - operationId: "root" + operationId: "root-get" produces: - "application/json" responses: From 0801e8ff2df35e5a11e7771ddcc7afd70c9852d3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 00:55:17 -0700 Subject: [PATCH 034/323] -> Turns out the ESP doesn't support / so that's a thing --- main.py | 2 +- openapi-appengine.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 0d3bd2ca..7c79e068 100644 --- a/main.py +++ b/main.py @@ -26,7 +26,7 @@ app = Flask(__name__) -@app.route('/', methods=['GET', 'POST']) +@app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" os.listdir('./') diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index 8e1e7dd6..ebe1093c 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -13,7 +13,7 @@ produces: schemes: - "https" paths: - "/": + /apiv4: post: description: "Basic Response." operationId: "root-post" From b06c8a4c06297f51a9a509bea1ba21c397c88112 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 01:11:44 -0700 Subject: [PATCH 035/323] -> Checking contents --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index 7c79e068..792165c7 100644 --- a/main.py +++ b/main.py @@ -29,7 +29,8 @@ @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" - os.listdir('./') + logging.info("Directory listing: ") + logging.info(os.listdir('./')) response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' From f38c9b0ebc1c4e478389b1a02432dc9cac1d7486 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 01:34:31 -0700 Subject: [PATCH 036/323] -> Checking contents --- main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 792165c7..0080fe82 100644 --- a/main.py +++ b/main.py @@ -23,14 +23,16 @@ from flask_cors import cross_origin # import cohorts.views +logger = logging.getLogger(__name__) + app = Flask(__name__) @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" - logging.info("Directory listing: ") - logging.info(os.listdir('./')) + logger.info("Directory listing: ") + logger.info(os.listdir('./')) response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' From 7e0b7a72d7320626200e30c860b7478855589d40 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 01:53:28 -0700 Subject: [PATCH 037/323] -> Checking contents --- appengine_config.py | 8 ++++++-- main.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/appengine_config.py b/appengine_config.py index 8a18f042..66775bda 100644 --- a/appengine_config.py +++ b/appengine_config.py @@ -2,6 +2,10 @@ import sys from google.appengine.ext import vendor +import logging + +logger = logging.getLogger(__name__) + # Add any libraries installed in the "lib" folder. vendor.add('lib') @@ -15,7 +19,7 @@ for path in SHARED_SOURCE_DIRECTORIES: sys.path.append(path) -print("Checking dir listing in appengine_config:") -os.listdir() +logger.info("Checking dir listing in appengine_config:") +logger.info(os.listdir()) import django django.setup() diff --git a/main.py b/main.py index 0080fe82..22b47b4f 100644 --- a/main.py +++ b/main.py @@ -33,6 +33,7 @@ def base(): """Base response""" logger.info("Directory listing: ") logger.info(os.listdir('./')) + logger.error("Testing logger") response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' From 2ebab86ffd0b3776acbad73c13930742ea9131e5 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 12:54:53 -0700 Subject: [PATCH 038/323] -> Fun with logging --- main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/main.py b/main.py index 22b47b4f..63634263 100644 --- a/main.py +++ b/main.py @@ -25,6 +25,8 @@ logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + app = Flask(__name__) From 721d42df2a0f026820d5d8894d0f2104f986c408 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 13:11:38 -0700 Subject: [PATCH 039/323] -> Fun with Docker images --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e584ad09..c3cf9bbb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: name: Building Deployment Container working_directory: ~/ISB-CGC-API docker: - - image: circleci/python:2.7.15-stretch + - image: circleci/python:2.7.14 - image: circleci/mysql:5.7 environment: MYSQL_ROOT_HOST: "%" @@ -64,7 +64,7 @@ jobs: TZ: "/usr/share/zoneinfo/America/Los_Angeles" name: Deploy to AppEngine Flex docker: - - image: circleci/python:2.7.15-stretch + - image: circleci/python:2.7.14 working_directory: ~/ISB-CGC-API steps: - restore_cache: From 9e775230ea054d943a1739add795a94a7d2abcd3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 13:13:02 -0700 Subject: [PATCH 040/323] -> Let's use Python 3, like we should be. --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c3cf9bbb..22f37d6c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,7 +6,7 @@ jobs: name: Building Deployment Container working_directory: ~/ISB-CGC-API docker: - - image: circleci/python:2.7.14 + - image: circleci/python:3.6.4 - image: circleci/mysql:5.7 environment: MYSQL_ROOT_HOST: "%" @@ -64,7 +64,7 @@ jobs: TZ: "/usr/share/zoneinfo/America/Los_Angeles" name: Deploy to AppEngine Flex docker: - - image: circleci/python:2.7.14 + - image: circleci/python:3.6.4 working_directory: ~/ISB-CGC-API steps: - restore_cache: From d159964b54f3ee170427ff11b68f61bca992106a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 13:27:29 -0700 Subject: [PATCH 041/323] -> Swap to error --- main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 63634263..0d26a13a 100644 --- a/main.py +++ b/main.py @@ -33,8 +33,8 @@ @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" - logger.info("Directory listing: ") - logger.info(os.listdir('./')) + logger.error("Directory listing: ") + logger.error(os.listdir('./')) logger.error("Testing logger") response = jsonify({ 'code': 200, From 33793e38b9dd72eb37e14b73a73404058bf57b55 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 13:35:46 -0700 Subject: [PATCH 042/323] -> More logging tests --- main.py | 13 ++++++++----- requirements.txt | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/main.py b/main.py index 0d26a13a..22451b94 100644 --- a/main.py +++ b/main.py @@ -17,15 +17,17 @@ """ import logging +from google.cloud import logging import os from flask import Flask, jsonify, request from flask_cors import cross_origin # import cohorts.views -logger = logging.getLogger(__name__) - -logger.setLevel(logging.INFO) +handler = client.get_default_handler() +cloud_logger = logging.getLogger(__name__) +cloud_logger.setLevel(logging.INFO) +cloud_logger.addHandler(handler) app = Flask(__name__) @@ -33,9 +35,10 @@ @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" - logger.error("Directory listing: ") + logger.error("[ERROR] Directory listing: ") logger.error(os.listdir('./')) - logger.error("Testing logger") + logger.info("[INFO] Directory listing: ") + logger.info(os.listdir('./')) response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' diff --git a/requirements.txt b/requirements.txt index bfebfbd6..e75f771d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,5 +24,5 @@ flask-cors==3.0.7 gunicorn==19.9.0 pyjwt==1.6.1 cryptography==2.4.2 - +google-cloud-logging==1.10.0 From 7e6e75c8ed32a6d1b39823ce155b81971b7b1b92 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 14:01:19 -0700 Subject: [PATCH 043/323] -> Enable cohorts, turn on security --- main.py | 3 ++- openapi-appengine.yaml | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index 22451b94..c5704acc 100644 --- a/main.py +++ b/main.py @@ -22,8 +22,9 @@ from flask import Flask, jsonify, request from flask_cors import cross_origin -# import cohorts.views +import cohorts.views +client = logging.Client() handler = client.get_default_handler() cloud_logger = logging.getLogger(__name__) cloud_logger.setLevel(logging.INFO) diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index ebe1093c..8a38d99d 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -35,6 +35,21 @@ paths: schema: $ref: "#/definitions/responseMessage" + /cohorts: + get: + destription: "Get a list of the user's cohorts." + operationId: "getCohorts" + produces: + - "application/json" + responses: + 200: + description: "A list of the user's cohorts." + schema: + $ref: "#/definitions/cohortListResponse" + security: + - google_id_token: [] + + definitions: responseMessage: type: "object" @@ -43,6 +58,12 @@ definitions: type: "string" code: type: "integer" + cohortListResponse: + type: "object" + properties: + data: "string" + code: + type: "integer" ## [START securityDef] #securityDefinitions: # # This section configures basic authentication with an API key. @@ -93,11 +114,11 @@ definitions: # # This section configures authentication using Google OAuth2 ID Tokens. # # ID Tokens can be obtained using OAuth2 clients, and can be used to access # # your API on behalf of a particular user. -# google_id_token: -# authorizationUrl: "" -# flow: "implicit" -# type: "oauth2" -# x-google-issuer: "https://accounts.google.com" -# x-google-jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" -# # Your OAuth2 client's Client ID must be added here. You can add multiple client IDs to accept tokens form multiple clients. -# x-google-audiences: "YOUR-CLIENT-ID" + google_id_token: + authorizationUrl: "" + flow: "implicit" + type: "oauth2" + x-google-issuer: "https://accounts.google.com" + x-google-jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" + # Your OAuth2 client's Client ID must be added here. You can add multiple client IDs to accept tokens form multiple clients. + x-google-audiences: "907668440978-ej0l2i5e9qqb5lt0tge1klh1ht0ci6ea.apps.googleusercontent.com" From 155f786bd4b579b0901c214290a5ef6da8f4e208 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 14:05:08 -0700 Subject: [PATCH 044/323] -> Typo --- openapi-appengine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index 8a38d99d..3bae6848 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -37,7 +37,7 @@ paths: /cohorts: get: - destription: "Get a list of the user's cohorts." + description: "Get a list of the user's cohorts." operationId: "getCohorts" produces: - "application/json" From 348ac1c93b4d160c82467f11a5c825a2ef573a83 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 14:19:05 -0700 Subject: [PATCH 045/323] -> Fleshing out openapi.yaml --- openapi-appengine.yaml | 140 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 125 insertions(+), 15 deletions(-) diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index 3bae6848..dd0e0566 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -24,6 +24,10 @@ paths: description: "Basic Response" schema: $ref: "#/definitions/responseMessage" + 500: + description: "Server error message" + schema: + $ref: "#/definitions/errorMessage" get: description: "Basic Response." operationId: "root-get" @@ -34,21 +38,104 @@ paths: description: "Basic Response" schema: $ref: "#/definitions/responseMessage" + 500: + description: "Server error message" + schema: + $ref: "#/definitions/errorMessage" /cohorts: get: - description: "Get a list of the user's cohorts." - operationId: "getCohorts" - produces: - - "application/json" - responses: - 200: - description: "A list of the user's cohorts." - schema: - $ref: "#/definitions/cohortListResponse" - security: - - google_id_token: [] + description: "Get a list of the user's cohorts." + operationId: "getCohorts" + produces: + - "application/json" + responses: + 200: + description: "A list of the user's cohorts." + schema: + $ref: "#/definitions/cohortListResponse" + 500: + description: "Server error message" + schema: + $ref: "#/definitions/errorMessage" + security: + - google_id_token: [] + + /cohorts/{cohort_id}: + get: + description: "Get information for the specified cohort." + operationId: "getCohort" + produces: + - "application/json" + responses: + 200: + description: "Information about the specified cohort" + schema: + $ref: "#/definitions/cohortResponse" + 500: + description: "Server error message" + schema: + $ref: "#/definitions/errorMessage" + security: + - google_id_token: [] + /cohorts/{cohort_id}/file_manifest: + get: + description: "Retrieve a cohort's file manifest." + operationId: "getCohortFileManifest" + produces: + - "application/json" + parameters: + - name: cohort_id + in: path + description: "Numeric ISB-CGC Cohort ID" + type: integer + required: true + responses: + 200: + description: "File manifest for the specified cohort." + schema: + $ref: "#/definitions/cohortFileManifestResponse" + 500: + description: "Server error message" + schema: + $ref: "#/definitions/errorMessage" + security: + - google_id_token: [] + post: + description: "Retrieve a cohort's file manifest based on filters provided." + operationId: "postCohortFileManifest" + produces: + - "application/json" + parameters: + - name: cohort_id + in: path + description: "Numeric ISB-CGC Cohort ID" + type: integer + required: true + - name: "filters" + in: body + description: "Filters to apply to the file manifest." + schema: + type: object + properties: + program: + type: string + disease_code: + type: string + project: + type: string + responses: + 200: + description: "File manifest for the specified cohort." + schema: + $ref: "#/definitions/cohortFileManifestResponse" + 500: + description: "Server error message" + schema: + $ref: "#/definitions/errorMessage" + security: + - google_id_token: [] definitions: responseMessage: @@ -58,12 +145,35 @@ definitions: type: "string" code: type: "integer" - cohortListResponse: - type: "object" - properties: - data: "string" + cohortListResponse: + type: "object" + properties: + data: + type: "string" + code: + type: "integer" + cohortResponse: + type: "object" + properties: + data: + type: "string" + code: + type: "integer" + cohortFileManifestResponse: + type: "object" + properties: + data: + type: "string" code: type: "integer" + errorMessage: + type: "object" + properties: + message: + type: "string" + code: + type: "integer" + ## [START securityDef] #securityDefinitions: # # This section configures basic authentication with an API key. From c2bdc29d2d5df4e47b58a35c5c730f3fd6be84d1 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 14:21:57 -0700 Subject: [PATCH 046/323] -> Missing security header --- openapi-appengine.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml index dd0e0566..0544e5be 100644 --- a/openapi-appengine.yaml +++ b/openapi-appengine.yaml @@ -175,7 +175,7 @@ definitions: type: "integer" ## [START securityDef] -#securityDefinitions: +securityDefinitions: # # This section configures basic authentication with an API key. # api_key: # type: "apiKey" From cd367681bf758c0778d284307382d13d18171493 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 14:27:26 -0700 Subject: [PATCH 047/323] -> Since it has sensitive information, no more openAPI YAML in GitHub... --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d500e575..a00c6a4c 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,7 @@ target/ *.patch .env *.env +openapi-appengine.yaml scripts/metadata_featdef_tables.sql # dummy datasets From 5be1d6dda162f1c8e55c2c04995ea7e95a8f113a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 15:08:32 -0700 Subject: [PATCH 048/323] -> Since it has sensitive information, no more openAPI YAML in GitHub... --- openapi-appengine.yaml | 234 ----------------------------- shell/gcloud-pull-staging-files.sh | 1 + 2 files changed, 1 insertion(+), 234 deletions(-) delete mode 100644 openapi-appengine.yaml diff --git a/openapi-appengine.yaml b/openapi-appengine.yaml deleted file mode 100644 index 0544e5be..00000000 --- a/openapi-appengine.yaml +++ /dev/null @@ -1,234 +0,0 @@ -# [START swagger] -swagger: "2.0" -info: - description: "ISB-CGC API Endpoint, version 4" - title: "ISB-CGC API v4" - version: "4.0.0" -host: "api-v4-dot-isb-cgc.appspot.com" -# [END swagger] -consumes: -- "application/json" -produces: -- "application/json" -schemes: -- "https" -paths: - /apiv4: - post: - description: "Basic Response." - operationId: "root-post" - produces: - - "application/json" - responses: - 200: - description: "Basic Response" - schema: - $ref: "#/definitions/responseMessage" - 500: - description: "Server error message" - schema: - $ref: "#/definitions/errorMessage" - get: - description: "Basic Response." - operationId: "root-get" - produces: - - "application/json" - responses: - 200: - description: "Basic Response" - schema: - $ref: "#/definitions/responseMessage" - 500: - description: "Server error message" - schema: - $ref: "#/definitions/errorMessage" - - /cohorts: - get: - description: "Get a list of the user's cohorts." - operationId: "getCohorts" - produces: - - "application/json" - responses: - 200: - description: "A list of the user's cohorts." - schema: - $ref: "#/definitions/cohortListResponse" - 500: - description: "Server error message" - schema: - $ref: "#/definitions/errorMessage" - security: - - google_id_token: [] - - /cohorts/{cohort_id}: - get: - description: "Get information for the specified cohort." - operationId: "getCohort" - produces: - - "application/json" - responses: - 200: - description: "Information about the specified cohort" - schema: - $ref: "#/definitions/cohortResponse" - 500: - description: "Server error message" - schema: - $ref: "#/definitions/errorMessage" - security: - - google_id_token: [] - - /cohorts/{cohort_id}/file_manifest: - get: - description: "Retrieve a cohort's file manifest." - operationId: "getCohortFileManifest" - produces: - - "application/json" - parameters: - - name: cohort_id - in: path - description: "Numeric ISB-CGC Cohort ID" - type: integer - required: true - responses: - 200: - description: "File manifest for the specified cohort." - schema: - $ref: "#/definitions/cohortFileManifestResponse" - 500: - description: "Server error message" - schema: - $ref: "#/definitions/errorMessage" - security: - - google_id_token: [] - post: - description: "Retrieve a cohort's file manifest based on filters provided." - operationId: "postCohortFileManifest" - produces: - - "application/json" - parameters: - - name: cohort_id - in: path - description: "Numeric ISB-CGC Cohort ID" - type: integer - required: true - - name: "filters" - in: body - description: "Filters to apply to the file manifest." - schema: - type: object - properties: - program: - type: string - disease_code: - type: string - project: - type: string - responses: - 200: - description: "File manifest for the specified cohort." - schema: - $ref: "#/definitions/cohortFileManifestResponse" - 500: - description: "Server error message" - schema: - $ref: "#/definitions/errorMessage" - security: - - google_id_token: [] - -definitions: - responseMessage: - type: "object" - properties: - message: - type: "string" - code: - type: "integer" - cohortListResponse: - type: "object" - properties: - data: - type: "string" - code: - type: "integer" - cohortResponse: - type: "object" - properties: - data: - type: "string" - code: - type: "integer" - cohortFileManifestResponse: - type: "object" - properties: - data: - type: "string" - code: - type: "integer" - errorMessage: - type: "object" - properties: - message: - type: "string" - code: - type: "integer" - -## [START securityDef] -securityDefinitions: -# # This section configures basic authentication with an API key. -# api_key: -# type: "apiKey" -# name: "key" -# in: "query" -## [END securityDef] -# # This section configures authentication using Google API Service Accounts -# # to sign a json web token. This is mostly used for server-to-server -# # communication. -# google_jwt: -# authorizationUrl: "" -# flow: "implicit" -# type: "oauth2" -# # This must match the 'iss' field in the JWT. -# x-google-issuer: "jwt-client.endpoints.sample.google.com" -# # Update this with your service account's email address. -# x-google-jwks_uri: "https://www.googleapis.com/service_accounts/v1/jwk/YOUR-SERVICE-ACCOUNT-EMAIL" -# # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. -# x-google-audiences: "echo.endpoints.sample.google.com" -# # This section configures authentication using Google App Engine default -# # service account to sign a json web token. This is mostly used for -# # server-to-server communication. -# gae_default_service_account: -# authorizationUrl: "" -# flow: "implicit" -# type: "oauth2" -# # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. -# x-google-issuer: "YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" -# # Replace YOUR-CLIENT-PROJECT-ID with your client project ID. -# x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-CLIENT-PROJECT-ID@appspot.gserviceaccount.com" -# # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. -# x-google-audiences: "echo.endpoints.sample.google.com" -# # This section configures authentication using a service account -# # to sign a json web token. This is mostly used for server-to-server -# # communication. -# google_service_account: -# authorizationUrl: "" -# flow: "implicit" -# type: "oauth2" -# # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. -# x-google-issuer: "YOUR-SERVICE-ACCOUNT-EMAIL" -# # Replace YOUR-SERVICE-ACCOUNT-EMAIL with your service account email. -# x-google-jwks_uri: "https://www.googleapis.com/robot/v1/metadata/x509/YOUR-SERVICE-ACCOUNT-EMAIL" -# # This must match the "aud" field in the JWT. You can add multiple audiences to accept JWTs from multiple clients. -# x-google-audiences: "echo.endpoints.sample.google.com" -# # This section configures authentication using Google OAuth2 ID Tokens. -# # ID Tokens can be obtained using OAuth2 clients, and can be used to access -# # your API on behalf of a particular user. - google_id_token: - authorizationUrl: "" - flow: "implicit" - type: "oauth2" - x-google-issuer: "https://accounts.google.com" - x-google-jwks_uri: "https://www.googleapis.com/oauth2/v3/certs" - # Your OAuth2 client's Client ID must be added here. You can add multiple client IDs to accept tokens form multiple clients. - x-google-audiences: "907668440978-ej0l2i5e9qqb5lt0tge1klh1ht0ci6ea.apps.googleusercontent.com" diff --git a/shell/gcloud-pull-staging-files.sh b/shell/gcloud-pull-staging-files.sh index 8f2af4a7..19151cc8 100644 --- a/shell/gcloud-pull-staging-files.sh +++ b/shell/gcloud-pull-staging-files.sh @@ -2,6 +2,7 @@ mkdir ./json mkdir ./txt ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_ENV_FILE}" ./.env +./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_OPEN_API_YAML}" ./openapi-appengine.yaml ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_SECRETS_FILE}" ./client_secrets.json ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_JSON_FILE}" ./privatekey.json ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET_DEV}/${DEV_USER_GCP_KEY}" ./ From 06131fd4b52c1e068fad504524e40f198f2dfd43 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 15:36:16 -0700 Subject: [PATCH 049/323] -> Logger fixes -> appengine_config.py not being run? --- appengine_config.py | 11 +++++++---- main.py | 18 +++++++++++------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/appengine_config.py b/appengine_config.py index 66775bda..984d1a15 100644 --- a/appengine_config.py +++ b/appengine_config.py @@ -1,10 +1,14 @@ import os import sys from google.appengine.ext import vendor - import logging +from google.cloud import logging as gcloud_logging -logger = logging.getLogger(__name__) +client = gcloud_logging.Client() +handler = client.get_default_handler() +cloud_logger = logging.getLogger(__name__) +cloud_logger.setLevel(logging.INFO) +cloud_logger.addHandler(handler) # Add any libraries installed in the "lib" folder. vendor.add('lib') @@ -19,7 +23,6 @@ for path in SHARED_SOURCE_DIRECTORIES: sys.path.append(path) -logger.info("Checking dir listing in appengine_config:") -logger.info(os.listdir()) +cloud_logger.info("Loading Django...") import django django.setup() diff --git a/main.py b/main.py index c5704acc..57e776d7 100644 --- a/main.py +++ b/main.py @@ -17,29 +17,33 @@ """ import logging -from google.cloud import logging +from google.cloud import logging as gcloud_logging import os from flask import Flask, jsonify, request from flask_cors import cross_origin -import cohorts.views -client = logging.Client() +client = gcloud_logging.Client() handler = client.get_default_handler() cloud_logger = logging.getLogger(__name__) cloud_logger.setLevel(logging.INFO) cloud_logger.addHandler(handler) +import django +django.setup() + +import cohorts.views + app = Flask(__name__) @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" - logger.error("[ERROR] Directory listing: ") - logger.error(os.listdir('./')) - logger.info("[INFO] Directory listing: ") - logger.info(os.listdir('./')) + cloud_logger.error("[ERROR] Directory listing: ") + cloud_logger.error(os.listdir('./')) + cloud_logger.info("[INFO] Directory listing: ") + cloud_logger.info(os.listdir('./')) response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' From 893ad0421c8d5ac987e8fdde19487df13c12add7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 15:58:35 -0700 Subject: [PATCH 050/323] -> Can't find Common... --- main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 57e776d7..14ce6774 100644 --- a/main.py +++ b/main.py @@ -29,6 +29,9 @@ cloud_logger.setLevel(logging.INFO) cloud_logger.addHandler(handler) +cloud_logger.info("[INFO] Directory listing: ") +cloud_logger.info(os.listdir('./')) + import django django.setup() @@ -42,8 +45,6 @@ def base(): """Base response""" cloud_logger.error("[ERROR] Directory listing: ") cloud_logger.error(os.listdir('./')) - cloud_logger.info("[INFO] Directory listing: ") - cloud_logger.info(os.listdir('./')) response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' From 787bb94a8e8339a4767d161e3be7527835b7c12a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 16:11:51 -0700 Subject: [PATCH 051/323] -> I guess appengine_config.py isn't being run at all --- main.py | 8 ++++++++ settings.py | 2 -- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 14ce6774..02d05c30 100644 --- a/main.py +++ b/main.py @@ -32,6 +32,14 @@ cloud_logger.info("[INFO] Directory listing: ") cloud_logger.info(os.listdir('./')) +SHARED_SOURCE_DIRECTORIES = [ + os.path.abspath('./ISB-CGC-Common') +] + +# Add the shared Django application subdirectory to the Python module search path +for path in SHARED_SOURCE_DIRECTORIES: + sys.path.append(path) + import django django.setup() diff --git a/settings.py b/settings.py index f2930103..9f483b33 100644 --- a/settings.py +++ b/settings.py @@ -52,8 +52,6 @@ # Testing health checks problem # ALLOWED_HOSTS = ['*'] -SSL_DIR = os.path.abspath(os.path.dirname(__file__))+os.sep - ADMINS = () MANAGERS = ADMINS From 9d52813ef9ee40d63fcc33d614834fd63d6509db Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 16:14:13 -0700 Subject: [PATCH 052/323] -> Forgot sys --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 02d05c30..d89cb3d1 100644 --- a/main.py +++ b/main.py @@ -19,6 +19,7 @@ import logging from google.cloud import logging as gcloud_logging import os +import sys from flask import Flask, jsonify, request from flask_cors import cross_origin From 3f344a3541e9b2d412d952350f3b3e820d283408 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 16:24:07 -0700 Subject: [PATCH 053/323] -> No need for session_security -> Looks like appengine_config isn't a thing anymore... --- .circleci/config.yml | 1 - appengine_config.py | 28 ---------------------------- settings.py | 14 -------------- 3 files changed, 43 deletions(-) delete mode 100644 appengine_config.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 22f37d6c..9f82ea64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -50,7 +50,6 @@ jobs: - ./shell - ./ISB-CGC-Common - ./lib - - ./appengine_config.py - ./main.py - ./app.yaml - ./openapi-appengine.yaml diff --git a/appengine_config.py b/appengine_config.py deleted file mode 100644 index 984d1a15..00000000 --- a/appengine_config.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys -from google.appengine.ext import vendor -import logging -from google.cloud import logging as gcloud_logging - -client = gcloud_logging.Client() -handler = client.get_default_handler() -cloud_logger = logging.getLogger(__name__) -cloud_logger.setLevel(logging.INFO) -cloud_logger.addHandler(handler) - -# Add any libraries installed in the "lib" folder. -vendor.add('lib') - -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep - -SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common') -] - -# Add the shared Django application subdirectory to the Python module search path -for path in SHARED_SOURCE_DIRECTORIES: - sys.path.append(path) - -cloud_logger.info("Loading Django...") -import django -django.setup() diff --git a/settings.py b/settings.py index 9f483b33..c40c8966 100644 --- a/settings.py +++ b/settings.py @@ -258,20 +258,6 @@ def GET_BQ_COHORT_SETTINGS(): 'data_upload', ) -############################# -# django-session-security # -############################# - -# testing "session security works at the moment" commit -INSTALLED_APPS += ('session_security',) -SESSION_SECURITY_WARN_AFTER = 540 -SESSION_SECURITY_EXPIRE_AFTER = 600 -SESSION_EXPIRE_AT_BROWSER_CLOSE = True -MIDDLEWARE.append( - # for django-session-security -- must go *after* AuthenticationMiddleware - 'session_security.middleware.SessionSecurityMiddleware', -) - ############################### # End django-session-security # ############################### From 3084c4c556fd74b8f822ef6d676273b25f918fdd Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 16:35:24 -0700 Subject: [PATCH 054/323] -> I guess we need future?? --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index e75f771d..19d18265 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,4 +25,5 @@ gunicorn==19.9.0 pyjwt==1.6.1 cryptography==2.4.2 google-cloud-logging==1.10.0 +future==0.17.1 From 6fec0c3e55fd701e3c2a72ff80d1eb68a00cc419 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 16:47:13 -0700 Subject: [PATCH 055/323] -> ...and appengine... --- main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.py b/main.py index d89cb3d1..8ee6d8df 100644 --- a/main.py +++ b/main.py @@ -34,7 +34,8 @@ cloud_logger.info(os.listdir('./')) SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common') + os.path.abspath('./ISB-CGC-Common'), + os.path.abspath('./google_appengine') ] # Add the shared Django application subdirectory to the Python module search path From 2f18df0012db072f7bb3e7b40af4a21712c02aa6 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 17:00:16 -0700 Subject: [PATCH 056/323] -> ...in both... ? --- main.py | 15 +++++++++------ settings.py | 6 +----- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/main.py b/main.py index 8ee6d8df..d4eb4247 100644 --- a/main.py +++ b/main.py @@ -30,17 +30,20 @@ cloud_logger.setLevel(logging.INFO) cloud_logger.addHandler(handler) -cloud_logger.info("[INFO] Directory listing: ") -cloud_logger.info(os.listdir('./')) +print("[PRINT] Directory listing: ") +print(os.listdir('./')) + +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common'), - os.path.abspath('./google_appengine') + 'ISB-CGC-Common', + 'google_appengine' ] # Add the shared Django application subdirectory to the Python module search path -for path in SHARED_SOURCE_DIRECTORIES: - sys.path.append(path) +for directory_name in SHARED_SOURCE_DIRECTORIES: + sys.path.append(os.path.join(BASE_DIR, directory_name)) + import django django.setup() diff --git a/settings.py b/settings.py index c40c8966..7a18d5b7 100644 --- a/settings.py +++ b/settings.py @@ -20,11 +20,6 @@ import dotenv from socket import gethostname, gethostbyname - -print("Directory contents: ") -os.listdir('./') -os.listdir('/home/vmagent/app/') - env_path = '' if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') @@ -38,6 +33,7 @@ SHARED_SOURCE_DIRECTORIES = [ 'ISB-CGC-Common', + 'google_appengine' ] # Add the shared Django application subdirectory to the Python module search path From d071c9219fce86aa4299890bc3ab1ebe1fadeb73 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 17:00:54 -0700 Subject: [PATCH 057/323] -> Let's try settings only --- main.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/main.py b/main.py index d4eb4247..b3e075b8 100644 --- a/main.py +++ b/main.py @@ -30,21 +30,6 @@ cloud_logger.setLevel(logging.INFO) cloud_logger.addHandler(handler) -print("[PRINT] Directory listing: ") -print(os.listdir('./')) - -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep - -SHARED_SOURCE_DIRECTORIES = [ - 'ISB-CGC-Common', - 'google_appengine' -] - -# Add the shared Django application subdirectory to the Python module search path -for directory_name in SHARED_SOURCE_DIRECTORIES: - sys.path.append(os.path.join(BASE_DIR, directory_name)) - - import django django.setup() From e39822a771169d1100a5f66d0d16407e41ca6a0d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 17:25:08 -0700 Subject: [PATCH 058/323] -> ...or not. --- main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/main.py b/main.py index b3e075b8..36fb1ec2 100644 --- a/main.py +++ b/main.py @@ -30,6 +30,17 @@ cloud_logger.setLevel(logging.INFO) cloud_logger.addHandler(handler) +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep + +SHARED_SOURCE_DIRECTORIES = [ + 'ISB-CGC-Common', + 'google_appengine' +] + +# Add the shared Django application subdirectory to the Python module search path +for directory_name in SHARED_SOURCE_DIRECTORIES: + sys.path.append(os.path.join(BASE_DIR, directory_name)) + import django django.setup() From f3181691a24768c2c7864fc5964b8a2d6cb0e244 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 17:35:26 -0700 Subject: [PATCH 059/323] -> The other way, then --- main.py | 10 ++++------ settings.py | 10 ++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index 36fb1ec2..21474363 100644 --- a/main.py +++ b/main.py @@ -30,16 +30,14 @@ cloud_logger.setLevel(logging.INFO) cloud_logger.addHandler(handler) -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep - SHARED_SOURCE_DIRECTORIES = [ - 'ISB-CGC-Common', - 'google_appengine' + os.path.abspath('./ISB-CGC-Common'), + os.path.abspath('./google_appengine') ] # Add the shared Django application subdirectory to the Python module search path -for directory_name in SHARED_SOURCE_DIRECTORIES: - sys.path.append(os.path.join(BASE_DIR, directory_name)) +for path in SHARED_SOURCE_DIRECTORIES: + sys.path.append(path) import django django.setup() diff --git a/settings.py b/settings.py index 7a18d5b7..65894fc3 100644 --- a/settings.py +++ b/settings.py @@ -29,16 +29,14 @@ APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep - SHARED_SOURCE_DIRECTORIES = [ - 'ISB-CGC-Common', - 'google_appengine' + os.path.abspath('./ISB-CGC-Common'), + os.path.abspath('./google_appengine') ] # Add the shared Django application subdirectory to the Python module search path -for directory_name in SHARED_SOURCE_DIRECTORIES: - sys.path.append(os.path.join(BASE_DIR, directory_name)) +for path in SHARED_SOURCE_DIRECTORIES: + sys.path.append(path) DEBUG = (os.environ.get('DEBUG', 'False') == 'True') From ec568f4db13ba63b14987bfd45c54dee926b09e4 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 17:45:43 -0700 Subject: [PATCH 060/323] -> Add base dir back in --- settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/settings.py b/settings.py index 65894fc3..bbd77a89 100644 --- a/settings.py +++ b/settings.py @@ -29,6 +29,8 @@ APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep + SHARED_SOURCE_DIRECTORIES = [ os.path.abspath('./ISB-CGC-Common'), os.path.abspath('./google_appengine') From 7b147daaa08e6bf66c884f5b257b53bcdad5e1f8 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 17:59:10 -0700 Subject: [PATCH 061/323] -> Trouble finding appengine... --- main.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.py b/main.py index 21474363..14791c72 100644 --- a/main.py +++ b/main.py @@ -39,6 +39,9 @@ for path in SHARED_SOURCE_DIRECTORIES: sys.path.append(path) +print(os.listdir()) +print(os.listdir('./google_appengine/')) + import django django.setup() From 4b1c224b909a2f63b04de7f86b8f53eefa25e0f5 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 18:25:47 -0700 Subject: [PATCH 062/323] -> One last attempt before adding in a dockerfile --- appengine_config.py | 23 +++++++++++++++++++++++ settings.py | 3 +++ 2 files changed, 26 insertions(+) create mode 100644 appengine_config.py diff --git a/appengine_config.py b/appengine_config.py new file mode 100644 index 00000000..e109cb03 --- /dev/null +++ b/appengine_config.py @@ -0,0 +1,23 @@ +import os +import sys +from google.appengine.ext import vendor +import logging + +logger = logging.getLogger(__name__) + +print("In appengine_config.py") + +# Add any libraries installed in the "lib" folder. +vendor.add('lib') + +SHARED_SOURCE_DIRECTORIES = [ + os.path.abspath('./ISB-CGC-Common'), + os.path.abspath('./google_appengine') +] + +# Add the shared Django application subdirectory to the Python module search path +for path in SHARED_SOURCE_DIRECTORIES: + sys.path.append(path) + +import django +django.setup() diff --git a/settings.py b/settings.py index bbd77a89..cb33ffa8 100644 --- a/settings.py +++ b/settings.py @@ -516,3 +516,6 @@ def GET_BQ_COHORT_SETTINGS(): # Explicitly check for known items BLACKLIST_RE = r'((?i)|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i))' + +print("Done with settings.py") + From db37c7d167d1c0cb8a06708b8f9b4d5f20a9cb46 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 19:46:56 -0700 Subject: [PATCH 063/323] -> Dockerfile ahoy --- .circleci/config.yml | 1 - Dockerfile | 69 ++++++++++++++++++++++++++++++++++++++++++++ app.yaml | 1 - 3 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 Dockerfile diff --git a/.circleci/config.yml b/.circleci/config.yml index 9f82ea64..8139b51f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,7 +49,6 @@ jobs: - ./cohorts - ./shell - ./ISB-CGC-Common - - ./lib - ./main.py - ./app.yaml - ./openapi-appengine.yaml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..82681fa5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,69 @@ +### +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +### + +# Dockerfile extending the generic Python image with application files for a +# single application. +FROM gcr.io/google_appengine/python + +RUN apt-get update +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get install -y wget +RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.9-1_all.deb" -P /tmp + +# install lsb-release (a dependency of mysql-apt-config), since dpkg doesn't +# do dependency resolution +RUN apt-get install -y lsb-release +# add a debconf entry to select mysql-5.7 as the server version when we install +# the mysql config package +RUN echo "mysql-apt-config mysql-apt-config/select-server select mysql-5.7" | debconf-set-selections +# having 'selected' mysql-5.7 for 'server', install the mysql config package +RUN echo 'download mysql public build key' +RUN apt-key del 1550412832 +RUN wget -O - -q 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x8C718D3B5072E1F5' | grep -v '>' | grep -v '<' | grep -v '{' > mysql_pubkey.asc +RUN apt-key add mysql_pubkey.asc || exit 1 +RUN dpkg --install /tmp/mysql-apt-config_0.8.9-1_all.deb + +# fetch the updated package metadata (in particular, mysql-server-5.7) +RUN apt-get update + +# aaaand now let's install mysql-server +RUN apt-get install -y mysql-server + +# Get pip3 installed +RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3 + +RUN apt-get -y install build-essential +RUN apt-get -y install --reinstall python-m2crypto python3-crypto +RUN apt-get -y install libxml2-dev libxmlsec1-dev swig +RUN pip3 install pexpect + +RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl +RUN easy_install -U distribute + +ADD . /app + +# We need to recompile some of the items because of differences in compiler versions +RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade + +ENV PYTHONPATH=/app:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/google_appengine/lib/protorpc-1.0 + +# Until we figure out a way to do it in CircleCI without whitelisting IPs this has to be done by a dev from +# ISB +# RUN python /app/manage.py migrate --noinput + +CMD gunicorn -c gunicorn.conf.py -b :$PORT app:main -w 3 -t 130 diff --git a/app.yaml b/app.yaml index d6f9da04..62763260 100644 --- a/app.yaml +++ b/app.yaml @@ -1,7 +1,6 @@ runtime: python service: api-v4 env: flex -entrypoint: gunicorn -b :$PORT main:app runtime_config: python_version: 3 From 8e6028ca01795c8ec65f230a3fa8fc80b7c4cc9d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 19:51:53 -0700 Subject: [PATCH 064/323] -> Dockerfile ahoy --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8139b51f..513fee4a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -49,6 +49,7 @@ jobs: - ./cohorts - ./shell - ./ISB-CGC-Common + - ./Dockerfile - ./main.py - ./app.yaml - ./openapi-appengine.yaml From b4f325074d57eb5268dd2aa29a59633c5ebe92e4 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 19:55:12 -0700 Subject: [PATCH 065/323] -> Dockerfile ahoy --- app.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/app.yaml b/app.yaml index 62763260..d28defce 100644 --- a/app.yaml +++ b/app.yaml @@ -1,9 +1,8 @@ -runtime: python +runtime: custom service: api-v4 env: flex - -runtime_config: - python_version: 3 +api_version: 1 +threadsafe: true # [START configuration] endpoints_api_service: From ae803ccd7195a3bae25cc40de3ac248dbed8b341 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 20:12:47 -0700 Subject: [PATCH 066/323] -> No gunicorn conf this time --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 82681fa5..b34b5311 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,4 +66,4 @@ ENV PYTHONPATH=/app:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/goog # ISB # RUN python /app/manage.py migrate --noinput -CMD gunicorn -c gunicorn.conf.py -b :$PORT app:main -w 3 -t 130 +CMD gunicorn -b :$PORT app:main -w 3 -t 130 From 712e5a5e19a6bc132d1b357e3bc6aff86fe77f56 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 20:26:13 -0700 Subject: [PATCH 067/323] -> Move gunicorn install out of lib --- Dockerfile | 1 + requirements.txt | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b34b5311..168b00b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -59,6 +59,7 @@ ADD . /app # We need to recompile some of the items because of differences in compiler versions RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade +RUN pip3 install gunicorn==19.9.0 ENV PYTHONPATH=/app:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/google_appengine/lib/protorpc-1.0 diff --git a/requirements.txt b/requirements.txt index 19d18265..2fd40a28 100644 --- a/requirements.txt +++ b/requirements.txt @@ -21,7 +21,6 @@ jsonschema==2.6.0 enum34==1.1.6 Flask==1.0.2 flask-cors==3.0.7 -gunicorn==19.9.0 pyjwt==1.6.1 cryptography==2.4.2 google-cloud-logging==1.10.0 From 1d3857e2baad293f6d41da7fe726e83cfacc7170 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 20:41:23 -0700 Subject: [PATCH 068/323] -> No enum34 --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2fd40a28..984a3ad3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,7 +18,6 @@ django-dotenv==1.4.2 django-finalware==1.0.0 django-allauth==0.35.0 jsonschema==2.6.0 -enum34==1.1.6 Flask==1.0.2 flask-cors==3.0.7 pyjwt==1.6.1 From 04cd5c947fd4ac1eb3a09935041e9dfd689ea5b6 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 21:11:18 -0700 Subject: [PATCH 069/323] -> main:app, not app:main --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 168b00b9..3f536da7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -67,4 +67,4 @@ ENV PYTHONPATH=/app:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/goog # ISB # RUN python /app/manage.py migrate --noinput -CMD gunicorn -b :$PORT app:main -w 3 -t 130 +CMD gunicorn -b :$PORT main:app -w 3 -t 130 From 1469916eaf0d0699782dc41d7b8c025bec19bb42 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 21:48:30 -0700 Subject: [PATCH 070/323] -> Cloud logging issue... --- main.py | 2 +- requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/main.py b/main.py index 14791c72..0f5e22aa 100644 --- a/main.py +++ b/main.py @@ -17,7 +17,7 @@ """ import logging -from google.cloud import logging as gcloud_logging +import google.cloud.logging as gcloud_logging import os import sys diff --git a/requirements.txt b/requirements.txt index 984a3ad3..7dd37325 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,4 +24,3 @@ pyjwt==1.6.1 cryptography==2.4.2 google-cloud-logging==1.10.0 future==0.17.1 - From 0444a0bec835a6e75063e83153a6d1bc8bd4c933 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 22:05:25 -0700 Subject: [PATCH 071/323] -> Suspect google.appengine and google.cloud are NOT compatible --- cohorts/views.py | 2 +- main.py | 7 +------ requirements.txt | 1 - 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/cohorts/views.py b/cohorts/views.py index b1dc2c5e..1fe8ba91 100644 --- a/cohorts/views.py +++ b/cohorts/views.py @@ -34,7 +34,7 @@ from .. import auth -logger = logging.getLogger(__name__) +logger = logging.getLogger('main_logger') def validate_user(user_email, cohort_id): diff --git a/main.py b/main.py index 0f5e22aa..06586d2c 100644 --- a/main.py +++ b/main.py @@ -17,18 +17,13 @@ """ import logging -import google.cloud.logging as gcloud_logging import os import sys from flask import Flask, jsonify, request from flask_cors import cross_origin -client = gcloud_logging.Client() -handler = client.get_default_handler() -cloud_logger = logging.getLogger(__name__) -cloud_logger.setLevel(logging.INFO) -cloud_logger.addHandler(handler) +logger = logging.getLogger('main_logger') SHARED_SOURCE_DIRECTORIES = [ os.path.abspath('./ISB-CGC-Common'), diff --git a/requirements.txt b/requirements.txt index 7dd37325..048a4ef3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,5 +22,4 @@ Flask==1.0.2 flask-cors==3.0.7 pyjwt==1.6.1 cryptography==2.4.2 -google-cloud-logging==1.10.0 future==0.17.1 From dffbf31e069fb126ce5425c9b416d35f2ed71a22 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 22:24:01 -0700 Subject: [PATCH 072/323] -> Can't find credentials? --- settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/settings.py b/settings.py index cb33ffa8..4a18753f 100644 --- a/settings.py +++ b/settings.py @@ -402,6 +402,7 @@ def GET_BQ_COHORT_SETTINGS(): ########################## GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(os.path.dirname(__file__)), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' # Path to privatekey.json +print("Google App Creds location: "+GOOGLE_APPLICATION_CREDENTIALS) CLIENT_SECRETS = os.path.join(os.path.dirname(os.path.dirname(__file__)), os.environ.get('CLIENT_SECRETS')) if os.environ.get('CLIENT_SECRETS') else '' WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID', '') # Client ID from client_secrets.json IGV_WEB_CLIENT_ID = os.environ.get('IGV_WEB_CLIENT_ID', WEB_CLIENT_ID) From 918ca9594bb3aaef3a809ef4bb7aaba5531a971f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 22:37:17 -0700 Subject: [PATCH 073/323] -> One too many steps back --- settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/settings.py b/settings.py index 4a18753f..ee21f6aa 100644 --- a/settings.py +++ b/settings.py @@ -401,9 +401,9 @@ def GET_BQ_COHORT_SETTINGS(): # End django-allauth # ########################## -GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(os.path.dirname(__file__)), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' # Path to privatekey.json +GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(__file__), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' # Path to privatekey.json print("Google App Creds location: "+GOOGLE_APPLICATION_CREDENTIALS) -CLIENT_SECRETS = os.path.join(os.path.dirname(os.path.dirname(__file__)), os.environ.get('CLIENT_SECRETS')) if os.environ.get('CLIENT_SECRETS') else '' +CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), os.environ.get('CLIENT_SECRETS')) if os.environ.get('CLIENT_SECRETS') else '' WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID', '') # Client ID from client_secrets.json IGV_WEB_CLIENT_ID = os.environ.get('IGV_WEB_CLIENT_ID', WEB_CLIENT_ID) INSTALLED_APP_CLIENT_ID = os.environ.get('INSTALLED_APP_CLIENT_ID', '') # Native Client ID From 3b36770a3f78eeec3355a65287a55a18c26b119c Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 23:41:56 -0700 Subject: [PATCH 074/323] -> ...still can't find ISB-CGC-Common... --- cohorts/views.py | 9 +++++++++ main.py | 3 --- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cohorts/views.py b/cohorts/views.py index 1fe8ba91..c7851a72 100644 --- a/cohorts/views.py +++ b/cohorts/views.py @@ -27,6 +27,15 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.conf import settings +SHARED_SOURCE_DIRECTORIES = [ + os.path.abspath('./ISB-CGC-Common'), + os.path.abspath('./google_appengine') +] + +# Add the shared Django application subdirectory to the Python module search path +for path in SHARED_SOURCE_DIRECTORIES: + sys.path.append(path) + from cohorts.models import Cohort_Perms, Cohort, Filters from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files diff --git a/main.py b/main.py index 06586d2c..a17da623 100644 --- a/main.py +++ b/main.py @@ -34,9 +34,6 @@ for path in SHARED_SOURCE_DIRECTORIES: sys.path.append(path) -print(os.listdir()) -print(os.listdir('./google_appengine/')) - import django django.setup() From ebf2c642fbc6d0e3c75ce68c87bf6517abbd6854 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 14 Mar 2019 23:44:12 -0700 Subject: [PATCH 075/323] -> Rename module so it doesn't conflict --- {cohorts => cohort}/__init__.py | 0 {cohorts => cohort}/routes.py | 0 {cohorts => cohort}/views.py | 9 --------- main.py | 2 +- 4 files changed, 1 insertion(+), 10 deletions(-) rename {cohorts => cohort}/__init__.py (100%) rename {cohorts => cohort}/routes.py (100%) rename {cohorts => cohort}/views.py (93%) diff --git a/cohorts/__init__.py b/cohort/__init__.py similarity index 100% rename from cohorts/__init__.py rename to cohort/__init__.py diff --git a/cohorts/routes.py b/cohort/routes.py similarity index 100% rename from cohorts/routes.py rename to cohort/routes.py diff --git a/cohorts/views.py b/cohort/views.py similarity index 93% rename from cohorts/views.py rename to cohort/views.py index c7851a72..1fe8ba91 100644 --- a/cohorts/views.py +++ b/cohort/views.py @@ -27,15 +27,6 @@ from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from django.conf import settings -SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common'), - os.path.abspath('./google_appengine') -] - -# Add the shared Django application subdirectory to the Python module search path -for path in SHARED_SOURCE_DIRECTORIES: - sys.path.append(path) - from cohorts.models import Cohort_Perms, Cohort, Filters from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files diff --git a/main.py b/main.py index a17da623..365caf78 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ import django django.setup() -import cohorts.views +import cohort.views app = Flask(__name__) From 94effe3f1485bff2f509ee86d4db81e1224065c1 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 00:21:43 -0700 Subject: [PATCH 076/323] -> Rename module so it doesn't conflict --- {cohort => api_cohort}/__init__.py | 0 {cohort => api_cohort}/routes.py | 0 {cohort => api_cohort}/views.py | 0 main.py | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) rename {cohort => api_cohort}/__init__.py (100%) rename {cohort => api_cohort}/routes.py (100%) rename {cohort => api_cohort}/views.py (100%) diff --git a/cohort/__init__.py b/api_cohort/__init__.py similarity index 100% rename from cohort/__init__.py rename to api_cohort/__init__.py diff --git a/cohort/routes.py b/api_cohort/routes.py similarity index 100% rename from cohort/routes.py rename to api_cohort/routes.py diff --git a/cohort/views.py b/api_cohort/views.py similarity index 100% rename from cohort/views.py rename to api_cohort/views.py diff --git a/main.py b/main.py index 365caf78..d3f76275 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ import django django.setup() -import cohort.views +import api_cohort.views app = Flask(__name__) From 7b14c23b5c0dd9b8b2629a9f618c8ce07d105cac Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 00:56:17 -0700 Subject: [PATCH 077/323] -> Fun with imports --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index d3f76275..61df4e01 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ import django django.setup() -import api_cohort.views +from .api_cohort import views app = Flask(__name__) From 84fce79e409ed4b385210d2016511be089fbee1c Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 01:07:55 -0700 Subject: [PATCH 078/323] -> Fun with imports --- api_cohort/routes.py | 2 ++ main.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api_cohort/routes.py b/api_cohort/routes.py index c24390fd..2963b829 100644 --- a/api_cohort/routes.py +++ b/api_cohort/routes.py @@ -20,6 +20,8 @@ import json from flask import jsonify, request +from ..main import app + from . import views diff --git a/main.py b/main.py index 61df4e01..01ed13e3 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ import django django.setup() -from .api_cohort import views +from .api_cohort import routes app = Flask(__name__) From 2f69b7a9f83e334e407709438a2a0f422054d1ae Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 01:28:18 -0700 Subject: [PATCH 079/323] -> Fun with imports --- Dockerfile | 10 ++++++++++ api_auth/__init__.py | 0 auth.py => api_auth/auth.py | 0 api_cohort/views.py | 2 +- main.py | 2 +- 5 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 api_auth/__init__.py rename auth.py => api_auth/auth.py (100%) diff --git a/Dockerfile b/Dockerfile index 3f536da7..26d0a779 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,6 +20,16 @@ # single application. FROM gcr.io/google_appengine/python +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +# Use -p python3 or -p python3.7 to select python version. Default is version 2. +RUN virtualenv /env + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + RUN apt-get update ENV DEBIAN_FRONTEND=noninteractive RUN apt-get install -y wget diff --git a/api_auth/__init__.py b/api_auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/auth.py b/api_auth/auth.py similarity index 100% rename from auth.py rename to api_auth/auth.py diff --git a/api_cohort/views.py b/api_cohort/views.py index 1fe8ba91..21adb357 100644 --- a/api_cohort/views.py +++ b/api_cohort/views.py @@ -31,7 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -from .. import auth +from ..api_auth.auth import auth_info logger = logging.getLogger('main_logger') diff --git a/main.py b/main.py index 01ed13e3..5765f6e5 100644 --- a/main.py +++ b/main.py @@ -37,7 +37,7 @@ import django django.setup() -from .api_cohort import routes +import api_cohort.routes app = Flask(__name__) From 43d868a4f87dbafb2cc4efd38af46cc08d9466ee Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 01:28:47 -0700 Subject: [PATCH 080/323] -> Fun with imports --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 26d0a779..591d46c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,7 +23,7 @@ FROM gcr.io/google_appengine/python # Create a virtualenv for dependencies. This isolates these packages from # system-level packages. # Use -p python3 or -p python3.7 to select python version. Default is version 2. -RUN virtualenv /env +RUN virtualenv /env -p python3 # Setting these environment variables are the same as running # source /env/bin/activate. From 27b17e55c8379edebd4651ee25df8fb1e86425e9 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 01:52:38 -0700 Subject: [PATCH 081/323] -> Testing a theory... --- main.py | 9 --------- settings.py | 9 --------- 2 files changed, 18 deletions(-) diff --git a/main.py b/main.py index 5765f6e5..5e8200ed 100644 --- a/main.py +++ b/main.py @@ -25,15 +25,6 @@ logger = logging.getLogger('main_logger') -SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common'), - os.path.abspath('./google_appengine') -] - -# Add the shared Django application subdirectory to the Python module search path -for path in SHARED_SOURCE_DIRECTORIES: - sys.path.append(path) - import django django.setup() diff --git a/settings.py b/settings.py index ee21f6aa..3f22025f 100644 --- a/settings.py +++ b/settings.py @@ -31,15 +31,6 @@ BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep -SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common'), - os.path.abspath('./google_appengine') -] - -# Add the shared Django application subdirectory to the Python module search path -for path in SHARED_SOURCE_DIRECTORIES: - sys.path.append(path) - DEBUG = (os.environ.get('DEBUG', 'False') == 'True') print("[STATUS] DEBUG mode is "+str(DEBUG)) From 64ccb5f14275fa0fce7e49e701319af33f1b14e3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 02:08:29 -0700 Subject: [PATCH 082/323] -> Don't forget to add them to the cache --- .circleci/config.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 513fee4a..cee5cac2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,7 +46,8 @@ jobs: - save_cache: key: isb-cgc-api-{{ epoch }} paths: - - ./cohorts + - ./api_cohort + - ./api_auth - ./shell - ./ISB-CGC-Common - ./Dockerfile From 28797f119641b2ae263af0688b477e41aafdb652 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 02:33:16 -0700 Subject: [PATCH 083/323] -> New structure --- .circleci/config.yml | 4 +--- Dockerfile | 2 +- main.py => apiv4/__init__.py | 0 {api_auth => apiv4/api_auth}/__init__.py | 0 {api_auth => apiv4/api_auth}/auth.py | 0 {api_cohort => apiv4/api_cohort}/__init__.py | 0 {api_cohort => apiv4/api_cohort}/routes.py | 2 +- {api_cohort => apiv4/api_cohort}/views.py | 0 8 files changed, 3 insertions(+), 5 deletions(-) rename main.py => apiv4/__init__.py (100%) rename {api_auth => apiv4/api_auth}/__init__.py (100%) rename {api_auth => apiv4/api_auth}/auth.py (100%) rename {api_cohort => apiv4/api_cohort}/__init__.py (100%) rename {api_cohort => apiv4/api_cohort}/routes.py (99%) rename {api_cohort => apiv4/api_cohort}/views.py (100%) diff --git a/.circleci/config.yml b/.circleci/config.yml index cee5cac2..5a34d28c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -46,12 +46,10 @@ jobs: - save_cache: key: isb-cgc-api-{{ epoch }} paths: - - ./api_cohort - - ./api_auth + - ./apiv4 - ./shell - ./ISB-CGC-Common - ./Dockerfile - - ./main.py - ./app.yaml - ./openapi-appengine.yaml - ./settings.py diff --git a/Dockerfile b/Dockerfile index 591d46c6..cd0f9215 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,4 +77,4 @@ ENV PYTHONPATH=/app:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/goog # ISB # RUN python /app/manage.py migrate --noinput -CMD gunicorn -b :$PORT main:app -w 3 -t 130 +CMD gunicorn -b :$PORT apiv4:app -w 3 -t 130 diff --git a/main.py b/apiv4/__init__.py similarity index 100% rename from main.py rename to apiv4/__init__.py diff --git a/api_auth/__init__.py b/apiv4/api_auth/__init__.py similarity index 100% rename from api_auth/__init__.py rename to apiv4/api_auth/__init__.py diff --git a/api_auth/auth.py b/apiv4/api_auth/auth.py similarity index 100% rename from api_auth/auth.py rename to apiv4/api_auth/auth.py diff --git a/api_cohort/__init__.py b/apiv4/api_cohort/__init__.py similarity index 100% rename from api_cohort/__init__.py rename to apiv4/api_cohort/__init__.py diff --git a/api_cohort/routes.py b/apiv4/api_cohort/routes.py similarity index 99% rename from api_cohort/routes.py rename to apiv4/api_cohort/routes.py index 2963b829..71c6521f 100644 --- a/api_cohort/routes.py +++ b/apiv4/api_cohort/routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request -from ..main import app +from apiv4 import app from . import views diff --git a/api_cohort/views.py b/apiv4/api_cohort/views.py similarity index 100% rename from api_cohort/views.py rename to apiv4/api_cohort/views.py From e281ba5be114bab8b8bffb16f3db79bc3d027431 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 02:52:44 -0700 Subject: [PATCH 084/323] -> New structure --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index cd0f9215..e10fa5cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -71,7 +71,7 @@ ADD . /app RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade RUN pip3 install gunicorn==19.9.0 -ENV PYTHONPATH=/app:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/google_appengine/lib/protorpc-1.0 +ENV PYTHONPATH=/app:/app/apiv4:/app/lib:/app/ISB-CGC-Common:/app/google_appengine:/app/google_appengine/lib/protorpc-1.0 # Until we figure out a way to do it in CircleCI without whitelisting IPs this has to be done by a dev from # ISB From 551e380982d8ff0ffb3106e1786a77afa5bb74ee Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 03:09:14 -0700 Subject: [PATCH 085/323] -> New structure --- apiv4/api_cohort/routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/api_cohort/routes.py b/apiv4/api_cohort/routes.py index 71c6521f..10454940 100644 --- a/apiv4/api_cohort/routes.py +++ b/apiv4/api_cohort/routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request -from apiv4 import app +from .. import app from . import views From 51474298cccb2be86c4812d1a6b3b47f06bf2721 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 03:36:19 -0700 Subject: [PATCH 086/323] -> New structure --- apiv4/api_auth/__init__.py | 0 apiv4/api_cohort/__init__.py | 0 apiv4/{api_auth => }/auth.py | 0 apiv4/{api_cohort/routes.py => cohorts_routes.py} | 4 ++-- apiv4/{api_cohort/views.py => cohorts_views.py} | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 apiv4/api_auth/__init__.py delete mode 100644 apiv4/api_cohort/__init__.py rename apiv4/{api_auth => }/auth.py (100%) rename apiv4/{api_cohort/routes.py => cohorts_routes.py} (97%) rename apiv4/{api_cohort/views.py => cohorts_views.py} (99%) diff --git a/apiv4/api_auth/__init__.py b/apiv4/api_auth/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apiv4/api_cohort/__init__.py b/apiv4/api_cohort/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/apiv4/api_auth/auth.py b/apiv4/auth.py similarity index 100% rename from apiv4/api_auth/auth.py rename to apiv4/auth.py diff --git a/apiv4/api_cohort/routes.py b/apiv4/cohorts_routes.py similarity index 97% rename from apiv4/api_cohort/routes.py rename to apiv4/cohorts_routes.py index 10454940..193f29c4 100644 --- a/apiv4/api_cohort/routes.py +++ b/apiv4/cohorts_routes.py @@ -20,9 +20,9 @@ import json from flask import jsonify, request -from .. import app +from . import app -from . import views +from .cohorts_views import get_cohort_info, get_cohorts, get_file_manifest @app.route('/cohorts/', methods=['GET']) diff --git a/apiv4/api_cohort/views.py b/apiv4/cohorts_views.py similarity index 99% rename from apiv4/api_cohort/views.py rename to apiv4/cohorts_views.py index 21adb357..7198c4b3 100644 --- a/apiv4/api_cohort/views.py +++ b/apiv4/cohorts_views.py @@ -31,7 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -from ..api_auth.auth import auth_info +from .auth import auth_info logger = logging.getLogger('main_logger') From fc7b6625d3f2c6feae70bd586c857e7a8b3f4f45 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 03:39:21 -0700 Subject: [PATCH 087/323] -> New structure --- apiv4/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 5e8200ed..38110e1e 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -28,7 +28,7 @@ import django django.setup() -import api_cohort.routes +from .cohorts_routes import * app = Flask(__name__) From 845a4a844cb55f291e5cdee0c7a8377e098f7fec Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 04:24:26 -0700 Subject: [PATCH 088/323] -> New structure --- apiv4/cohorts_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 193f29c4..2b16c7bc 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request -from . import app +from apiv4 import app from .cohorts_views import get_cohort_info, get_cohorts, get_file_manifest From 9b3bc760a9295d4d0072f7b26a0d2701465f13a7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 04:39:13 -0700 Subject: [PATCH 089/323] -> New structure --- apiv4/__init__.py | 2 +- apiv4/cohorts_routes.py | 2 +- apiv4/cohorts_views.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 38110e1e..e3ad63dd 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -28,7 +28,7 @@ import django django.setup() -from .cohorts_routes import * +from cohorts_routes import * app = Flask(__name__) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 2b16c7bc..343c6280 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -22,7 +22,7 @@ from apiv4 import app -from .cohorts_views import get_cohort_info, get_cohorts, get_file_manifest +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest @app.route('/cohorts/', methods=['GET']) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 7198c4b3..d2b5888b 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -31,7 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -from .auth import auth_info +from auth import auth_info logger = logging.getLogger('main_logger') From 53bc32d09a5352355814cb5bd8d0408138c2ce2f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 14:33:06 -0700 Subject: [PATCH 090/323] -> New structure --- apiv4/__init__.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index e3ad63dd..5b8b9fdf 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -25,14 +25,13 @@ logger = logging.getLogger('main_logger') +app = Flask(__name__) + import django django.setup() from cohorts_routes import * -app = Flask(__name__) - - @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" From bd634544b47308a66d1654d227267500c20a03ac Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 15:02:54 -0700 Subject: [PATCH 091/323] -> Fix paths -> Fix logger --- apiv4/__init__.py | 2 -- apiv4/cohorts_routes.py | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 5b8b9fdf..c6deca00 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -35,8 +35,6 @@ @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" - cloud_logger.error("[ERROR] Directory listing: ") - cloud_logger.error(os.listdir('./')) response = jsonify({ 'code': 200, 'message': 'Welcome to the ISB-CGC API, Version 4.' diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 343c6280..4d8c1d86 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -25,7 +25,7 @@ from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest -@app.route('/cohorts/', methods=['GET']) +@app.route('/apiv4/cohorts/', methods=['GET']) def cohort(): """Retrieve extended information for a specific cohort""" user_info = auth_info() @@ -56,7 +56,7 @@ def cohort(): return response -@app.route('/cohorts', methods=['GET']) +@app.route('/apiv4/cohorts', methods=['GET']) def cohorts(): """Retrieve a user's list of cohorts""" user_info = auth_info() @@ -83,7 +83,7 @@ def cohorts(): return response -@app.route('/cohorts//file_manifest', methods=['POST', 'GET']) +@app.route('/apiv4/cohorts//file_manifest', methods=['POST', 'GET']) def cohort_file_manifest(): """Retrieve a cohort's file manifest""" user_info = auth_info() From 7d497f6b505bb405936e139274ca3cb16fff94d9 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 17:13:01 -0700 Subject: [PATCH 092/323] -> Various fixes --- apiv4/cohorts_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 4d8c1d86..7732ac98 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -26,7 +26,7 @@ @app.route('/apiv4/cohorts/', methods=['GET']) -def cohort(): +def cohort(cohort_id): """Retrieve extended information for a specific cohort""" user_info = auth_info() user = validate_user(user_info['email'], cohort_id) @@ -84,7 +84,7 @@ def cohorts(): @app.route('/apiv4/cohorts//file_manifest', methods=['POST', 'GET']) -def cohort_file_manifest(): +def cohort_file_manifest(cohort_id): """Retrieve a cohort's file manifest""" user_info = auth_info() user = validate_user(user_info['email'], cohort_id) From 141236adffd27da225dc9d96f07c8610a44c871c Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 15 Mar 2019 17:45:02 -0700 Subject: [PATCH 093/323] -> Various fixes --- apiv4/auth.py | 1 + apiv4/cohorts_routes.py | 2 ++ apiv4/cohorts_views.py | 3 --- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index 4dc7e7a2..bcd44580 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -42,6 +42,7 @@ def auth_info(): info_json = _base64_decode(encoded_info) user_info = json.loads(info_json) else: + logger.info("[STATUS] No user encoded info found.") user_info = {'id': 'anonymous'} return jsonify(user_info) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 7732ac98..5ee0ec74 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -22,6 +22,8 @@ from apiv4 import app +from auth import auth_info + from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index d2b5888b..ec730ce1 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -31,9 +31,6 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -from auth import auth_info - - logger = logging.getLogger('main_logger') From 8fcb8ded1726ac0e06cf017f21c53c9c8b383f5d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Sat, 16 Mar 2019 20:20:09 -0700 Subject: [PATCH 094/323] -> Various fixes --- apiv4/__init__.py | 3 +++ apiv4/auth.py | 5 ++++- apiv4/cohorts_routes.py | 2 -- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index c6deca00..6346514c 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -23,6 +23,8 @@ from flask import Flask, jsonify, request from flask_cors import cross_origin +from auth import auth_info + logger = logging.getLogger('main_logger') app = Flask(__name__) @@ -32,6 +34,7 @@ from cohorts_routes import * + @app.route('/apiv4', methods=['GET', 'POST']) def base(): """Base response""" diff --git a/apiv4/auth.py b/apiv4/auth.py index bcd44580..70eb3b7f 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -19,8 +19,11 @@ import logging import base64 import json +import google_auth_oauthlib.flow +import requests +from six.moves import urllib -from flask import request +from flask import request, jsonify logger = logging.getLogger(__name__) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 5ee0ec74..7732ac98 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -22,8 +22,6 @@ from apiv4 import app -from auth import auth_info - from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest From 2850116006f35c9813b6ba581e6e854c04bfca46 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Sat, 16 Mar 2019 20:42:22 -0700 Subject: [PATCH 095/323] -> Various fixes --- apiv4/auth.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index 70eb3b7f..ebe8c54b 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -19,9 +19,7 @@ import logging import base64 import json -import google_auth_oauthlib.flow import requests -from six.moves import urllib from flask import request, jsonify From 9c6cae9d5e227c5ba6a299720d4055ce69408e0d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Sun, 17 Mar 2019 00:40:40 -0700 Subject: [PATCH 096/323] -> Various fixes --- apiv4/cohorts_routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 7732ac98..27d1486e 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -23,6 +23,7 @@ from apiv4 import app from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest +from auth import auth_info @app.route('/apiv4/cohorts/', methods=['GET']) From 308ec0bb01ae29e6123ee067478afeb8981f1b69 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Sun, 17 Mar 2019 00:42:16 -0700 Subject: [PATCH 097/323] -> Various fixes --- apiv4/cohorts_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 27d1486e..ff1b68df 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -37,7 +37,7 @@ def cohort(cohort_id): if not user: response = jsonify({ 'code': 403, - 'message': "User {} does not have access to cohort ID {}".format(user_info['email'],str(cohort_id))}) + 'message': "User {} does not have access to cohort ID {}".format(user_info['email'] if 'email' in user_info else 'Anonymous',str(cohort_id))}) response.status_code = 403 else: From cc58856da6bcce418ac0d4adda4076e126a59fae Mon Sep 17 00:00:00 2001 From: s-paquett Date: Sun, 17 Mar 2019 12:43:46 -0700 Subject: [PATCH 098/323] -> Various fixes --- apiv4/cohorts_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index ff1b68df..9ebd78f2 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -22,7 +22,7 @@ from apiv4 import app -from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, validate_user from auth import auth_info From 410694e6ec6b2d3864d541972b8419acaa72e9b7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 12:37:33 -0700 Subject: [PATCH 099/323] -> Various fixes --- apiv4/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index ebe8c54b..cbff67e1 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -46,5 +46,5 @@ def auth_info(): logger.info("[STATUS] No user encoded info found.") user_info = {'id': 'anonymous'} - return jsonify(user_info) + return user_info # END METHODS From 507246a2d0a767b45455b2f713fb74b48f0f4255 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 13:10:34 -0700 Subject: [PATCH 100/323] -> Various fixes --- apiv4/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index cbff67e1..6d5c9436 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -44,7 +44,7 @@ def auth_info(): user_info = json.loads(info_json) else: logger.info("[STATUS] No user encoded info found.") - user_info = {'id': 'anonymous'} + user_info = {'id': 'anonymous', 'email': 'Anonymous'} return user_info # END METHODS From 0ef15b6198f392534c7b19a404b7e5994577490f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 14:00:05 -0700 Subject: [PATCH 101/323] -> Fix circular import issue with regard to routes -> FIx local dev (swap to python-dotenv) --- apiv4/__init__.py | 13 +------------ apiv4/cohorts_routes.py | 11 +++++++++++ apiv4/cohorts_views.py | 5 +---- requirements.txt | 2 +- settings.py | 5 +++-- 5 files changed, 17 insertions(+), 19 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 6346514c..b8398054 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -35,17 +35,6 @@ from cohorts_routes import * -@app.route('/apiv4', methods=['GET', 'POST']) -def base(): - """Base response""" - response = jsonify({ - 'code': 200, - 'message': 'Welcome to the ISB-CGC API, Version 4.' - }) - response.status_code = 200 - return response - - # Error handlers @app.errorhandler(500) def unexpected_error(e): @@ -59,4 +48,4 @@ def unexpected_error(e): if __name__ == '__main__': - app.run(host='127.0.0.1', port=8080, debug=True) + app.run(host='127.0.0.1', port=8090, debug=True) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 9ebd78f2..93e49600 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -26,6 +26,17 @@ from auth import auth_info +@app.route('/apiv4', methods=['GET', 'POST']) +def base(): + """Base response""" + response = jsonify({ + 'code': 200, + 'message': 'Welcome to the ISB-CGC API, Version 4.' + }) + response.status_code = 200 + return response + + @app.route('/apiv4/cohorts/', methods=['GET']) def cohort(cohort_id): """Retrieve extended information for a specific cohort""" diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index ec730ce1..3945d383 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -41,10 +41,7 @@ def validate_user(user_email, cohort_id): user = Django_User.objects.get(email=user_email) Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) except ObjectDoesNotExist as e: - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}.".format(user_email, cohort_id) - logger.warn(err_msg) + logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) except Exception as e: logger.exception(e) diff --git a/requirements.txt b/requirements.txt index 048a4ef3..29d062cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,6 @@ PyYAML==4.2b1 six==1.12.0 uritemplate==3.0.0 GoogleAppEngineCloudStorageClient==1.9.22.1 -django-dotenv==1.4.2 django-finalware==1.0.0 django-allauth==0.35.0 jsonschema==2.6.0 @@ -23,3 +22,4 @@ flask-cors==3.0.7 pyjwt==1.6.1 cryptography==2.4.2 future==0.17.1 +python-dotenv==0.10.0 \ No newline at end of file diff --git a/settings.py b/settings.py index 3f22025f..ddb39f28 100644 --- a/settings.py +++ b/settings.py @@ -17,14 +17,15 @@ import os from os.path import join, dirname import sys -import dotenv +from pathlib import Path +from dotenv import load_dotenv from socket import gethostname, gethostbyname env_path = '' if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') -dotenv.read_dotenv(join(dirname(__file__), env_path+'.env')) +load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' From db3cd86a4ebf7d5f8844111eed72a8bb63f6aa66 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 14:16:19 -0700 Subject: [PATCH 102/323] -> Logger adjustments --- apiv4/__init__.py | 6 +++--- apiv4/auth.py | 4 ++-- apiv4/cohorts_routes.py | 6 ++++-- apiv4/cohorts_views.py | 2 +- apiv4/main_routes.py | 45 +++++++++++++++++++++++++++++++++++++++++ settings.py | 2 ++ 6 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 apiv4/main_routes.py diff --git a/apiv4/__init__.py b/apiv4/__init__.py index b8398054..336ec9ac 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -19,19 +19,19 @@ import logging import os import sys - from flask import Flask, jsonify, request from flask_cors import cross_origin - from auth import auth_info +from django.conf import settings -logger = logging.getLogger('main_logger') +logger = logging.getLogger(settings.LOGGER_NAME) app = Flask(__name__) import django django.setup() +from main_routes import * from cohorts_routes import * diff --git a/apiv4/auth.py b/apiv4/auth.py index 6d5c9436..255f688e 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -20,10 +20,10 @@ import base64 import json import requests - from flask import request, jsonify +from django.conf import settings -logger = logging.getLogger(__name__) +logger = logging.getLogger(setting.LOGGER_NAME) # BEGIN METHODS diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 93e49600..1625ba11 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -19,11 +19,12 @@ import logging import json from flask import jsonify, request - from apiv4 import app - from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, validate_user from auth import auth_info +from django.conf import settings + +logger = logging.getLogger(setting.LOGGER_NAME) @app.route('/apiv4', methods=['GET', 'POST']) @@ -41,6 +42,7 @@ def base(): def cohort(cohort_id): """Retrieve extended information for a specific cohort""" user_info = auth_info() + logger.info("[STATUS] User info: {}".format(str(user_info))) user = validate_user(user_info['email'], cohort_id) response = None diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 3945d383..a7b40651 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -31,7 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -logger = logging.getLogger('main_logger') +logger = logging.getLogger(setting.LOGGER_NAME) def validate_user(user_email, cohort_id): diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py new file mode 100644 index 00000000..28d35515 --- /dev/null +++ b/apiv4/main_routes.py @@ -0,0 +1,45 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +from flask import jsonify, request + +from apiv4 import app + + +@app.route('/apiv4', methods=['GET', 'POST']) +def apiv4(): + """Base response""" + response = jsonify({ + 'code': 200, + 'message': 'Welcome to the ISB-CGC API, Version 4.' + }) + response.status_code = 200 + return response + + +@app.route('/apiv4/programs', methods=['GET']) +def programs(): + """List the programs currently available on this API""" + response = jsonify({ + 'code': 200, + 'message': 'TCGA (HG19, HG38); TARGET (HG19, HG38), CCLE (HG19)' + }) + response.status_code = 200 + return response diff --git a/settings.py b/settings.py index ddb39f28..8a88937a 100644 --- a/settings.py +++ b/settings.py @@ -43,6 +43,8 @@ ADMINS = () MANAGERS = ADMINS +LOGGER_NAME = os.environ.get('API_LOGGER_NAME', 'main_logger') + GCLOUD_PROJECT_ID = os.environ.get('GCLOUD_PROJECT_ID', '') GCLOUD_PROJECT_NUMBER = os.environ.get('GCLOUD_PROJECT_NUMBER', '') BIGQUERY_PROJECT_ID = os.environ.get('BIGQUERY_PROJECT_ID', GCLOUD_PROJECT_ID) From 33938065e91d6a466bf4e5ad5736f94ce40626f6 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 14:19:56 -0700 Subject: [PATCH 103/323] -> Logger adjustments --- apiv4/__init__.py | 8 ++++---- apiv4/auth.py | 2 +- apiv4/cohorts_routes.py | 2 +- apiv4/cohorts_views.py | 2 +- apiv4/main_routes.py | 4 +++- 5 files changed, 10 insertions(+), 8 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 336ec9ac..41a49ea3 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -21,19 +21,19 @@ import sys from flask import Flask, jsonify, request from flask_cors import cross_origin -from auth import auth_info -from django.conf import settings - -logger = logging.getLogger(settings.LOGGER_NAME) app = Flask(__name__) import django django.setup() +from django.conf import settings +from auth import auth_info from main_routes import * from cohorts_routes import * +logger = logging.getLogger(settings.LOGGER_NAME) + # Error handlers @app.errorhandler(500) diff --git a/apiv4/auth.py b/apiv4/auth.py index 255f688e..e17f760e 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -23,7 +23,7 @@ from flask import request, jsonify from django.conf import settings -logger = logging.getLogger(setting.LOGGER_NAME) +logger = logging.getLogger(settings.LOGGER_NAME) # BEGIN METHODS diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 1625ba11..1c190fb0 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -24,7 +24,7 @@ from auth import auth_info from django.conf import settings -logger = logging.getLogger(setting.LOGGER_NAME) +logger = logging.getLogger(settings.LOGGER_NAME) @app.route('/apiv4', methods=['GET', 'POST']) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index a7b40651..f38d74fb 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -31,7 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -logger = logging.getLogger(setting.LOGGER_NAME) +logger = logging.getLogger(settings.LOGGER_NAME) def validate_user(user_email, cohort_id): diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py index 28d35515..ddacc212 100644 --- a/apiv4/main_routes.py +++ b/apiv4/main_routes.py @@ -19,9 +19,11 @@ import logging import json from flask import jsonify, request - +from django.conf import settings from apiv4 import app +logger = logging.getLogger(settings.LOGGER_NAME) + @app.route('/apiv4', methods=['GET', 'POST']) def apiv4(): From 7c4c6ebe28a3e11373ae80ec3c1b36696c254e2c Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 14:47:16 -0700 Subject: [PATCH 104/323] -> Logger adjustments --- apiv4/cohorts_routes.py | 11 ----------- apiv4/main_routes.py | 11 ----------- 2 files changed, 22 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 1c190fb0..8d7e95d1 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -27,17 +27,6 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4', methods=['GET', 'POST']) -def base(): - """Base response""" - response = jsonify({ - 'code': 200, - 'message': 'Welcome to the ISB-CGC API, Version 4.' - }) - response.status_code = 200 - return response - - @app.route('/apiv4/cohorts/', methods=['GET']) def cohort(cohort_id): """Retrieve extended information for a specific cohort""" diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py index ddacc212..132206d8 100644 --- a/apiv4/main_routes.py +++ b/apiv4/main_routes.py @@ -34,14 +34,3 @@ def apiv4(): }) response.status_code = 200 return response - - -@app.route('/apiv4/programs', methods=['GET']) -def programs(): - """List the programs currently available on this API""" - response = jsonify({ - 'code': 200, - 'message': 'TCGA (HG19, HG38); TARGET (HG19, HG38), CCLE (HG19)' - }) - response.status_code = 200 - return response From 3291ddc15f13300da3d822a6345a8eadbdfb8e56 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 18 Mar 2019 23:34:37 -0700 Subject: [PATCH 105/323] -> Logger adjustments --- apiv4/cohorts_routes.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 8d7e95d1..cf2e1413 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -75,13 +75,11 @@ def cohorts(): response.status_code = 404 else: - cohort_info = get_cohort_info(cohort_id) - if cohort_info: - response = jsonify({ - 'code': 200, - 'data': jsonify(cohort_info) - }) - response.status_code = 200 + response = jsonify({ + 'code': 200, + 'data': jsonify(cohort_list) + }) + response.status_code = 200 return response From 859a8181a38086e1e8a0ed3ba2d1bf1bba2b8ea6 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 19 Mar 2019 12:33:53 -0700 Subject: [PATCH 106/323] -> Don't jsonify JSON... --- apiv4/cohorts_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index cf2e1413..22497905 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -47,7 +47,7 @@ def cohort(cohort_id): if cohort_info: response = jsonify({ 'code': 200, - 'data': jsonify(cohort_info) + 'data': cohort_info }) response.status_code = 200 else: @@ -77,7 +77,7 @@ def cohorts(): else: response = jsonify({ 'code': 200, - 'data': jsonify(cohort_list) + 'data': cohort_list }) response.status_code = 200 From e01e343ea5f9b0739932c807bffe2b4384c1df75 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 21 Mar 2019 12:56:50 -0700 Subject: [PATCH 107/323] -> Add in basic programs fetch --- apiv4/cohorts_routes.py | 6 +- apiv4/main_routes.py | 7 ++- apiv4/program_routes.py | 49 ++++++++++++++++ apiv4/program_views.py | 48 ++++++++++++++++ scripts/isb_auth.py | 107 ++++++++++++++++++++++++++++++++++ scripts/isb_curl.py | 123 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 335 insertions(+), 5 deletions(-) create mode 100644 apiv4/program_routes.py create mode 100644 apiv4/program_views.py create mode 100644 scripts/isb_auth.py create mode 100644 scripts/isb_curl.py diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 22497905..06a707d6 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -27,7 +27,7 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/cohorts/', methods=['GET']) +@app.route('/apiv4/cohorts//', methods=['GET']) def cohort(cohort_id): """Retrieve extended information for a specific cohort""" user_info = auth_info() @@ -59,7 +59,7 @@ def cohort(cohort_id): return response -@app.route('/apiv4/cohorts', methods=['GET']) +@app.route('/apiv4/cohorts/', methods=['GET']) def cohorts(): """Retrieve a user's list of cohorts""" user_info = auth_info() @@ -84,7 +84,7 @@ def cohorts(): return response -@app.route('/apiv4/cohorts//file_manifest', methods=['POST', 'GET']) +@app.route('/apiv4/cohorts//file_manifest/', methods=['POST', 'GET']) def cohort_file_manifest(cohort_id): """Retrieve a cohort's file manifest""" user_info = auth_info() diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py index 132206d8..e3649873 100644 --- a/apiv4/main_routes.py +++ b/apiv4/main_routes.py @@ -25,12 +25,15 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4', methods=['GET', 'POST']) +@app.route('/apiv4/', methods=['GET', 'POST']) def apiv4(): """Base response""" response = jsonify({ 'code': 200, - 'message': 'Welcome to the ISB-CGC API, Version 4.' + 'message': 'Welcome to the ISB-CGC API, Version 4.', + 'documentation': 'Documentation for this API is available at https://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html' }) response.status_code = 200 return response + + diff --git a/apiv4/program_routes.py b/apiv4/program_routes.py new file mode 100644 index 00000000..b2c87e0f --- /dev/null +++ b/apiv4/program_routes.py @@ -0,0 +1,49 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from django.conf import settings +from program_views import get_programs + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/apiv4/programs/', methods=['GET']) +def programs(): + """Retrieve the list of programs and builds currently available for cohort creation.""" + response = None + + program_info = get_programs() + + if program_info: + response = jsonify({ + 'code': 200, + 'data': program_info + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while retrieving the program list.' + }) + response.status_code =500 + + return response diff --git a/apiv4/program_views.py b/apiv4/program_views.py new file mode 100644 index 00000000..ed9df017 --- /dev/null +++ b/apiv4/program_views.py @@ -0,0 +1,48 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +import django + +from flask import request + +from django.core.signals import request_finished +from django.conf import settings + +from projects.models import Program, Project + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_programs(): + django.setup() + program_info = None + try: + program_info = [ + { + 'name': x.name, + 'description': x.description, + 'projects': [{'name': y.name, 'description': y.description } for y in Project.objects.filter(program=x,active=1)] + } + for x in Program.get_public_programs() + ] + except Exception as e: + logger.exception(e) + + return program_info diff --git a/scripts/isb_auth.py b/scripts/isb_auth.py new file mode 100644 index 00000000..25c23c59 --- /dev/null +++ b/scripts/isb_auth.py @@ -0,0 +1,107 @@ +''' +Copyright 2015, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Authenticates user for accessing the ISB-CGC Endpoint APIs. + +May be run from the command line or in scripts/ipython. + +The credentials file can be copied to any machine from which you want +to access the API. + +1. Command Line + python ./isb_auth.py saves the user's credentials; + OPTIONAL: + -v for verbose (returns token!) + -s FILE sets credentials file [default: ~/.isb_credentials] + -u URL-only: for use over terminal connections; + gives user a URL to paste into their browser, + and asks for an auth code in return + +2. Python + import isb_auth + isb_auth.get_credentials() + + # optional: to store credentials in a different location + from oauth2client.file import Storage + import isb_auth + import os + + storage_file = os.path.join(os.path.expanduser("~"), "{USER_CREDENTIALS_FILE_NAME}") + storage = Storage(storage_file) + isb_auth.get_credentials(storage=storage) +''' + +from argparse import ArgumentParser +import os + +from oauth2client.client import OAuth2WebServerFlow +from oauth2client import tools +from oauth2client.file import Storage + +VERBOSE = False +# for native application - same as settings.INSTALLED_APP_CLIENT_ID +CLIENT_ID = '907668440978-0ol0griu70qkeb6k3gnn2vipfa5mgl60.apps.googleusercontent.com' +# NOTE: this is NOT actually a 'secret' -- we're using the 'installed +# application' OAuth pattern here +CLIENT_SECRET = 'To_WJH7-1V-TofhNGcEqmEYi' + +EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email' +DEFAULT_STORAGE_FILE = os.path.join(os.path.expanduser("~"), '.isb_credentials') + + +def maybe_print(msg): + if VERBOSE: + print(msg) + + +def get_credentials(storage=None, oauth_flow_args=[]): + noweb = '--noauth_local_webserver' + if __name__ != '__main__' and noweb not in oauth_flow_args: + oauth_flow_args.append(noweb) + if storage is None: + storage = Storage(DEFAULT_STORAGE_FILE) + credentials = storage.get() + if not credentials or credentials.invalid: + maybe_print('Credentials file was missing or invalid, kicking off OAuth2 flow...') + flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, EMAIL_SCOPE) + flow.auth_uri = flow.auth_uri.rstrip('/') + '?approval_prompt=force' + credentials = tools.run_flow(flow, storage, tools.argparser.parse_args(oauth_flow_args)) + return credentials + + +def main(): + global VERBOSE + args = parse_args() + oauth_flow_args = [args.noauth_local_webserver] if args.noauth_local_webserver else [] + VERBOSE = args.verbose + maybe_print('--verbose: printing extra information') + storage = Storage(args.storage_file) + credentials = get_credentials(storage, oauth_flow_args) + maybe_print('credentials stored in ' + args.storage_file) + maybe_print('access_token: ' + credentials.access_token) + maybe_print('refresh_token: ' + credentials.refresh_token) + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument('--storage_file', '-s', default=DEFAULT_STORAGE_FILE, help='Storage file to use for the credentials (default is {})'.format(DEFAULT_STORAGE_FILE)) + parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', help='Display credentials storage location, access token, and refresh token') + parser.set_defaults(verbose=False) + parser.add_argument('--noauth_local_webserver','-u', action='store_const', const='--noauth_local_webserver') + return parser.parse_args() + + +if __name__ == '__main__': + main() diff --git a/scripts/isb_curl.py b/scripts/isb_curl.py new file mode 100644 index 00000000..b95a7163 --- /dev/null +++ b/scripts/isb_curl.py @@ -0,0 +1,123 @@ +#! /usr/bin/python2.7 +''' +Copyright 2015, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + +isb_curl can be called by commandline or used as a library + +URL = https://isb-cgc.appspot.com/_ah/api/{API-NAME}/{VERSION}/{ENDPOINT}?{QUERYSTRING-PARAMS} + e.g. for the "cohorts_list" endpoint: + https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/cohorts_list + + +A. Command Line: + python isb_auth.py # saves the user's credentials to their root directory + python isb_curl.py URL + note: if the endpoint takes a resource in the request body, such as the save_cohort endpoint, use the following: + python isb_curl.py https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/save_cohort?name={YOUR-COHORT-NAME} \ + -d '{"Study": "BRCA"}' -H "Content-Type: application/json" + + +B. Python: + import isb_auth + import isb_curl + import requests + + url = 'https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/cohorts_list' + token = isb_curl.get_access_token() + head = {'Authorization': 'Bearer ' + token} + + # for GET requests + resp = requests.get(url, headers=head) + # querystring parameters can be added to either the url itself... + url += '?cohort_id=1' + resp = requests.get(url, headers=head) + # ... or passed in with the params kwarg + url = 'https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/cohorts_list' + params = {'cohort_id': 1} + resp = requests.get(url, headers=head, params=params) + + # if the endpoint takes a resource in the request body, such as the save_cohort endpoint... + url = https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/save_cohort?name=my-new-cohort' + head.update({'Content-Type': 'application/json'}) + payload = {"SampleBarcode": "TCGA-02-0001-01C,TCGA-02-0001-10A,TCGA-01-0642-11A"} + resp = requests.post(url, headers=head, json=payload) + + # if requests version < 2.4.2 + import json + resp = requests.post(url, headers=head, data=json.dumps(payload)) + +''' + +import httplib2 +import os +import sys +from oauth2client.file import Storage +import json + +CREDENTIALS_LOC_ENV = 'ISB_CREDENTIALS' +DEFAULT_CREDENTIALS_LOC = os.path.join(os.path.expanduser("~"), '.isb_credentials') + + +def check(assertion, msg): + if not assertion: + error(msg) + + +def error(msg): + sys.stderr.write(msg + '\n') + sys.exit(1) + + +def get_credentials_location(): + credentials_location = os.environ.get(CREDENTIALS_LOC_ENV, DEFAULT_CREDENTIALS_LOC) + check(credentials_location, 'Couldn\'t locate the credentials file at {} - run isb_auth.py or check the DEFAULT_CREDENTIALS_LOC at the top of this script.'.format(credentials_location)) + return credentials_location + + +def load_credentials(credentials_location): + storage = Storage(credentials_location) + credentials = storage.get() + check(credentials and not credentials.invalid, + 'Couldn\'t locate the credentials file at {} - run isb_auth.py or check the DEFAULT_CREDENTIALS_LOC at the top of this script.'.format(credentials_location)) + return credentials + + +# Although we can use load_credentials to check the expiration (and re-up if needed), we need the +# encrypted ID token, NOT the access_token, to do a request via the ESP. To this end we load the +# file as JSON and pull the provided encrypted token there (it can also be reconstituted). +def get_id_token(credentials_location=get_credentials_location()): + credentials = load_credentials(credentials_location) + if credentials.access_token_expired: + credentials.refresh(httplib2.Http()) + creds_json = open(credentials_location, "r") + token = json.loads(creds_json.read()) + return token['token_response']['id_token'] + + +def main(): + args = sys.argv[1:] + check(args, 'usage: isb_curl.py ') + id_token = get_id_token() + curl_args = ['curl', '-H', 'Authorization: Bearer ' + id_token] + args + os.execvp('curl', curl_args) + + +# this allows us to call this from command line +if __name__ == '__main__': + main() + + From 4bcbbac2143da1d283aadeefa036cfd3bca1d022 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 21 Mar 2019 19:16:27 -0700 Subject: [PATCH 108/323] -> Program list fetch -> Turn off strict slash requirement --- apiv4/cohorts_routes.py | 6 +++--- apiv4/cohorts_views.py | 1 - apiv4/main_routes.py | 2 +- apiv4/program_routes.py | 4 ++-- 4 files changed, 6 insertions(+), 7 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 06a707d6..0a366d1d 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -27,7 +27,7 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/cohorts//', methods=['GET']) +@app.route('/apiv4/cohorts//', methods=['GET'], strict_slashes=False) def cohort(cohort_id): """Retrieve extended information for a specific cohort""" user_info = auth_info() @@ -59,7 +59,7 @@ def cohort(cohort_id): return response -@app.route('/apiv4/cohorts/', methods=['GET']) +@app.route('/apiv4/cohorts/', methods=['GET'], strict_slashes=False) def cohorts(): """Retrieve a user's list of cohorts""" user_info = auth_info() @@ -84,7 +84,7 @@ def cohorts(): return response -@app.route('/apiv4/cohorts//file_manifest/', methods=['POST', 'GET']) +@app.route('/apiv4/cohorts//file_manifest/', methods=['POST', 'GET'], strict_slashes=False) def cohort_file_manifest(cohort_id): """Retrieve a cohort's file manifest""" user_info = auth_info() diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index f38d74fb..f097d981 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -98,7 +98,6 @@ def get_cohort_info(cohort_id): 'name': cohort_obj.name, 'case_count': cohort_obj.case_size(), 'sample_count': cohort_obj.sample_size(), - 'creation_filters': cohort_obj.get_current_filters(), 'programs': cohort_obj.get_program_names() } except ObjectDoesNotExist as e: diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py index e3649873..c9c95abf 100644 --- a/apiv4/main_routes.py +++ b/apiv4/main_routes.py @@ -25,7 +25,7 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/', methods=['GET', 'POST']) +@app.route('/apiv4/', methods=['GET', 'POST'], strict_slashes=False) def apiv4(): """Base response""" response = jsonify({ diff --git a/apiv4/program_routes.py b/apiv4/program_routes.py index b2c87e0f..9a264202 100644 --- a/apiv4/program_routes.py +++ b/apiv4/program_routes.py @@ -26,7 +26,7 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/programs/', methods=['GET']) +@app.route('/apiv4/programs/', methods=['GET'], strict_slashes=False) def programs(): """Retrieve the list of programs and builds currently available for cohort creation.""" response = None @@ -44,6 +44,6 @@ def programs(): 'code': 500, 'message': 'Encountered an error while retrieving the program list.' }) - response.status_code =500 + response.status_code = 500 return response From db7591d538384b08dc867c968c595fa40de73f50 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 22 Mar 2019 18:38:05 -0700 Subject: [PATCH 109/323] -> Bugfixes... --- apiv4/__init__.py | 1 + apiv4/cohorts_views.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 41a49ea3..12274d8c 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -31,6 +31,7 @@ from auth import auth_info from main_routes import * from cohorts_routes import * +from program_routes import * logger = logging.getLogger(settings.LOGGER_NAME) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index f097d981..6b30c272 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -92,7 +92,7 @@ def get_file_manifest(cohort_id, user): def get_cohort_info(cohort_id): cohort = None try: - cohort_obj = Cohorts.objects.get(id=cohort_id) + cohort_obj = Cohort.objects.get(id=cohort_id) cohort = { 'id': cohort_obj.id, 'name': cohort_obj.name, From 76bde0f56c7c52d06d6cdeb2b6f62377eb3536ce Mon Sep 17 00:00:00 2001 From: s-paquett Date: Sat, 23 Mar 2019 15:08:22 -0700 Subject: [PATCH 110/323] -> Cohort preview counting --- apiv4/cohorts_routes.py | 29 +++++++++++++++++++++++++++-- apiv4/cohorts_views.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 2 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 0a366d1d..71b58b02 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request from apiv4 import app -from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, validate_user +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user from auth import auth_info from django.conf import settings @@ -109,7 +109,32 @@ def cohort_file_manifest(cohort_id): else: response = jsonify({ 'code': 500, - 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id))}) + 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)) + }) response.status_code = 500 return response + + +@app.route('/apiv4/cohorts/preview/', methods=['POST', 'GET'], strict_slashes=False) +def cohort_preview(): + """List the samples, cases, and counts a given set of cohort filters would produce""" + + response = None + + cohort_counts = get_cohort_counts() + + if cohort_counts: + response = jsonify({ + 'code': 200, + 'data': cohort_counts + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to retrieve case and sample counts for these filters." + }) + response.status_code = 500 + + return response \ No newline at end of file diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 6b30c272..127aa29a 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -30,6 +30,8 @@ from cohorts.models import Cohort_Perms, Cohort, Filters from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files +from cohorts.metadata_counting import public_metadata_counts +from projects.models import Program logger = logging.getLogger(settings.LOGGER_NAME) @@ -127,3 +129,36 @@ def get_cohorts(user_email): logger.info("No cohorts found for user {}".format(user_email)) return cohort_list + + +def get_cohort_counts(): + cohort_counts = None + + try: + request_data = request.get_json() + print("Data: " + str(request_data)) + for prog_name in request_data['filters']: + try: + prog_filters = None + this_program = Program.objects.get(name=prog_name.upper(), is_public=1, active=1) + if request_data['filters'][prog_name]: + prog_filters = request_data['filters'][prog_name] + prog_counts = public_metadata_counts(prog_filters, 0, None, this_program.id) + if prog_counts: + if not cohort_counts: + cohort_counts = {} + cohort_counts[prog_name] = prog_counts + else: + logger.warn("Could not obtain program counts for {} - skipping.".format( + prog_name + )) + except ObjectDoesNotExist as e: + logger.error("Program {} was not found as an active, public program - skipping.".format( + prog_name + )) + continue + except Exception as e: + logger.exception(e) + + return cohort_counts + From d9f2d566c74a0962f7f784b8cb0af1e24da8c8ca Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 25 Mar 2019 12:56:38 -0700 Subject: [PATCH 111/323] -> Cohort preview counting --- apiv4/cohorts_views.py | 4 ++-- settings.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 127aa29a..e6bf38c0 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -30,7 +30,7 @@ from cohorts.models import Cohort_Perms, Cohort, Filters from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -from cohorts.metadata_counting import public_metadata_counts +from cohorts.metadata_helpers import get_sample_case_list from projects.models import Program logger = logging.getLogger(settings.LOGGER_NAME) @@ -143,7 +143,7 @@ def get_cohort_counts(): this_program = Program.objects.get(name=prog_name.upper(), is_public=1, active=1) if request_data['filters'][prog_name]: prog_filters = request_data['filters'][prog_name] - prog_counts = public_metadata_counts(prog_filters, 0, None, this_program.id) + prog_counts = get_sample_case_list(None, prog_filters, None, this_program.id) if prog_counts: if not cohort_counts: cohort_counts = {} diff --git a/settings.py b/settings.py index 8a88937a..64d295e8 100644 --- a/settings.py +++ b/settings.py @@ -25,7 +25,7 @@ if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') -load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) +load_dotenv(dotenv_path=join(dirname(__file__), env_path+'api.env')) APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' From 06fe33a8554e33c81fbc2745238e0ca58eef310b Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 25 Mar 2019 15:44:57 -0700 Subject: [PATCH 112/323] -> Sample and case metadatadata fetching --- apiv4/sample_case_routes.py | 70 +++++++++++++++++++++++++++++++++++++ apiv4/sample_case_views.py | 46 ++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 apiv4/sample_case_routes.py create mode 100644 apiv4/sample_case_views.py diff --git a/apiv4/sample_case_routes.py b/apiv4/sample_case_routes.py new file mode 100644 index 00000000..1f75fdef --- /dev/null +++ b/apiv4/sample_case_routes.py @@ -0,0 +1,70 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from django.conf import settings +from sample_case_views import get_full_sample_metadata, get_case_metadata + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/apiv4/samples//', methods=['GET'], strict_slashes=False) +def sample_metadata(sample_barcode): + + response = None + + sample_metadata = get_full_sample_metadata(sample_barcode) + + if sample_metadata: + response = jsonify({ + 'code': 200, + 'data': sample_metadata + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Sample barcode {} was not found.".format(sample_barcode)}) + response.status_code = 404 + + return response + + +@app.route('/apiv4/cases//', methods=['GET'], strict_slashes=False) +def sample_metadata(case_metadata): + response = None + + case_metadata = get_case_metadata(case_barcode) + + if case_metadata: + response = jsonify({ + 'code': 200, + 'data': case_metadata + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Sample barcode {} was not found.".format(case_barcode)}) + response.status_code = 404 + + return response + diff --git a/apiv4/sample_case_views.py b/apiv4/sample_case_views.py new file mode 100644 index 00000000..15266d7f --- /dev/null +++ b/apiv4/sample_case_views.py @@ -0,0 +1,46 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +import django + +from flask import request + +from django.conf import settings +from cohorts.metadata_helpers import get_sample_metadata + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_full_sample_metadata(sample_barcode): + metadata = get_sample_metadata(sample_barcode, False, True) + + if len(metadata.keys()): + return metadata + else: + return None + + +def get_case_metadata(case_barcode): + metadata = get_sample_metadata(case_barcode, True, True) + + if len(metadata.keys()): + return metadata + else: + return None From 26bd6ac9e6748e6533e96b42412d0570baa7421a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 25 Mar 2019 18:33:58 -0700 Subject: [PATCH 113/323] -> Sample and case metadatadata fetching --- apiv4/__init__.py | 1 + apiv4/sample_case_routes.py | 47 +++++++++++++++++++++++++++++++++++-- apiv4/sample_case_views.py | 6 ++--- 3 files changed, 49 insertions(+), 5 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 12274d8c..92f9a33d 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -32,6 +32,7 @@ from main_routes import * from cohorts_routes import * from program_routes import * +from sample_case_routes import * logger = logging.getLogger(settings.LOGGER_NAME) diff --git a/apiv4/sample_case_routes.py b/apiv4/sample_case_routes.py index 1f75fdef..589b36b2 100644 --- a/apiv4/sample_case_routes.py +++ b/apiv4/sample_case_routes.py @@ -49,7 +49,7 @@ def sample_metadata(sample_barcode): @app.route('/apiv4/cases//', methods=['GET'], strict_slashes=False) -def sample_metadata(case_metadata): +def case_metadata(case_barcode): response = None case_metadata = get_case_metadata(case_barcode) @@ -63,8 +63,51 @@ def sample_metadata(case_metadata): else: response = jsonify({ 'code': 404, - 'message': "Sample barcode {} was not found.".format(case_barcode)}) + 'message': "Case barcode {} was not found.".format(case_barcode)}) + response.status_code = 404 + + return response + + +@app.route('/apiv4/samples/', methods=['POST'], strict_slashes=False) +def sample_metadata_list(): + response = None + + sample_metadata = get_full_sample_metadata(sample_barcode) + + if sample_metadata: + response = jsonify({ + 'code': 200, + 'data': sample_metadata + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Sample barcode {} was not found.".format(sample_barcode)}) response.status_code = 404 return response + +@app.route('/apiv4/cases/', methods=['POST'], strict_slashes=False) +def case_metadata_list(): + response = None + + request_data = request.get_json() + + case_metadata = get_case_metadata(case_barcode) + + if case_metadata: + response = jsonify({ + 'code': 200, + 'data': case_metadata + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Sample barcode {} was not found.".format(case_barcode)}) + response.status_code = 404 + + return response diff --git a/apiv4/sample_case_views.py b/apiv4/sample_case_views.py index 15266d7f..c5c78318 100644 --- a/apiv4/sample_case_views.py +++ b/apiv4/sample_case_views.py @@ -23,13 +23,13 @@ from flask import request from django.conf import settings -from cohorts.metadata_helpers import get_sample_metadata +from cohorts.metadata_helpers import get_sample_case_metadata logger = logging.getLogger(settings.LOGGER_NAME) def get_full_sample_metadata(sample_barcode): - metadata = get_sample_metadata(sample_barcode, False, True) + metadata = get_sample_case_metadata([sample_barcode], False) if len(metadata.keys()): return metadata @@ -38,7 +38,7 @@ def get_full_sample_metadata(sample_barcode): def get_case_metadata(case_barcode): - metadata = get_sample_metadata(case_barcode, True, True) + metadata = get_sample_case_metadata([case_barcode], True) if len(metadata.keys()): return metadata From 754c9a0c97e55800aac03b0745f1d2558d80323c Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 25 Mar 2019 18:44:06 -0700 Subject: [PATCH 114/323] -> Sample and case metadatadata fetching --- apiv4/sample_case_routes.py | 20 +++++++++++--------- apiv4/sample_case_views.py | 12 ++++++------ 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/apiv4/sample_case_routes.py b/apiv4/sample_case_routes.py index 589b36b2..5852391a 100644 --- a/apiv4/sample_case_routes.py +++ b/apiv4/sample_case_routes.py @@ -31,12 +31,12 @@ def sample_metadata(sample_barcode): response = None - sample_metadata = get_full_sample_metadata(sample_barcode) + metadata = get_full_sample_metadata([sample_barcode]) if sample_metadata: response = jsonify({ 'code': 200, - 'data': sample_metadata + 'data': metadata }) response.status_code = 200 else: @@ -52,12 +52,12 @@ def sample_metadata(sample_barcode): def case_metadata(case_barcode): response = None - case_metadata = get_case_metadata(case_barcode) + metadata = get_case_metadata([case_barcode]) - if case_metadata: + if metadata: response = jsonify({ 'code': 200, - 'data': case_metadata + 'data': metadata }) response.status_code = 200 else: @@ -73,12 +73,14 @@ def case_metadata(case_barcode): def sample_metadata_list(): response = None - sample_metadata = get_full_sample_metadata(sample_barcode) + request_data = request.get_json() + + metadata = get_full_sample_metadata(request_data['sample_barcodes']) if sample_metadata: response = jsonify({ 'code': 200, - 'data': sample_metadata + 'data': metadata }) response.status_code = 200 else: @@ -96,12 +98,12 @@ def case_metadata_list(): request_data = request.get_json() - case_metadata = get_case_metadata(case_barcode) + metadata = get_case_metadata(request_data['case_barcodes']) if case_metadata: response = jsonify({ 'code': 200, - 'data': case_metadata + 'data': metadata }) response.status_code = 200 else: diff --git a/apiv4/sample_case_views.py b/apiv4/sample_case_views.py index c5c78318..942145c7 100644 --- a/apiv4/sample_case_views.py +++ b/apiv4/sample_case_views.py @@ -28,19 +28,19 @@ logger = logging.getLogger(settings.LOGGER_NAME) -def get_full_sample_metadata(sample_barcode): - metadata = get_sample_case_metadata([sample_barcode], False) +def get_sample_metadata(sample_barcodes): + metadata = get_sample_case_metadata(sample_barcodes, False) - if len(metadata.keys()): + if metadata['total_found']: return metadata else: return None -def get_case_metadata(case_barcode): - metadata = get_sample_case_metadata([case_barcode], True) +def get_case_metadata(case_barcodes): + metadata = get_sample_case_metadata(case_barcodes, True) - if len(metadata.keys()): + if metadata['total_found']: return metadata else: return None From 9aa34613999361a5ec742e152b35043348149dd9 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 26 Mar 2019 11:51:39 -0700 Subject: [PATCH 115/323] -> Sample and case metadatadata fetching --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 64d295e8..8a88937a 100644 --- a/settings.py +++ b/settings.py @@ -25,7 +25,7 @@ if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') -load_dotenv(dotenv_path=join(dirname(__file__), env_path+'api.env')) +load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' From c2236f0e0ecd432c6970613b8bb1d4a5197ffde2 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 27 Mar 2019 19:33:10 -0700 Subject: [PATCH 116/323] -> Sample and case metadatadata fetching --- apiv4/sample_case_views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apiv4/sample_case_views.py b/apiv4/sample_case_views.py index 942145c7..5b1b9ebd 100644 --- a/apiv4/sample_case_views.py +++ b/apiv4/sample_case_views.py @@ -23,13 +23,13 @@ from flask import request from django.conf import settings -from cohorts.metadata_helpers import get_sample_case_metadata +from cohorts.metadata_helpers import get_full_case_metadata, get_full_sample_metadata logger = logging.getLogger(settings.LOGGER_NAME) def get_sample_metadata(sample_barcodes): - metadata = get_sample_case_metadata(sample_barcodes, False) + metadata = get_full_sample_metadata(sample_barcodes) if metadata['total_found']: return metadata @@ -38,7 +38,7 @@ def get_sample_metadata(sample_barcodes): def get_case_metadata(case_barcodes): - metadata = get_sample_case_metadata(case_barcodes, True) + metadata = get_full_case_metadata(case_barcodes) if metadata['total_found']: return metadata From 57b51cf7bda643fc9882fdb68f034c08097e5194 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 28 Mar 2019 00:53:33 -0700 Subject: [PATCH 117/323] -> Sample and case metadatadata fetching --- apiv4/sample_case_routes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apiv4/sample_case_routes.py b/apiv4/sample_case_routes.py index 5852391a..91058bdd 100644 --- a/apiv4/sample_case_routes.py +++ b/apiv4/sample_case_routes.py @@ -21,7 +21,7 @@ from flask import jsonify, request from apiv4 import app from django.conf import settings -from sample_case_views import get_full_sample_metadata, get_case_metadata +from sample_case_views import get_full_sample_metadata, get_full_case_metadata logger = logging.getLogger(settings.LOGGER_NAME) @@ -36,7 +36,7 @@ def sample_metadata(sample_barcode): if sample_metadata: response = jsonify({ 'code': 200, - 'data': metadata + 'data': metadata['samples'][0] }) response.status_code = 200 else: @@ -52,12 +52,12 @@ def sample_metadata(sample_barcode): def case_metadata(case_barcode): response = None - metadata = get_case_metadata([case_barcode]) + metadata = get_full_case_metadata([case_barcode]) if metadata: response = jsonify({ 'code': 200, - 'data': metadata + 'data': metadata['cases'][0] }) response.status_code = 200 else: @@ -98,7 +98,7 @@ def case_metadata_list(): request_data = request.get_json() - metadata = get_case_metadata(request_data['case_barcodes']) + metadata = get_full_case_metadata(request_data['case_barcodes']) if case_metadata: response = jsonify({ From 3ae5839526858f90f48c2d28b1e17df988a16ad4 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 29 Mar 2019 19:10:35 -0700 Subject: [PATCH 118/323] -> Swap to new case list builder --- apiv4/cohorts_views.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index e6bf38c0..8db668e3 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -30,7 +30,7 @@ from cohorts.models import Cohort_Perms, Cohort, Filters from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files -from cohorts.metadata_helpers import get_sample_case_list +from cohorts.metadata_helpers import get_sample_case_list_bq from projects.models import Program logger = logging.getLogger(settings.LOGGER_NAME) @@ -136,27 +136,7 @@ def get_cohort_counts(): try: request_data = request.get_json() - print("Data: " + str(request_data)) - for prog_name in request_data['filters']: - try: - prog_filters = None - this_program = Program.objects.get(name=prog_name.upper(), is_public=1, active=1) - if request_data['filters'][prog_name]: - prog_filters = request_data['filters'][prog_name] - prog_counts = get_sample_case_list(None, prog_filters, None, this_program.id) - if prog_counts: - if not cohort_counts: - cohort_counts = {} - cohort_counts[prog_name] = prog_counts - else: - logger.warn("Could not obtain program counts for {} - skipping.".format( - prog_name - )) - except ObjectDoesNotExist as e: - logger.error("Program {} was not found as an active, public program - skipping.".format( - prog_name - )) - continue + cohort_counts = get_sample_case_list_bq(None, request_data['filters']) except Exception as e: logger.exception(e) From c2ad83b1db74be420efa074806129d54eb7d6209 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 1 Apr 2019 14:49:45 -0700 Subject: [PATCH 119/323] -> Add error returning for cohort preview --- apiv4/cohorts_routes.py | 19 ++++++++++++++----- apiv4/cohorts_views.py | 2 +- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 71b58b02..7d1e00a1 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -125,11 +125,20 @@ def cohort_preview(): cohort_counts = get_cohort_counts() if cohort_counts: - response = jsonify({ - 'code': 200, - 'data': cohort_counts - }) - response.status_code = 200 + # Presence of a message means something went wrong with the filters we received + if 'msg' in cohort_counts: + response = jsonify({ + 'code': 400, + 'data': cohort_counts + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 200, + 'data': cohort_counts + }) + response.status_code = 200 + # Lack of a valid object means something went wrong on the server else: response = jsonify({ 'code': 500, diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 8db668e3..9d257ccf 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -139,6 +139,6 @@ def get_cohort_counts(): cohort_counts = get_sample_case_list_bq(None, request_data['filters']) except Exception as e: logger.exception(e) - + return cohort_counts From 067c2312190c8d49680534f5646b535e87b7f11f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 2 Apr 2019 12:17:28 -0700 Subject: [PATCH 120/323] -> JSON schema additions --- apiv4/cohorts_views.py | 11 +++++-- apiv4/schemas/cohort_filter_schema.py | 41 +++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 apiv4/schemas/cohort_filter_schema.py diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 9d257ccf..22d66cca 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -33,8 +33,10 @@ from cohorts.metadata_helpers import get_sample_case_list_bq from projects.models import Program -logger = logging.getLogger(settings.LOGGER_NAME) +from jsonschema import validate as schema_validate, ValidationError +from schemas.cohort_filter_schema import COHORT_FILTER_SCHEMA +logger = logging.getLogger(settings.LOGGER_NAME) def validate_user(user_email, cohort_id): user = None @@ -136,9 +138,14 @@ def get_cohort_counts(): try: request_data = request.get_json() + schema_validate(request_data, COHORT_FILTER_SCHEMA) cohort_counts = get_sample_case_list_bq(None, request_data['filters']) + except ValidationError as e: + logger.warn("Filters rejected for improper formatting: {}".format(e)) + cohort_counts = { + 'msg': "Filters were improperly formatted." + } except Exception as e: logger.exception(e) - return cohort_counts diff --git a/apiv4/schemas/cohort_filter_schema.py b/apiv4/schemas/cohort_filter_schema.py new file mode 100644 index 00000000..a63861c6 --- /dev/null +++ b/apiv4/schemas/cohort_filter_schema.py @@ -0,0 +1,41 @@ +COHORT_FILTER_SCHEMA = { + 'type': 'object', + 'properties': { + 'filters': { + 'type': 'object', + 'properties': { + 'TCGA': { + 'type': 'object', + 'properties': { + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + } + } + }, + 'TARGET': { + 'type': 'object', + 'properties': { + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + } + } + }, + 'CCLE': { + 'type': 'object', + 'properties': { + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + } + } + }, + }, + "additionalProperties": False + } + }, + 'required': [ + 'filters' + ] + } From ad4d75e3c43f2ee2ac61112879609d740127f7a5 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 3 Apr 2019 00:04:06 -0700 Subject: [PATCH 121/323] -> First attempt at full openapi doc AND JSON schema --- apiv4/cohorts_views.py | 5 + apiv4/schemas/cohort_filter_schema.py | 162 +++++++++++++++++++++++++- 2 files changed, 164 insertions(+), 3 deletions(-) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 22d66cca..e282c9e8 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -140,6 +140,11 @@ def get_cohort_counts(): request_data = request.get_json() schema_validate(request_data, COHORT_FILTER_SCHEMA) cohort_counts = get_sample_case_list_bq(None, request_data['filters']) + + for prog in cohort_counts: + if cohort_counts[prog]['case_count'] <= 0: + cohort_counts[prog]['msg'] = "No cases or samples found which meet the filter criteria for this program." + cohort_counts[prog]['provided_filters'] = request_data['filters'][prog] except ValidationError as e: logger.warn("Filters rejected for improper formatting: {}".format(e)) cohort_counts = { diff --git a/apiv4/schemas/cohort_filter_schema.py b/apiv4/schemas/cohort_filter_schema.py index a63861c6..a58bdb6e 100644 --- a/apiv4/schemas/cohort_filter_schema.py +++ b/apiv4/schemas/cohort_filter_schema.py @@ -7,10 +7,165 @@ 'TCGA': { 'type': 'object', 'properties': { + 'avg_percent_lymphocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'batch_number': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'bcr': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'days_to_collection': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_sample_procurement': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, 'disease_code': { - 'type': ['array', 'string'], + 'type': ['array', 'string'], 'items': { 'type': 'string' } - } + }, + 'max_percent_lymphocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'metadata_biospecimen_id': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_lymphocyte_infiltration': { + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'pathology_report_uuid': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'preservation_method': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, } }, 'TARGET': { @@ -37,5 +192,6 @@ }, 'required': [ 'filters' - ] + ], + "additionalProperties": False } From 6f7c88f36ac011c7cba797868ae4f913049c6b7e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 3 Apr 2019 15:16:36 -0700 Subject: [PATCH 122/323] -> Fill out filter schemas for Cohorts and File Manifests --- apiv4/schemas/cohort_filter_schema.py | 1083 ++++++++++++++++++++++++- apiv4/schemas/file_filter_schema.py | 127 +++ 2 files changed, 1193 insertions(+), 17 deletions(-) create mode 100644 apiv4/schemas/file_filter_schema.py diff --git a/apiv4/schemas/cohort_filter_schema.py b/apiv4/schemas/cohort_filter_schema.py index a58bdb6e..1f4616c4 100644 --- a/apiv4/schemas/cohort_filter_schema.py +++ b/apiv4/schemas/cohort_filter_schema.py @@ -39,6 +39,102 @@ 'type': ['array', 'string', 'number'], 'items': { 'type': ['string', 'number'] } }, + 'avg_percent_lymphocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_lymphocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_lymphocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, 'batch_number': { 'type': ['array', 'string', 'integer'], 'items': { 'type': ['string', 'integer'] } @@ -99,11 +195,104 @@ 'type': ['array', 'string', 'number'], 'items': { 'type': ['string', 'number'] } }, - 'metadata_biospecimen_id': { + 'max_percent_lymphocyte_infiltration_lte': { 'type': ['array', 'string', 'number'], 'items': { 'type': ['string', 'number'] } }, + 'max_percent_monocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_lymphocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_lymphocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, 'min_percent_lymphocyte_infiltration': { + 'type': ['array', 'string', 'number'], 'items': { 'type': ['string', 'number'] } }, 'min_percent_monocyte_infiltration': { @@ -142,6 +331,126 @@ 'type': ['array', 'string', 'integer'], 'items': { 'type': ['string', 'integer'] } }, + 'min_percent_lymphocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'min_percent_lymphocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'min_percent_lymphocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, 'pathology_report_uuid': { 'type': ['array', 'string'], 'items': { 'type': 'string' } @@ -166,24 +475,764 @@ 'type': ['array', 'string'], 'items': { 'type': 'string' } }, - } - }, - 'TARGET': { - 'type': 'object', - 'properties': { - 'disease_code': { + 'weight_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_tobacco_smoking_onset_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_began_smoking_in_years_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'anatomic_neoplasm_subdivision_gte': { 'type': ['array', 'string'], - 'items': { 'type': 'string' } - } - } - }, - 'CCLE': { - 'type': 'object', - 'properties': { - 'disease_code': { + 'items': {'type': 'string'} + }, + 'batch_number_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'bmi_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'days_to_birth_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_initial_pathologic_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_submitted_specimen_dx_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'height_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'psa_value_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'gleason_score_combined_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'weight_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_tobacco_smoking_onset_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_began_smoking_in_years_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'anatomic_neoplasm_subdivision_lte': { 'type': ['array', 'string'], - 'items': { 'type': 'string' } - } + 'items': {'type': 'string'} + }, + 'batch_number_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'bmi_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'days_to_birth_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_initial_pathologic_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_submitted_specimen_dx_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'height_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'psa_value_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'gleason_score_combined_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'weight_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'year_of_tobacco_smoking_onset_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'age_began_smoking_in_years_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'anatomic_neoplasm_subdivision_btw': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'batch_number_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'bmi_btw': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'days_to_birth_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_initial_pathologic_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_submitted_specimen_dx_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'height_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'psa_value_btw': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'gleason_score_combined_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'weight': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'year_of_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'year_of_tobacco_smoking_onset': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'age_at_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'age_began_smoking_in_years': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'anatomic_neoplasm_subdivision': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'batch_number': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'bmi': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'bcr': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'days_to_birth': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_death': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_initial_pathologic_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_last_followup': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_last_known_alive': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_submitted_specimen_dx': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'height': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'psa_value': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'gleason_score_combined': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'clinical_M': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'clinical_N': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'clinical_stage': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'clinical_T': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'colorectal_cancer': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'country': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'histological_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'history_of_colon_polyps': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'history_of_neoadjuvant_treatment': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'hpv_calls': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'hpv_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'h_pylori_infection': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'icd_10': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'icd_o_3_histology': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'icd_o_3_site': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'lymphatic_invasion': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'lymphnodes_examined': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'lymphovascular_invasion_present': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'menopause_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'mononucleotide_and_dinucleotide_marker_panel_analysis_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'neoplasm_histologic_grade': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'new_tumor_event_after_initial_treatment': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'number_of_lymphnodes_examined': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'number_of_lymphnodes_positive_by_he': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'number_pack_years_smoked': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_dx': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_malignancy_anatomic_site': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_malignancy_histological_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_malignancy_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_M': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_N': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_stage': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_T': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'person_neoplasm_cancer_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pregnancies': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'primary_neoplasm_melanoma_dx': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'primary_therapy_outcome_success': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'race': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'residual_tumor': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'stopped_smoking_year': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tobacco_smoking_history': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tss_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tumor_tissue_site': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tumor_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'venous_invasion': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'vital_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'ethnicity': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'gender': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + } + }, + 'TARGET': { + 'type': 'object', + 'properties': { + 'age_at_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_type': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tumor_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'days_to_birth_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_birth_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_birth_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'ethnicity': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'gender': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'year_of_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_followup_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_followup_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_followup': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_followup_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'wbc_at_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'wbc_at_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'wbc_at_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'wbc_at_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'vital_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'race': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'summary_file_count': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'protocol': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'first_event': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'event_free_survival': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + } + }, + 'CCLE': { + 'type': 'object', + 'properties': { + 'project_short_name': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_type': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'summary_file_count': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'gender': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'hist_subtype': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'histology': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'site_primary': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'source': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, } }, }, diff --git a/apiv4/schemas/file_filter_schema.py b/apiv4/schemas/file_filter_schema.py new file mode 100644 index 00000000..13e24cb9 --- /dev/null +++ b/apiv4/schemas/file_filter_schema.py @@ -0,0 +1,127 @@ +FILE_FILTER_SCHEMA = { + 'type': 'object', + 'properties': { + 'filters': { + 'type': 'object', + 'properties': { + 'access': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'acl': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'data_category': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'data_format': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'data_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'experimental_strategy': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'file_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'file_name_key': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'index_file_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'index_file_name_key': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'platform': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'program_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'file_size': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'file_size_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'file_size_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'file_size_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'fetch_count': { + 'type': 'integer' + }, + 'offset': { + 'type': 'integer' + }, + 'genomic_build': { + 'type': 'string' + }, + } + } + } +} \ No newline at end of file From 9d51e66e569afe9cccadefcdeebb33132cc5c483 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 3 Apr 2019 18:53:06 -0700 Subject: [PATCH 123/323] -> Cohort creation and editing --- apiv4/cohorts_routes.py | 85 ++++++++++++++++++++++-- apiv4/cohorts_views.py | 93 +++++++++++++++++++++++++-- apiv4/schemas/cohort_filter_schema.py | 9 ++- 3 files changed, 175 insertions(+), 12 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 7d1e00a1..3b9665fa 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request from apiv4 import app -from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user, create_cohort from auth import auth_info from django.conf import settings @@ -59,6 +59,37 @@ def cohort(cohort_id): return response +@app.route('/apiv4/cohorts//', methods=['PATCH'], strict_slashes=False) +def cohort(cohort_id): + """Edit an extent cohort's filters, name, or description""" + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + response = None + + if not user: + response = jsonify({ + 'code': 403, + 'message': "User {} does not have access to cohort ID {}".format(user_info['email'] if 'email' in user_info else 'Anonymous',str(cohort_id))}) + response.status_code = 403 + + else: + cohort_info = edit_cohort(cohort_id) + if cohort_info: + response = jsonify({ + 'code': 200, + 'data': cohort_info + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Cohort ID {} was not found.".format(str(cohort_id))}) + response.status_code = 404 + + return response + + @app.route('/apiv4/cohorts/', methods=['GET'], strict_slashes=False) def cohorts(): """Retrieve a user's list of cohorts""" @@ -116,7 +147,7 @@ def cohort_file_manifest(cohort_id): return response -@app.route('/apiv4/cohorts/preview/', methods=['POST', 'GET'], strict_slashes=False) +@app.route('/apiv4/cohorts/preview/', methods=['POST'], strict_slashes=False) def cohort_preview(): """List the samples, cases, and counts a given set of cohort filters would produce""" @@ -131,7 +162,7 @@ def cohort_preview(): 'code': 400, 'data': cohort_counts }) - response.status_code = 200 + response.status_code = 400 else: response = jsonify({ 'code': 200, @@ -146,4 +177,50 @@ def cohort_preview(): }) response.status_code = 500 - return response \ No newline at end of file + return response + + +@app.route('/apiv4/cohorts/', methods=['POST'], strict_slashes=False) +def cohort_create(): + """Create a cohort based on a set of supplied filters""" + + response = None + + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + response = None + + if not user: + response = jsonify({ + 'code': 403, + 'message': "User not found - please register on the ISB-CGC WebApp first!" + }) + response.status_code = 403 + else: + cohort_info = create_cohort(user) + + if cohort_info: + if 'msg' in cohort_info: + # Presence of a message means something was wrong. + response = jsonify({ + 'code': 400, + 'data': cohort_info + }) + response.status_code = 400 + else: + response = jsonify({ + 'code': 200, + 'data': cohort_counts + }) + response.status_code = 200 + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to create this cohort." + }) + response.status_code = 500 + + return response + diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index e282c9e8..b062120a 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -19,6 +19,7 @@ import logging import json import django +import re from flask import request @@ -31,11 +32,14 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from cohorts.file_helpers import cohort_files from cohorts.metadata_helpers import get_sample_case_list_bq +from cohorts.utils import create_cohort as make_cohort from projects.models import Program from jsonschema import validate as schema_validate, ValidationError from schemas.cohort_filter_schema import COHORT_FILTER_SCHEMA +BLACKLIST_RE = settings.BLACKLIST_RE + logger = logging.getLogger(settings.LOGGER_NAME) def validate_user(user_email, cohort_id): @@ -139,12 +143,18 @@ def get_cohort_counts(): try: request_data = request.get_json() schema_validate(request_data, COHORT_FILTER_SCHEMA) - cohort_counts = get_sample_case_list_bq(None, request_data['filters']) - for prog in cohort_counts: - if cohort_counts[prog]['case_count'] <= 0: - cohort_counts[prog]['msg'] = "No cases or samples found which meet the filter criteria for this program." - cohort_counts[prog]['provided_filters'] = request_data['filters'][prog] + if 'filters' not in request_data: + cohort_counts = { + 'msg': 'No filters were provided; ensure that the request body contains a \'filters\' property.' + } + else: + cohort_counts = get_sample_case_list_bq(None, request_data['filters']) + + for prog in cohort_counts: + if cohort_counts[prog]['case_count'] <= 0: + cohort_counts[prog]['msg'] = "No cases or samples found which meet the filter criteria for this program." + cohort_counts[prog]['provided_filters'] = request_data['filters'][prog] except ValidationError as e: logger.warn("Filters rejected for improper formatting: {}".format(e)) cohort_counts = { @@ -154,3 +164,76 @@ def get_cohort_counts(): logger.exception(e) return cohort_counts + +def create_cohort(user): + cohort_info = None + + try: + request_data = request.get_json() + schema_validate(request_data, COHORT_FILTER_SCHEMA) + + if 'name' not in request_data: + cohort_info = { + 'msg': 'A name was not provided for this cohort. Cohort was not made.' + } + return cohort_info + + if 'filters' not in request_data: + cohort_info = { + 'msg': 'Filters were not provided; at least one filter must be provided for a cohort to be valid. Cohort was not made.' + } + return cohort_info + + name = request_data['name'] + filters = request_data['filters'] + + blacklist = re.compile(BLACKLIST_RE, re.UNICODE) + match = blacklist.search(str(name)) + if match: + # XSS risk, log and fail this cohort save + match = blacklist.findall(str(name)) + cohort_info = { + 'msg': "Your cohort's name contains invalid characters; please choose another name. [Saw {}]".format(str(match)) + } + else: + desc = None + if 'description' in request_data: + desc = request_data['description'] + + cohort_info = make_cohort(user, filters, name, desc) + + except ValidationError as e: + logger.warn("Filters rejected for improper formatting: {}".format(e)) + cohort_info = { + 'msg': "Filters were improperly formatted - cohort not created." + } + + return cohort_info + + +def edit_cohort(cohort_id): + cohort_info = None + + try: + request_data = request.get_json() + schema_validate(request_data, COHORT_FILTER_SCHEMA) + + name = request_data['name'] + blacklist = re.compile(BLACKLIST_RE, re.UNICODE) + match = blacklist.search(str(name)) + if match: + # XSS risk, log and fail this cohort save + match = blacklist.findall(str(name)) + cohort_info = { + 'msg': "Your cohort's name contains invalid characters; please choose another name. [Saw {}]".format(str(match)) + } + else: + logger.warn("Make cohort") + except ValidationError as e: + logger.warn("Filters rejected for improper formatting: {}".format(e)) + cohort_info = { + 'msg': "Filters were improperly formatted - cohort not created." + } + + return cohort_info + diff --git a/apiv4/schemas/cohort_filter_schema.py b/apiv4/schemas/cohort_filter_schema.py index 1f4616c4..d6d4b1b0 100644 --- a/apiv4/schemas/cohort_filter_schema.py +++ b/apiv4/schemas/cohort_filter_schema.py @@ -1237,10 +1237,13 @@ }, }, "additionalProperties": False + }, + 'name': { + 'type': 'string' + }, + 'desc': { + 'type': 'string' } }, - 'required': [ - 'filters' - ], "additionalProperties": False } From 3dddc87ece7870c35d02c52a6c9f123b96b0f9b1 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 4 Apr 2019 13:55:59 -0700 Subject: [PATCH 124/323] -> Cohort creation --- apiv4/cohorts_routes.py | 40 +++++++++++++++++++++++++++++----------- apiv4/cohorts_views.py | 30 +++++++++++++++++++++++------- settings.py | 2 +- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 3b9665fa..03571d6c 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request from apiv4 import app -from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user, create_cohort +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user, create_cohort, edit_cohort from auth import auth_info from django.conf import settings @@ -36,12 +36,18 @@ def cohort(cohort_id): response = None - if not user: + if 'msg' in user: response = jsonify({ 'code': 403, - 'message': "User {} does not have access to cohort ID {}".format(user_info['email'] if 'email' in user_info else 'Anonymous',str(cohort_id))}) + 'message': user['msg'] + }) response.status_code = 403 - + elif not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 else: cohort_info = get_cohort_info(cohort_id) if cohort_info: @@ -60,7 +66,7 @@ def cohort(cohort_id): @app.route('/apiv4/cohorts//', methods=['PATCH'], strict_slashes=False) -def cohort(cohort_id): +def cohort_edit(cohort_id): """Edit an extent cohort's filters, name, or description""" user_info = auth_info() user = validate_user(user_info['email'], cohort_id) @@ -123,12 +129,18 @@ def cohort_file_manifest(cohort_id): response = None - if not user: + if 'msg' in user: response = jsonify({ 'code': 403, - 'message': "User {} does not have access to cohort ID {}".format(user_info['email'],str(cohort_id))}) + 'message': user['msg'] + }) response.status_code = 403 - + elif not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 else: file_manifest = get_file_manifest(cohort_id, user) if file_manifest: @@ -187,16 +199,22 @@ def cohort_create(): response = None user_info = auth_info() - user = validate_user(user_info['email'], cohort_id) + user = validate_user(user_info['email']) response = None - if not user: + if 'msg' in user: response = jsonify({ 'code': 403, - 'message': "User not found - please register on the ISB-CGC WebApp first!" + 'message': user['msg'] }) response.status_code = 403 + elif not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 else: cohort_info = create_cohort(user) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index b062120a..34a2e914 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -42,17 +42,28 @@ logger = logging.getLogger(settings.LOGGER_NAME) -def validate_user(user_email, cohort_id): + +def validate_user(user_email, cohort_id=None): user = None + django.setup() try: - user = Django_User.objects.get(email=user_email) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) - except ObjectDoesNotExist as e: - logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) + try: + user = Django_User.objects.get(email=user_email) + except ObjectDoesNotExist as e: + logger.warn("User {} does not exist in our system.".format(user_email)) + return { 'msg': "User {} does not exist in our system. Please register with our Web Application first: ".format(user_email) } + + try: + if cohort_id: + Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) + except ObjectDoesNotExist as e: + logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) + return {'msg': "User {} does not have access to cohort {}".format(user_email, cohort_id)} + except Exception as e: logger.exception(e) - + return user @@ -200,7 +211,12 @@ def create_cohort(user): if 'description' in request_data: desc = request_data['description'] - cohort_info = make_cohort(user, filters, name, desc) + result = make_cohort(user, filters, name, desc) + + if 'msg' in result: + cohort_infp = result + else: + cohort_info = get_cohort_info(cohort_info['cohort_id']) except ValidationError as e: logger.warn("Filters rejected for improper formatting: {}".format(e)) diff --git a/settings.py b/settings.py index 8a88937a..64d295e8 100644 --- a/settings.py +++ b/settings.py @@ -25,7 +25,7 @@ if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') -load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) +load_dotenv(dotenv_path=join(dirname(__file__), env_path+'api.env')) APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' From eac29606a853125a8be66923537bc48b18330b99 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 4 Apr 2019 14:08:56 -0700 Subject: [PATCH 125/323] -> Cohort creation --- settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.py b/settings.py index 64d295e8..8a88937a 100644 --- a/settings.py +++ b/settings.py @@ -25,7 +25,7 @@ if os.environ.get('SECURE_LOCAL_PATH', None): env_path += os.environ.get('SECURE_LOCAL_PATH') -load_dotenv(dotenv_path=join(dirname(__file__), env_path+'api.env')) +load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' From e37f69f276ebfbdbfa3c1d01ff6e118f0f5dce1c Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 4 Apr 2019 14:33:57 -0700 Subject: [PATCH 126/323] -> Error updates. --- apiv4/cohorts_views.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 34a2e914..20258e2b 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -52,7 +52,10 @@ def validate_user(user_email, cohort_id=None): user = Django_User.objects.get(email=user_email) except ObjectDoesNotExist as e: logger.warn("User {} does not exist in our system.".format(user_email)) - return { 'msg': "User {} does not exist in our system. Please register with our Web Application first: ".format(user_email) } + return { + 'msg': "User {} does not exist in our system.".format(user_email) + + " Please register with our Web Application first: " + } try: if cohort_id: @@ -167,9 +170,9 @@ def get_cohort_counts(): cohort_counts[prog]['msg'] = "No cases or samples found which meet the filter criteria for this program." cohort_counts[prog]['provided_filters'] = request_data['filters'][prog] except ValidationError as e: - logger.warn("Filters rejected for improper formatting: {}".format(e)) + logger.warn('Filters rejected for improper formatting: {}'.format(e)) cohort_counts = { - 'msg': "Filters were improperly formatted." + 'msg': 'Filters were improperly formatted.' } except Exception as e: logger.exception(e) @@ -191,7 +194,8 @@ def create_cohort(user): if 'filters' not in request_data: cohort_info = { - 'msg': 'Filters were not provided; at least one filter must be provided for a cohort to be valid. Cohort was not made.' + 'msg': 'Filters were not provided; at least one filter must be provided for a cohort to be valid.' + + ' Cohort was not made.' } return cohort_info @@ -204,7 +208,8 @@ def create_cohort(user): # XSS risk, log and fail this cohort save match = blacklist.findall(str(name)) cohort_info = { - 'msg': "Your cohort's name contains invalid characters; please choose another name. [Saw {}]".format(str(match)) + 'msg': 'Your cohort\'s name contains invalid characters; please choose another name. ' + + '[Saw {}]'.format(str(match)) } else: desc = None @@ -221,7 +226,7 @@ def create_cohort(user): except ValidationError as e: logger.warn("Filters rejected for improper formatting: {}".format(e)) cohort_info = { - 'msg': "Filters were improperly formatted - cohort not created." + 'msg': 'Filters were improperly formatted - cohort not created.' } return cohort_info @@ -241,15 +246,15 @@ def edit_cohort(cohort_id): # XSS risk, log and fail this cohort save match = blacklist.findall(str(name)) cohort_info = { - 'msg': "Your cohort's name contains invalid characters; please choose another name. [Saw {}]".format(str(match)) + 'msg': 'Your cohort\'s name contains invalid characters; please choose another name.' + + ' [Saw {}]'.format(str(match)) } else: logger.warn("Make cohort") except ValidationError as e: logger.warn("Filters rejected for improper formatting: {}".format(e)) cohort_info = { - 'msg': "Filters were improperly formatted - cohort not created." + 'msg': 'Filters were improperly formatted - cohort not created.' } return cohort_info - From d3dafb3b25302c579894d6eadf246bf0d1dc0e94 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 4 Apr 2019 15:23:12 -0700 Subject: [PATCH 127/323] -> Collapse cohorts resource route --- apiv4/cohorts_routes.py | 103 +++++++++++++++------------------------- 1 file changed, 39 insertions(+), 64 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 03571d6c..678f9e5a 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -96,27 +96,54 @@ def cohort_edit(cohort_id): return response -@app.route('/apiv4/cohorts/', methods=['GET'], strict_slashes=False) +@app.route('/apiv4/cohorts/', methods=['GET', 'POST'], strict_slashes=False) def cohorts(): """Retrieve a user's list of cohorts""" user_info = auth_info() - - cohort_list = get_cohorts(user_info['email']) + user = validate_user(user_info['email']) response = None + info = None - if not cohort_list: + if 'msg' in user: response = jsonify({ - 'code': 404, - 'message': "No cohorts were found for user {}".format(user_info['email'])}) - response.status_code = 404 - - else: + 'code': 403, + 'message': user['msg'] + }) + response.status_code = 403 + elif not user: response = jsonify({ - 'code': 200, - 'data': cohort_list + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' }) - response.status_code = 200 + response.status_code = 500 + else: + if request.method == 'GET': + info = get_cohorts(user_info['email']) + else: + info = create_cohort(user) + + if info: + if 'msg' in info: + # Presence of a message means something was wrong. + response = jsonify({ + 'code': 400, + 'data': info + }) + response.status_code = 400 + else: + response = jsonify({ + 'code': 200, + 'data': info + }) + response.status_code = 200 + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to create this cohort." + }) + response.status_code = 500 return response @@ -190,55 +217,3 @@ def cohort_preview(): response.status_code = 500 return response - - -@app.route('/apiv4/cohorts/', methods=['POST'], strict_slashes=False) -def cohort_create(): - """Create a cohort based on a set of supplied filters""" - - response = None - - user_info = auth_info() - user = validate_user(user_info['email']) - - response = None - - if 'msg' in user: - response = jsonify({ - 'code': 403, - 'message': user['msg'] - }) - response.status_code = 403 - elif not user: - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while attempting to identify this user.' - }) - response.status_code = 500 - else: - cohort_info = create_cohort(user) - - if cohort_info: - if 'msg' in cohort_info: - # Presence of a message means something was wrong. - response = jsonify({ - 'code': 400, - 'data': cohort_info - }) - response.status_code = 400 - else: - response = jsonify({ - 'code': 200, - 'data': cohort_counts - }) - response.status_code = 200 - # Lack of a valid object means something went wrong on the server - else: - response = jsonify({ - 'code': 500, - 'message': "Error while attempting to create this cohort." - }) - response.status_code = 500 - - return response - From efb9ea466f05d11c1c9dc310c363c35b9b081a5d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 4 Apr 2019 16:10:21 -0700 Subject: [PATCH 128/323] -> Fix routes --- apiv4/cohorts_routes.py | 52 ++++++++++++++--------------------------- 1 file changed, 17 insertions(+), 35 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 678f9e5a..0c10af77 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -27,11 +27,14 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/cohorts//', methods=['GET'], strict_slashes=False) +@app.route('/apiv4/cohorts//', methods=['GET', 'PATCH'], strict_slashes=False) def cohort(cohort_id): - """Retrieve extended information for a specific cohort""" + """ + GET: Retrieve extended information for a specific cohort + PATCH: Edit an extent cohort + """ + user_info = auth_info() - logger.info("[STATUS] User info: {}".format(str(user_info))) user = validate_user(user_info['email'], cohort_id) response = None @@ -49,38 +52,11 @@ def cohort(cohort_id): }) response.status_code = 500 else: - cohort_info = get_cohort_info(cohort_id) - if cohort_info: - response = jsonify({ - 'code': 200, - 'data': cohort_info - }) - response.status_code = 200 + if request.method == 'GET': + cohort_info = get_cohort_info(cohort_id) else: - response = jsonify({ - 'code': 404, - 'message': "Cohort ID {} was not found.".format(str(cohort_id))}) - response.status_code = 404 + cohort_info = edit_cohort(cohort_id) - return response - - -@app.route('/apiv4/cohorts//', methods=['PATCH'], strict_slashes=False) -def cohort_edit(cohort_id): - """Edit an extent cohort's filters, name, or description""" - user_info = auth_info() - user = validate_user(user_info['email'], cohort_id) - - response = None - - if not user: - response = jsonify({ - 'code': 403, - 'message': "User {} does not have access to cohort ID {}".format(user_info['email'] if 'email' in user_info else 'Anonymous',str(cohort_id))}) - response.status_code = 403 - - else: - cohort_info = edit_cohort(cohort_id) if cohort_info: response = jsonify({ 'code': 200, @@ -98,7 +74,10 @@ def cohort_edit(cohort_id): @app.route('/apiv4/cohorts/', methods=['GET', 'POST'], strict_slashes=False) def cohorts(): - """Retrieve a user's list of cohorts""" + """ + GET: Retrieve a user's list of cohorts + POST: Add a new cohort + """ user_info = auth_info() user = validate_user(user_info['email']) @@ -150,7 +129,10 @@ def cohorts(): @app.route('/apiv4/cohorts//file_manifest/', methods=['POST', 'GET'], strict_slashes=False) def cohort_file_manifest(cohort_id): - """Retrieve a cohort's file manifest""" + """ + GET: Retrieve a cohort's file manifest + POST: Retrieve a cohort's file manifest with applied filters + """ user_info = auth_info() user = validate_user(user_info['email'], cohort_id) From 2c86cba60279f0f8726984a3aeb080dee61c99eb Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 11:34:49 -0700 Subject: [PATCH 129/323] -> Adjust user validation return --- apiv4/cohorts_routes.py | 2 ++ apiv4/cohorts_views.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 0c10af77..f9435ec9 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -97,6 +97,7 @@ def cohorts(): }) response.status_code = 500 else: + user = user['user'] if request.method == 'GET': info = get_cohorts(user_info['email']) else: @@ -151,6 +152,7 @@ def cohort_file_manifest(cohort_id): }) response.status_code = 500 else: + user = user['user'] file_manifest = get_file_manifest(cohort_id, user) if file_manifest: response = jsonify({ diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 20258e2b..c718273f 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -67,7 +67,7 @@ def validate_user(user_email, cohort_id=None): except Exception as e: logger.exception(e) - return user + return {'user': user} def get_file_manifest(cohort_id, user): From bdf54682f5a6ff59859545d3e66ad8146dfe23d7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 14:23:04 -0700 Subject: [PATCH 130/323] -> Switch around auth error management --- apiv4/cohorts_routes.py | 187 +++++++++++++++++++++++----------------- apiv4/cohorts_views.py | 41 +++++---- 2 files changed, 130 insertions(+), 98 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index f9435ec9..2c97d9fa 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -34,40 +34,49 @@ def cohort(cohort_id): PATCH: Edit an extent cohort """ - user_info = auth_info() - user = validate_user(user_info['email'], cohort_id) + try: + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) - response = None + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + if request.method == 'GET': + cohort_info = get_cohort_info(cohort_id) + else: + cohort_info = edit_cohort(cohort_id) + + if cohort_info: + response = jsonify({ + 'code': 200, + 'data': cohort_info + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Cohort ID {} was not found.".format(str(cohort_id))}) + response.status_code = 404 - if 'msg' in user: + except UserValidationException as e: response = jsonify({ 'code': 403, - 'message': user['msg'] + 'message': str(e) }) response.status_code = 403 - elif not user: + + except Exception as e: response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to identify this user.' }) response.status_code = 500 - else: - if request.method == 'GET': - cohort_info = get_cohort_info(cohort_id) - else: - cohort_info = edit_cohort(cohort_id) - - if cohort_info: - response = jsonify({ - 'code': 200, - 'data': cohort_info - }) - response.status_code = 200 - else: - response = jsonify({ - 'code': 404, - 'message': "Cohort ID {} was not found.".format(str(cohort_id))}) - response.status_code = 404 return response @@ -78,52 +87,62 @@ def cohorts(): GET: Retrieve a user's list of cohorts POST: Add a new cohort """ - user_info = auth_info() - user = validate_user(user_info['email']) - response = None - info = None + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + response = None + info = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + user = user['user'] + if request.method == 'GET': + info = get_cohorts(user_info['email']) + else: + info = create_cohort(user) + + if info: + if 'msg' in info: + # Presence of a message means something was wrong. + response = jsonify({ + 'code': 400, + 'data': info + }) + response.status_code = 400 + else: + response = jsonify({ + 'code': 200, + 'data': info + }) + response.status_code = 200 + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to create this cohort." + }) + response.status_code = 500 - if 'msg' in user: + except UserValidationException as e: response = jsonify({ 'code': 403, - 'message': user['msg'] + 'message': str(e) }) response.status_code = 403 - elif not user: + + except Exception as e: response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to identify this user.' }) response.status_code = 500 - else: - user = user['user'] - if request.method == 'GET': - info = get_cohorts(user_info['email']) - else: - info = create_cohort(user) - - if info: - if 'msg' in info: - # Presence of a message means something was wrong. - response = jsonify({ - 'code': 400, - 'data': info - }) - response.status_code = 400 - else: - response = jsonify({ - 'code': 200, - 'data': info - }) - response.status_code = 200 - # Lack of a valid object means something went wrong on the server - else: - response = jsonify({ - 'code': 500, - 'message': "Error while attempting to create this cohort." - }) - response.status_code = 500 return response @@ -134,38 +153,48 @@ def cohort_file_manifest(cohort_id): GET: Retrieve a cohort's file manifest POST: Retrieve a cohort's file manifest with applied filters """ - user_info = auth_info() - user = validate_user(user_info['email'], cohort_id) - response = None + try: + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + response = None - if 'msg' in user: + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + user = user['user'] + file_manifest = get_file_manifest(cohort_id, user) + if file_manifest: + response = jsonify({ + 'code': 200, + 'data': file_manifest + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)) + }) + response.status_code = 500 + + except UserValidationException as e: response = jsonify({ 'code': 403, - 'message': user['msg'] + 'message': str(e) }) response.status_code = 403 - elif not user: + + except Exception as e: response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to identify this user.' }) response.status_code = 500 - else: - user = user['user'] - file_manifest = get_file_manifest(cohort_id, user) - if file_manifest: - response = jsonify({ - 'code': 200, - 'data': file_manifest - }) - response.status_code = 200 - else: - response = jsonify({ - 'code': 500, - 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)) - }) - response.status_code = 500 return response diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index c718273f..c9491e40 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -43,31 +43,34 @@ logger = logging.getLogger(settings.LOGGER_NAME) +class UserValidationException(Exception): + pass + + def validate_user(user_email, cohort_id=None): user = None django.setup() try: - try: - user = Django_User.objects.get(email=user_email) - except ObjectDoesNotExist as e: - logger.warn("User {} does not exist in our system.".format(user_email)) - return { - 'msg': "User {} does not exist in our system.".format(user_email) + - " Please register with our Web Application first: " - } - - try: - if cohort_id: - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) - except ObjectDoesNotExist as e: - logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) - return {'msg': "User {} does not have access to cohort {}".format(user_email, cohort_id)} - - except Exception as e: - logger.exception(e) + user = Django_User.objects.get(email=user_email) + except ObjectDoesNotExist as e: + logger.warn("User {} does not exist in our system.".format(user_email)) + raise UserValidationException( + "User {} wasn't found in our system.".format(user_email) + + " Please register with our Web Application first: " + ) + + try: + if cohort_id: + Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) + except ObjectDoesNotExist as e: + logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) + raise UserValidationException( + "User {} does not have access to cohort {}.".format(user_email, cohort_id) + + " Please contact this cohort owner to obtain permission." + ) - return {'user': user} + return user def get_file_manifest(cohort_id, user): From e5f53332dc86f2993db9588b56ca5d0ef373bd89 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 14:42:11 -0700 Subject: [PATCH 131/323] -> Switch around auth error management --- apiv4/program_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/program_views.py b/apiv4/program_views.py index ed9df017..3acaa021 100644 --- a/apiv4/program_views.py +++ b/apiv4/program_views.py @@ -38,7 +38,7 @@ def get_programs(): { 'name': x.name, 'description': x.description, - 'projects': [{'name': y.name, 'description': y.description } for y in Project.objects.filter(program=x,active=1)] + 'projects': [{'name': y.name, 'description': y.description} for y in Project.objects.filter(program=x,active=1)] } for x in Program.get_public_programs() ] From b57f57c3e46af6a35002c3d62de10b7a1ec8663a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 15:00:28 -0700 Subject: [PATCH 132/323] -> Helps to import the exception... --- apiv4/cohorts_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 2c97d9fa..90cf204e 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -20,7 +20,7 @@ import json from flask import jsonify, request from apiv4 import app -from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user, create_cohort, edit_cohort +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user, create_cohort, edit_cohort, UserValidationException from auth import auth_info from django.conf import settings From 9a2ae485595fb77fc6e5632fdd98320eb20fce8f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 15:23:24 -0700 Subject: [PATCH 133/323] -> Also, report it. --- apiv4/cohorts_routes.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 90cf204e..8d542390 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -72,6 +72,7 @@ def cohort(cohort_id): response.status_code = 403 except Exception as e: + logger.exception(e) response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to identify this user.' @@ -138,6 +139,7 @@ def cohorts(): response.status_code = 403 except Exception as e: + logger.exception(e) response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to identify this user.' @@ -190,6 +192,7 @@ def cohort_file_manifest(cohort_id): response.status_code = 403 except Exception as e: + logger.exception(e) response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to identify this user.' From 64c02ac3493dd7f7060b65f3e80746b73a20cf73 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 15:41:14 -0700 Subject: [PATCH 134/323] -> Stop looking for old code. --- apiv4/cohorts_routes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 8d542390..cdd88dee 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -75,7 +75,7 @@ def cohort(cohort_id): logger.exception(e) response = jsonify({ 'code': 500, - 'message': 'Encountered an error while attempting to identify this user.' + 'message': 'Encountered an error while attempting to retrieve this cohort\'s information.' }) response.status_code = 500 @@ -103,7 +103,6 @@ def cohorts(): }) response.status_code = 500 else: - user = user['user'] if request.method == 'GET': info = get_cohorts(user_info['email']) else: @@ -142,7 +141,9 @@ def cohorts(): logger.exception(e) response = jsonify({ 'code': 500, - 'message': 'Encountered an error while attempting to identify this user.' + 'message': 'Encountered an error while attempting to {}.'.format( + "retrieve a list of cohorts" if request.method == 'GET' else "create this cohort." + ) }) response.status_code = 500 @@ -169,7 +170,6 @@ def cohort_file_manifest(cohort_id): }) response.status_code = 500 else: - user = user['user'] file_manifest = get_file_manifest(cohort_id, user) if file_manifest: response = jsonify({ From fad8d667c0cac056b756e944289aeb46b540f114 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 17:20:58 -0700 Subject: [PATCH 135/323] -> Typo in error --- apiv4/cohorts_routes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index cdd88dee..56531831 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -142,7 +142,7 @@ def cohorts(): response = jsonify({ 'code': 500, 'message': 'Encountered an error while attempting to {}.'.format( - "retrieve a list of cohorts" if request.method == 'GET' else "create this cohort." + "retrieve a list of cohorts" if request.method == 'GET' else "create this cohort" ) }) response.status_code = 500 From 65de6d2667ed06b04d08a5ea591428b19414bdaf Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 18:22:21 -0700 Subject: [PATCH 136/323] -> Testing use of flask talisman --- apiv4/__init__.py | 2 ++ requirements.txt | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 92f9a33d..797c7c04 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -21,8 +21,10 @@ import sys from flask import Flask, jsonify, request from flask_cors import cross_origin +from flask_talisman import Talisman app = Flask(__name__) +Talisman(app, strict_transport_security_max_age=300) import django django.setup() diff --git a/requirements.txt b/requirements.txt index 29d062cd..50ac809c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,4 +22,5 @@ flask-cors==3.0.7 pyjwt==1.6.1 cryptography==2.4.2 future==0.17.1 -python-dotenv==0.10.0 \ No newline at end of file +python-dotenv==0.10.0 +flask-talisman==0.6.0 From 9fa1bc3db0f90265738241d32e16819edc2ea625 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 18:52:45 -0700 Subject: [PATCH 137/323] -> Testing without Talisman... --- apiv4/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 797c7c04..2e6b182e 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -21,10 +21,10 @@ import sys from flask import Flask, jsonify, request from flask_cors import cross_origin -from flask_talisman import Talisman +#from flask_talisman import Talisman app = Flask(__name__) -Talisman(app, strict_transport_security_max_age=300) +#Talisman(app, strict_transport_security_max_age=300) import django django.setup() From e3d8854f3451fc985a1a9cdcd1baf261cc28b2bc Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 8 Apr 2019 19:21:46 -0700 Subject: [PATCH 138/323] -> Turn Talisman back on --- apiv4/__init__.py | 4 ++-- apiv4/cohorts_views.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 2e6b182e..797c7c04 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -21,10 +21,10 @@ import sys from flask import Flask, jsonify, request from flask_cors import cross_origin -#from flask_talisman import Talisman +from flask_talisman import Talisman app = Flask(__name__) -#Talisman(app, strict_transport_security_max_age=300) +Talisman(app, strict_transport_security_max_age=300) import django django.setup() diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index c9491e40..199aeaf6 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -222,9 +222,9 @@ def create_cohort(user): result = make_cohort(user, filters, name, desc) if 'msg' in result: - cohort_infp = result + cohort_info = result else: - cohort_info = get_cohort_info(cohort_info['cohort_id']) + cohort_info = get_cohort_info(result['cohort_id']) except ValidationError as e: logger.warn("Filters rejected for improper formatting: {}".format(e)) From 68789870a4a084abc3b46556f23a9353133dd65e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 9 Apr 2019 19:22:44 -0700 Subject: [PATCH 139/323] -> Signed URIs, file paths -> Moved validation into auth for broader scope --- apiv4/auth.py | 90 +++++++++++++++++ apiv4/cohorts_routes.py | 4 +- apiv4/cohorts_views.py | 30 ------ apiv4/file_routes.py | 195 ++++++++++++++++++++++++++++++++++++ apiv4/file_views.py | 51 ++++++++++ apiv4/sample_case_routes.py | 5 +- 6 files changed, 341 insertions(+), 34 deletions(-) create mode 100644 apiv4/file_routes.py create mode 100644 apiv4/file_views.py diff --git a/apiv4/auth.py b/apiv4/auth.py index e17f760e..4129e296 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -22,10 +22,20 @@ import requests from flask import request, jsonify from django.conf import settings +from cohorts.metadata_helpers import get_acls_by_uuid +from django.contrib.auth.models import User as Django_User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from cohorts.models import Cohort_Perms +from accounts.sa_utils import auth_dataset_whitelists_for_user +from accounts.dcf_support import refresh_at_dcf, TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired logger = logging.getLogger(settings.LOGGER_NAME) +class UserValidationException(Exception): + pass + + # BEGIN METHODS def _base64_decode(encoded_str): # Add paddings manually if necessary. @@ -47,4 +57,84 @@ def auth_info(): user_info = {'id': 'anonymous', 'email': 'Anonymous'} return user_info + + +def validate_user(user_email=None, cohort_id=None, uuids=None): + + # Assume this is for the user information in the request header if none is + # provided (it could be for someone else on a project, etc.) + if not user_email: + user_email = auth_info()['email'] + + user = None + + django.setup() + try: + user = Django_User.objects.get(email=user_email) + except ObjectDoesNotExist as e: + logger.warn("User {} does not exist in our system.".format(user_email)) + raise UserValidationException( + "User {} wasn't found in our system.".format(user_email) + + " Please register with our Web Application first: " + ) + + try: + if cohort_id: + Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) + except ObjectDoesNotExist as e: + logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) + raise UserValidationException( + "User {} does not have access to cohort {}.".format(user_email, cohort_id) + + " Please contact this cohort owner to obtain permission." + ) + + if uuids: + acls_needed = get_acls_by_uuid(uuids) + user_acls = auth_dataset_whitelists_for_user(user) + + if not user_acls: + try: + err_msg, expr_str = refresh_at_dcf(user.id) + if err_msg: + logger.warn(err_msg) + raise UserValidationException("User {} not currently logged in via DCF and failed to refresh.".format(user_email) + + " Please visit the web application at and attempt a login to DCF from" + + " your Account Settings page.") + else: + user_acls = auth_dataset_whitelists_for_user(user) + if not user_acls: + raise UserValidationException( + "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format(user_email) + + " Please visit the web application at and attempt a login to DCF from" + + " your Account Settings page, then verify your controlled dataset access." + ) + + except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: + msg = "" + if type(e) is RefreshTokenExpired: + raise UserValidationException("Unable to refresh your 24 hour access to controlled data. Please log in to Web " + + "Application at and visit your Account Details page to refresh " + + "your controlled dataset access.") + else: + if type(e) is DCFCommFailure: + msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format(user_email) + else: + msg = "There is an internal inconsistency with user tokens for user {}".format(user_email) + raise Exception(msg) + + inaccessible = [] + + for acl in acls_needed: + if acl not in user_acls: + inaccessible.append(acl) + + if len(inaccessible): + logger.warn( + "User {} does not have access to one or more programs in this barcode set.".format(user_email)) + raise UserValidationException( + "User {} does not have access to one or more programs in this barcode set.".format(user_email) + + " Please double-check that you have linked the email '{}' to your eRA Commons ID via DCF and are currently signed in." + ) + + return user # END METHODS diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 56531831..80485f08 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -20,8 +20,8 @@ import json from flask import jsonify, request from apiv4 import app -from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, validate_user, create_cohort, edit_cohort, UserValidationException -from auth import auth_info +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, create_cohort, edit_cohort +from auth import auth_info, UserValidationException, validate_user from django.conf import settings logger = logging.getLogger(settings.LOGGER_NAME) diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 199aeaf6..241bb0e3 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -43,36 +43,6 @@ logger = logging.getLogger(settings.LOGGER_NAME) -class UserValidationException(Exception): - pass - - -def validate_user(user_email, cohort_id=None): - user = None - - django.setup() - try: - user = Django_User.objects.get(email=user_email) - except ObjectDoesNotExist as e: - logger.warn("User {} does not exist in our system.".format(user_email)) - raise UserValidationException( - "User {} wasn't found in our system.".format(user_email) + - " Please register with our Web Application first: " - ) - - try: - if cohort_id: - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) - except ObjectDoesNotExist as e: - logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) - raise UserValidationException( - "User {} does not have access to cohort {}.".format(user_email, cohort_id) + - " Please contact this cohort owner to obtain permission." - ) - - return user - - def get_file_manifest(cohort_id, user): file_manifest = None inc_filters = {} diff --git a/apiv4/file_routes.py b/apiv4/file_routes.py new file mode 100644 index 00000000..34c2f7b3 --- /dev/null +++ b/apiv4/file_routes.py @@ -0,0 +1,195 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from django.conf import settings +from auth import validate_user, UserValidationException +from file_views import get_file_paths, get_signed_uris + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/apiv4/files/signed_uri/', methods=['GET'], strict_slashes=False) +def sample_metadata_list(): + response = None + + request_data = request.get_json() + + try: + user = validate_user(uuids=[file_uuid]) + signed_uris = get_signed_uris(user, file_uuid) + + if signed_uris: + response = jsonify({ + 'code': 200, + 'data': signed_uris, + 'README': '' + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "File UUID {} was not found.".format(file_uuid)}) + response.status_code = 404 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve signed URIs for file UUID {}.'.format(file_uuid) + }) + response.status_code = 500 + + return response + + +@app.route('/apiv4/files/signed_uris/', methods=['POST'], strict_slashes=False) +def sample_metadata_list(): + + response = None + + request_data = request.get_json() + + try: + + if 'uuids' not in request_data: + response = jsonify({ + 'code': 400, + 'message': "File UUIDs not provided in data payload." + }) + response.status_code = 400 + else: + user = validate_user() + signed_uris = get_signed_uris(user, request_data['uuids']) + + if signed_uris: + response = jsonify({ + 'code': 200, + 'data': signed_uris, + 'README': '' + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "The provided file UUIDs were not found."}) + response.status_code = 404 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve signed URIs for these file UUIDs.' + }) + response.status_code = 500 + + return response + + +@app.route('/apiv4/files/path/', methods=['GET'], strict_slashes=False) +def sample_metadata_list(): + response = None + + request_data = request.get_json() + + try: + file_paths = get_file_paths(file_uuid) + + if file_paths: + response = jsonify({ + 'code': 200, + 'data': file_paths, + 'README': '' + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "File UUID {} was not found.".format(file_uuid)}) + response.status_code = 404 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve the file path for file UUID {}.'.format(file_uuid) + }) + response.status_code = 500 + + return response + + +@app.route('/apiv4/files/paths/', methods=['POST'], strict_slashes=False) +def sample_metadata_list(): + + response = None + + request_data = request.get_json() + + try: + + if 'uuids' not in request_data: + response = jsonify({ + 'code': 400, + 'message': "File UUIDs not provided in data payload." + }) + response.status_code = 400 + else: + user = validate_user() + file_paths = get_file_paths(user, request_data['uuids']) + + if file_paths: + response = jsonify({ + 'code': 200, + 'data': file_paths, + 'README': '' + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "The provided file UUIDs were not found."}) + response.status_code = 404 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve file paths for these file UUIDs.' + }) + response.status_code = 500 + + return response + diff --git a/apiv4/file_views.py b/apiv4/file_views.py new file mode 100644 index 00000000..f4bf2d48 --- /dev/null +++ b/apiv4/file_views.py @@ -0,0 +1,51 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +import django + +from flask import request + +from django.core.signals import request_finished +from django.conf import settings + +from auth import UserValidationException + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_file_paths(file_uuids): + if not file_uuids or not len(file_uuids): + raise Exception("While attempting to obtain file paths, encountered an error: no file UUIDs were provided.") + + return [] + + +def get_signed_uris(user, file_uuids): + if not user: + logger.error("A user was not provided while attempting to obtained signed URIs!") + raise UserValidationException("A user was not provided while attempting to obtained signed URIs!") + if not file_uuids or not len(file_uuids): + raise Exception("While attempting to obtain signed URIs, encountered an error: no file UUIDs were provided.") + + return [] + + + + diff --git a/apiv4/sample_case_routes.py b/apiv4/sample_case_routes.py index 91058bdd..0c480102 100644 --- a/apiv4/sample_case_routes.py +++ b/apiv4/sample_case_routes.py @@ -22,6 +22,7 @@ from apiv4 import app from django.conf import settings from sample_case_views import get_full_sample_metadata, get_full_case_metadata +from auth import validate_user, UserValidationException logger = logging.getLogger(settings.LOGGER_NAME) @@ -86,7 +87,7 @@ def sample_metadata_list(): else: response = jsonify({ 'code': 404, - 'message': "Sample barcode {} was not found.".format(sample_barcode)}) + 'message': "Unable to retrieve case metadata for these barcodes: {}".format(str(sample_barcodes))}) response.status_code = 404 return response @@ -109,7 +110,7 @@ def case_metadata_list(): else: response = jsonify({ 'code': 404, - 'message': "Sample barcode {} was not found.".format(case_barcode)}) + 'message': "Unable to retrieve case metadata for these barcodes: {}".format(str(case_barcodes))}) response.status_code = 404 return response From a11bd1524a9adc40fe947f7d5b29bcff4da7efc0 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 10 Apr 2019 12:19:17 -0700 Subject: [PATCH 140/323] -> Wire up signed URI requests (note that signed URIs cannot be obtained yet) --- apiv4/__init__.py | 1 + apiv4/auth.py | 21 ++++++++++++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 797c7c04..c359adfe 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -35,6 +35,7 @@ from cohorts_routes import * from program_routes import * from sample_case_routes import * +from file_routes import * logger = logging.getLogger(settings.LOGGER_NAME) diff --git a/apiv4/auth.py b/apiv4/auth.py index 4129e296..34b4e087 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -95,22 +95,23 @@ def validate_user(user_email=None, cohort_id=None, uuids=None): if not user_acls: try: err_msg, expr_str = refresh_at_dcf(user.id) + exception_msg = None if err_msg: logger.warn(err_msg) - raise UserValidationException("User {} not currently logged in via DCF and failed to refresh.".format(user_email) + - " Please visit the web application at and attempt a login to DCF from" + - " your Account Settings page.") + exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user_email) + \ + " Please visit the web application at and attempt a login to DCF from" + \ + " your Account Settings page." else: user_acls = auth_dataset_whitelists_for_user(user) if not user_acls: - raise UserValidationException( - "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format(user_email) + - " Please visit the web application at and attempt a login to DCF from" + + exception_msg = "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format(user_email) + \ + " Please visit the web application at and attempt a login to DCF from" + \ " your Account Settings page, then verify your controlled dataset access." - ) + + if exception_msg: + raise UserValidationException(exception_msg) except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: - msg = "" if type(e) is RefreshTokenExpired: raise UserValidationException("Unable to refresh your 24 hour access to controlled data. Please log in to Web " + "Application at and visit your Account Details page to refresh " @@ -121,7 +122,9 @@ def validate_user(user_email=None, cohort_id=None, uuids=None): else: msg = "There is an internal inconsistency with user tokens for user {}".format(user_email) raise Exception(msg) - + + logger.info("User ACLs: {}".format(str(user_acls))) + logger.info("ACLs needed: {}".format(str(acls_needed))) inaccessible = [] for acl in acls_needed: From f1e6c2cdb612f8e968befa89292fd2f2a05a9a72 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 10 Apr 2019 12:36:38 -0700 Subject: [PATCH 141/323] -> Name methods separately... --- apiv4/file_routes.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apiv4/file_routes.py b/apiv4/file_routes.py index 34c2f7b3..f0b1d17e 100644 --- a/apiv4/file_routes.py +++ b/apiv4/file_routes.py @@ -28,7 +28,7 @@ @app.route('/apiv4/files/signed_uri/', methods=['GET'], strict_slashes=False) -def sample_metadata_list(): +def signed_uri(): response = None request_data = request.get_json() @@ -69,7 +69,7 @@ def sample_metadata_list(): @app.route('/apiv4/files/signed_uris/', methods=['POST'], strict_slashes=False) -def sample_metadata_list(): +def signed_uris(): response = None @@ -119,7 +119,7 @@ def sample_metadata_list(): @app.route('/apiv4/files/path/', methods=['GET'], strict_slashes=False) -def sample_metadata_list(): +def file_path(): response = None request_data = request.get_json() @@ -152,7 +152,7 @@ def sample_metadata_list(): @app.route('/apiv4/files/paths/', methods=['POST'], strict_slashes=False) -def sample_metadata_list(): +def file_paths(): response = None From cbf3e68e7f1290cb8be10bb8122d6e5284574a32 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 10 Apr 2019 13:02:33 -0700 Subject: [PATCH 142/323] -> Match routes with openAPI doc --- apiv4/file_routes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiv4/file_routes.py b/apiv4/file_routes.py index f0b1d17e..7120a234 100644 --- a/apiv4/file_routes.py +++ b/apiv4/file_routes.py @@ -27,7 +27,7 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/files/signed_uri/', methods=['GET'], strict_slashes=False) +@app.route('/apiv4/files/signed_uris//', methods=['GET'], strict_slashes=False) def signed_uri(): response = None @@ -118,7 +118,7 @@ def signed_uris(): return response -@app.route('/apiv4/files/path/', methods=['GET'], strict_slashes=False) +@app.route('/apiv4/files/paths//', methods=['GET'], strict_slashes=False) def file_path(): response = None From f317eb6bd6b3a48c0e69f5d8ebd65d3f82c2a6b9 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 10 Apr 2019 13:39:00 -0700 Subject: [PATCH 143/323] -> Make method names more in line with others -> Wire up file path fetcher --- apiv4/file_routes.py | 13 ++++++------- apiv4/file_views.py | 7 +++++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/apiv4/file_routes.py b/apiv4/file_routes.py index 7120a234..acbda41b 100644 --- a/apiv4/file_routes.py +++ b/apiv4/file_routes.py @@ -28,7 +28,7 @@ @app.route('/apiv4/files/signed_uris//', methods=['GET'], strict_slashes=False) -def signed_uri(): +def signed_uri(file_uuid): response = None request_data = request.get_json() @@ -69,7 +69,7 @@ def signed_uri(): @app.route('/apiv4/files/signed_uris/', methods=['POST'], strict_slashes=False) -def signed_uris(): +def signed_uri_list(): response = None @@ -119,13 +119,13 @@ def signed_uris(): @app.route('/apiv4/files/paths//', methods=['GET'], strict_slashes=False) -def file_path(): +def file_path(file_uuid): response = None request_data = request.get_json() try: - file_paths = get_file_paths(file_uuid) + file_paths = get_file_paths([file_uuid]) if file_paths: response = jsonify({ @@ -152,7 +152,7 @@ def file_path(): @app.route('/apiv4/files/paths/', methods=['POST'], strict_slashes=False) -def file_paths(): +def file_path_list(): response = None @@ -167,8 +167,7 @@ def file_paths(): }) response.status_code = 400 else: - user = validate_user() - file_paths = get_file_paths(user, request_data['uuids']) + file_paths = get_file_paths(request_data['uuids']) if file_paths: response = jsonify({ diff --git a/apiv4/file_views.py b/apiv4/file_views.py index f4bf2d48..2800424b 100644 --- a/apiv4/file_views.py +++ b/apiv4/file_views.py @@ -24,6 +24,7 @@ from django.core.signals import request_finished from django.conf import settings +from cohorts.metadata_helpers import get_paths_by_uuid from auth import UserValidationException @@ -33,8 +34,10 @@ def get_file_paths(file_uuids): if not file_uuids or not len(file_uuids): raise Exception("While attempting to obtain file paths, encountered an error: no file UUIDs were provided.") - - return [] + + paths = get_paths_by_uuid(file_uuids) + + return paths def get_signed_uris(user, file_uuids): From 8090f4df6c000106a2fb50a58fb5849784935dda Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 15 Apr 2019 19:35:50 -0700 Subject: [PATCH 144/323] -> User based routes (account details, GCP registration) --- apiv4/auth.py | 87 ++++++++------ apiv4/user_routes.py | 281 +++++++++++++++++++++++++++++++++++++++++++ apiv4/user_views.py | 120 ++++++++++++++++++ 3 files changed, 453 insertions(+), 35 deletions(-) create mode 100644 apiv4/user_routes.py create mode 100644 apiv4/user_views.py diff --git a/apiv4/auth.py b/apiv4/auth.py index 34b4e087..6eb4479d 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -59,13 +59,12 @@ def auth_info(): return user_info -def validate_user(user_email=None, cohort_id=None, uuids=None): - +def get_user(user_email=None): # Assume this is for the user information in the request header if none is # provided (it could be for someone else on a project, etc.) if not user_email: user_email = auth_info()['email'] - + user = None django.setup() @@ -77,6 +76,14 @@ def validate_user(user_email=None, cohort_id=None, uuids=None): "User {} wasn't found in our system.".format(user_email) + " Please register with our Web Application first: " ) + + return user + + +def validate_user(user=None, cohort_id=None, uuids=None): + + if not user: + user = get_user() try: if cohort_id: @@ -90,38 +97,7 @@ def validate_user(user_email=None, cohort_id=None, uuids=None): if uuids: acls_needed = get_acls_by_uuid(uuids) - user_acls = auth_dataset_whitelists_for_user(user) - - if not user_acls: - try: - err_msg, expr_str = refresh_at_dcf(user.id) - exception_msg = None - if err_msg: - logger.warn(err_msg) - exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user_email) + \ - " Please visit the web application at and attempt a login to DCF from" + \ - " your Account Settings page." - else: - user_acls = auth_dataset_whitelists_for_user(user) - if not user_acls: - exception_msg = "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format(user_email) + \ - " Please visit the web application at and attempt a login to DCF from" + \ - " your Account Settings page, then verify your controlled dataset access." - - if exception_msg: - raise UserValidationException(exception_msg) - - except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: - if type(e) is RefreshTokenExpired: - raise UserValidationException("Unable to refresh your 24 hour access to controlled data. Please log in to Web " - + "Application at and visit your Account Details page to refresh " - + "your controlled dataset access.") - else: - if type(e) is DCFCommFailure: - msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format(user_email) - else: - msg = "There is an internal inconsistency with user tokens for user {}".format(user_email) - raise Exception(msg) + user_acls = get_user_acls(user) logger.info("User ACLs: {}".format(str(user_acls))) logger.info("ACLs needed: {}".format(str(acls_needed))) @@ -140,4 +116,45 @@ def validate_user(user_email=None, cohort_id=None, uuids=None): ) return user + + +def get_user_acls(user): + user_acls = auth_dataset_whitelists_for_user(user) + + if not user_acls: + try: + err_msg, expr_str = refresh_at_dcf(user.id) + exception_msg = None + if err_msg: + logger.warn(err_msg) + exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user_email) + \ + " Please visit the web application at and attempt a login to DCF from" + \ + " your Account Settings page." + else: + user_acls = auth_dataset_whitelists_for_user(user) + if not user_acls: + exception_msg = "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format( + user_email) + \ + " Please visit the web application at and attempt a login to DCF from" + \ + " your Account Settings page, then verify your controlled dataset access." + + if exception_msg: + raise UserValidationException(exception_msg) + + except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: + if type(e) is RefreshTokenExpired: + raise UserValidationException( + "Unable to refresh your 24 hour access to controlled data. Please log in to Web " + + "Application at and visit your Account Details page to refresh " + + "your controlled dataset access.") + else: + if type(e) is DCFCommFailure: + msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format( + user_email) + else: + msg = "There is an internal inconsistency with user tokens for user {}".format(user_email) + raise Exception(msg) + + + # END METHODS diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py new file mode 100644 index 00000000..18830b35 --- /dev/null +++ b/apiv4/user_routes.py @@ -0,0 +1,281 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from auth import auth_info, UserValidationException, validate_user, get_user +from django.conf import settings + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/apiv4/users/account_details/', methods=['GET'], strict_slashes=False) +def account_details(): + """ + GET: Retrieve extended information for a specific user + """ + + try: + user_info = auth_info() + user = get_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + + account_info = get_account_details(user) + + if account_info: + response = jsonify({ + 'code': 200, + 'data': account_info + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 404, + 'message': "Unable to retrieve information for {}.".format(str(user_info['email']))}) + response.status_code = 404 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve user information.' + }) + response.status_code = 500 + + return response + + +@app.route('/apiv4/users/gcp/validate//', methods=['GET'], strict_slashes=False) +def validate_gcp(gcp_id): + """ + GET: Validate a Google Cloud Project for registration and return the results to the user + """ + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + validation = verify_gcp_for_reg(user, gcp_id) + + if validation: + response_obj = {} + code = None + + if 'message' in validation: + response_obj['message'] = validation['message'] + if 'notes' in validation: + response_obj['notes'] = validation['notes'] + + if 'roles' not in validation: + code = 400 + else: + code = 200 + response_obj['gcp_project_id'] = validation['gcp_id'] + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Encountered an error while attempting to validate Google Cloud Platform project ID {}.".format(gcp_id) + }) + response.status_code = 500 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to validate Google Cloud Platform project ID {}.'.format(gcp_id) + }) + response.status_code = 500 + + return response + + +@app.route('/apiv4/users/gcp/register//', methods=['POST'], strict_slashes=False) +def register_gcp(gcp_id): + """ + POST: Register a Google Cloud Project with ISB-CGC + """ + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + request_data = request.get_json() + + registration, success = gcp_registration(user, gcp_id, False) + + if registration: + response_obj = {} + code = None + + if 'message' in validation: + response_obj['message'] = validation['message'] + if 'notes' in validation: + response_obj['notes'] = validation['notes'] + + if not success: + code = 400 + else: + code = 200 + response_obj['gcp_project_id'] = validation['gcp_id'] + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Encountered an error while attempting to register Google Cloud Platform project ID {}.".format(gcp_id) + }) + response.status_code = 500 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to register Google Cloud Platform project ID {}.'.format( + gcp_id) + }) + response.status_code = 500 + + return response + + +@app.route('/apiv4/users/gcp/refresh//', methods=['PATCH'], strict_slashes=False) +def refresh_gcp(gcp_id): + """ + PATCH: Refresh a Google Cloud Project with ISB-CGC, updating its user list + """ + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + refresh, success = gcp_registration(user, gcp_id, True) + + if refresh: + response_obj = {} + code = None + + if 'message' in validation: + response_obj['message'] = validation['message'] + if 'notes' in validation: + response_obj['notes'] = validation['notes'] + + if not success: + code = 400 + else: + code = 200 + response_obj['gcp_project_id'] = validation['gcp_id'] + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Encountered an error while attempting to refresh Google Cloud Platform project ID {}.".format( + gcp_id) + }) + response.status_code = 500 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to refresh Google Cloud Platform project ID {}.'.format( + gcp_id) + }) + response.status_code = 500 + + return response diff --git a/apiv4/user_views.py b/apiv4/user_views.py new file mode 100644 index 00000000..195753dc --- /dev/null +++ b/apiv4/user_views.py @@ -0,0 +1,120 @@ +""" + +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +""" + +import logging +import json +import django +import re + +from flask import request + +from django.core.signals import request_finished +from django.contrib.auth.models import User as Django_User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.conf import settings + +from accounts.sa_utils import auth_dataset_whitelists_for_user +from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg +from accounts.sa_utils import auth_dataset_whitelists_for_user +from projects.models import Program +from auth import get_user_acls, UserValidationException + +from jsonschema import validate as schema_validate, ValidationError + +BLACKLIST_RE = settings.BLACKLIST_RE + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_account_details(user): + accounts_details = None + + try: + whitelists = get_user_acls(user) + + if whitelists: + accounts_details = {'dataset_access': whitelists} + + except UserValidationException as u: + accounts_details['message'] = str(u) + + except Exception as e: + logger.error("[ERROR] Encountered an error while retrieving user account details:") + logger.exception(e) + accounts_details['message'] = "Encountered an error while retrieving account details for {}.".format(user.email_address) + + return accounts_details + + +def gcp_validation(gcp_id, user): + validation = None + + try: + validation = verify_gcp_for_reg(user, gcp_id) + + if validation: + if 'roles' in validation: + unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered']] + + if len(unregs): + validation['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ + "is intended for use with controlled access data, all users must log in to the ISB-CGC " + \ + "web application at and link their Google Account to their eRA " + \ + "Commons ID. The link to do so is found in Account Settings. Unregistered users: " + \ + "{}".format("; ".join(unregs)) + + if 'message' not in validation: + validation['message'] = "Google Cloud Platform project ID {} was successfully validated for registration.".format(gcp_id) + + except Exception as e: + logger.exception(e) + + return validation + + +def gcp_registration(gcp_id, user, refresh): + + registration = None + success = False + try: + validation = verify_gcp_for_reg(user, gcp_id, refresh) + + if validation: + if 'roles' in validation: + + registered_users = [x for x, y in validation['roles'].items() if y['registered_user']] + + registration, status = register_or_refresh_gcp(user, gcp_id, registered_users, refresh) + + if status == 200: + success = True + unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered']] + if len(unregs): + registration['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ + "is intended for use with controlled access data, all users must log in to the ISB-CGC " + \ + "web application at and link their Google Account to their eRA " + \ + "Commons ID. The link to do so is found in Account Settings. Unregistered users: " + \ + "{}".format("; ".join(unregs)) + + if 'message' not in registration: + registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') + + except Exception as e: + logger.exception(e) + + return registration, success From 1531293f5527c2cde8c88ab0373bb07d42c32f35 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 15 Apr 2019 20:10:47 -0700 Subject: [PATCH 145/323] -> Activate user routes --- apiv4/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index c359adfe..4789b89e 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -36,6 +36,7 @@ from program_routes import * from sample_case_routes import * from file_routes import * +from user_routes import * logger = logging.getLogger(settings.LOGGER_NAME) From 90b28cb8ce7c65a08ee8620c3317c1afd119b23b Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 15 Apr 2019 20:43:08 -0700 Subject: [PATCH 146/323] -> Weird problem with Django... --- apiv4/auth.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apiv4/auth.py b/apiv4/auth.py index 6eb4479d..f37a8098 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -20,6 +20,7 @@ import base64 import json import requests +import django from flask import request, jsonify from django.conf import settings from cohorts.metadata_helpers import get_acls_by_uuid From 8fbe0a11012faf52283ca13798e884818c5d2276 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 16 Apr 2019 10:20:58 -0700 Subject: [PATCH 147/323] -> Activate methods --- apiv4/user_routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index 18830b35..4f63ac3f 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -21,6 +21,7 @@ from flask import jsonify, request from apiv4 import app from auth import auth_info, UserValidationException, validate_user, get_user +from user_views import get_user_acls, get_account_details, verify_gcp_for_reg, register_or_refresh_gcp from django.conf import settings logger = logging.getLogger(settings.LOGGER_NAME) From 96e2012f957e5531b5fc0788ebc58112577b18a9 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 16 Apr 2019 12:55:12 -0700 Subject: [PATCH 148/323] -> Assign the var... --- apiv4/user_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 195753dc..ee27773e 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -51,7 +51,7 @@ def get_account_details(user): accounts_details = {'dataset_access': whitelists} except UserValidationException as u: - accounts_details['message'] = str(u) + accounts_details = {'message': str(u)} except Exception as e: logger.error("[ERROR] Encountered an error while retrieving user account details:") From 78cb68696cbe8189872f95322b29195d867fdb80 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 16 Apr 2019 12:59:17 -0700 Subject: [PATCH 149/323] -> Assign the var... --- apiv4/user_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index ee27773e..0e6d4dde 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -56,7 +56,7 @@ def get_account_details(user): except Exception as e: logger.error("[ERROR] Encountered an error while retrieving user account details:") logger.exception(e) - accounts_details['message'] = "Encountered an error while retrieving account details for {}.".format(user.email_address) + accounts_details = {'message': "Encountered an error while retrieving account details for {}.".format(user.email_address)} return accounts_details From 1b4a21ef239485d5eb1b64eb76d96ddbd5faa9b7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 16 Apr 2019 16:00:57 -0700 Subject: [PATCH 150/323] -> Error management cleanup --- apiv4/user_routes.py | 16 +++++++++++----- apiv4/user_views.py | 1 + 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index 4f63ac3f..f517e45d 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -47,13 +47,19 @@ def account_details(): response.status_code = 500 else: - account_info = get_account_details(user) + account_info = get_account_details(user) if account_info: - response = jsonify({ - 'code': 200, - 'data': account_info - }) + response_obj = {} + code = None + + if 'message' in account_info: + code = 400 + else: + code = 200 + response_obj['data'] = account_info + response_obj['code'] = code + response = jsonify(response_obj) response.status_code = 200 else: response = jsonify({ diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 0e6d4dde..2131d2de 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -51,6 +51,7 @@ def get_account_details(user): accounts_details = {'dataset_access': whitelists} except UserValidationException as u: + logger.warn(u) accounts_details = {'message': str(u)} except Exception as e: From 498519a8070ae479d35b01850d5ffd0c76d5357f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 17 Apr 2019 01:40:26 -0700 Subject: [PATCH 151/323] -> Checking whitelists coming in... --- apiv4/user_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 2131d2de..a2fdb0d1 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -46,6 +46,8 @@ def get_account_details(user): try: whitelists = get_user_acls(user) + + logger.debug("Whitelists: "+str(whitelists)) if whitelists: accounts_details = {'dataset_access': whitelists} From 4931a9706d7680efa4823ecd176661764e2ed06f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 17 Apr 2019 11:40:12 -0700 Subject: [PATCH 152/323] -> Checking whitelists coming in... --- apiv4/auth.py | 2 ++ apiv4/user_views.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index f37a8098..0c1366c1 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -122,6 +122,8 @@ def validate_user(user=None, cohort_id=None, uuids=None): def get_user_acls(user): user_acls = auth_dataset_whitelists_for_user(user) + logger.info("Checking user ACLs: {}".format(str(user_acls))) + if not user_acls: try: err_msg, expr_str = refresh_at_dcf(user.id) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index a2fdb0d1..10c27185 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -46,8 +46,8 @@ def get_account_details(user): try: whitelists = get_user_acls(user) - - logger.debug("Whitelists: "+str(whitelists)) + + logger.debug("Whitelists: {}".format(str(whitelists))) if whitelists: accounts_details = {'dataset_access': whitelists} From 9149b684ced84607fe6a4c3ef936a62c9851412d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 18 Apr 2019 14:08:01 -0700 Subject: [PATCH 153/323] -> Checking whitelists coming in... --- apiv4/auth.py | 4 +--- apiv4/user_views.py | 5 +++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index 0c1366c1..b3df9b52 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -122,8 +122,6 @@ def validate_user(user=None, cohort_id=None, uuids=None): def get_user_acls(user): user_acls = auth_dataset_whitelists_for_user(user) - logger.info("Checking user ACLs: {}".format(str(user_acls))) - if not user_acls: try: err_msg, expr_str = refresh_at_dcf(user.id) @@ -158,6 +156,6 @@ def get_user_acls(user): msg = "There is an internal inconsistency with user tokens for user {}".format(user_email) raise Exception(msg) - + return user_acls # END METHODS diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 10c27185..fbd655a7 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -31,6 +31,7 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg from accounts.sa_utils import auth_dataset_whitelists_for_user +from accounts.models import AuthorizedDataset from projects.models import Program from auth import get_user_acls, UserValidationException @@ -47,10 +48,10 @@ def get_account_details(user): try: whitelists = get_user_acls(user) - logger.debug("Whitelists: {}".format(str(whitelists))) - if whitelists: + uads = AuthorizedDataset.objects.filter() accounts_details = {'dataset_access': whitelists} + except UserValidationException as u: logger.warn(u) From 61bb809d7c0fd1fdfc4e698c633222dfb43435b3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 18 Apr 2019 14:41:58 -0700 Subject: [PATCH 154/323] -> Fix user property access --- apiv4/auth.py | 20 +++++++++----------- apiv4/user_views.py | 2 +- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index b3df9b52..5485e71b 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -124,20 +124,19 @@ def get_user_acls(user): if not user_acls: try: - err_msg, expr_str = refresh_at_dcf(user.id) + err_msg, expr_str, _ = refresh_at_dcf(user.id) exception_msg = None if err_msg: logger.warn(err_msg) - exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user_email) + \ - " Please visit the web application at and attempt a login to DCF from" + \ - " your Account Settings page." + exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user.email) + \ + " Please visit the web application at and attempt a login to DCF from" + \ + " your Account Settings page." else: user_acls = auth_dataset_whitelists_for_user(user) if not user_acls: - exception_msg = "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format( - user_email) + \ - " Please visit the web application at and attempt a login to DCF from" + \ - " your Account Settings page, then verify your controlled dataset access." + exception_msg = "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format(user.email) + \ + " Please visit the web application at and attempt a login to DCF from" + \ + " your Account Settings page, then verify your controlled dataset access." if exception_msg: raise UserValidationException(exception_msg) @@ -150,10 +149,9 @@ def get_user_acls(user): + "your controlled dataset access.") else: if type(e) is DCFCommFailure: - msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format( - user_email) + msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format(user.email) else: - msg = "There is an internal inconsistency with user tokens for user {}".format(user_email) + msg = "There is an internal inconsistency with user tokens for user {}".format(user.email) raise Exception(msg) return user_acls diff --git a/apiv4/user_views.py b/apiv4/user_views.py index fbd655a7..64d75dfb 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -60,7 +60,7 @@ def get_account_details(user): except Exception as e: logger.error("[ERROR] Encountered an error while retrieving user account details:") logger.exception(e) - accounts_details = {'message': "Encountered an error while retrieving account details for {}.".format(user.email_address)} + accounts_details = {'message': "Encountered an error while retrieving account details for {}.".format(user.email)} return accounts_details From 088e5c24ad31c5070e273d102f1646699854ae12 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 18 Apr 2019 15:03:07 -0700 Subject: [PATCH 155/323] -> User passed in, not user.id --- apiv4/auth.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index 5485e71b..d279b2b0 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -120,7 +120,7 @@ def validate_user(user=None, cohort_id=None, uuids=None): def get_user_acls(user): - user_acls = auth_dataset_whitelists_for_user(user) + user_acls = auth_dataset_whitelists_for_user(user.id) if not user_acls: try: @@ -132,9 +132,10 @@ def get_user_acls(user): " Please visit the web application at and attempt a login to DCF from" + \ " your Account Settings page." else: - user_acls = auth_dataset_whitelists_for_user(user) + user_acls = auth_dataset_whitelists_for_user(user.id) + if not user_acls: - exception_msg = "Couldn't verify user controlled dasa access for user {} to provided UUID(s).".format(user.email) + \ + exception_msg = "Couldn't verify user controlled data access for user {}.".format(user.email) + \ " Please visit the web application at and attempt a login to DCF from" + \ " your Account Settings page, then verify your controlled dataset access." From da409558586498a6629d650e092f8882ea069b85 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 18 Apr 2019 16:58:33 -0700 Subject: [PATCH 156/323] -> Disable all DCF-related auth/validation for now --- apiv4/auth.py | 110 +++++++++++++------------- apiv4/file_routes.py | 178 +++++++++++++++++++++---------------------- 2 files changed, 147 insertions(+), 141 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index d279b2b0..f369fe36 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -23,12 +23,12 @@ import django from flask import request, jsonify from django.conf import settings -from cohorts.metadata_helpers import get_acls_by_uuid +# from cohorts.metadata_helpers import get_acls_by_uuid from django.contrib.auth.models import User as Django_User from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned from cohorts.models import Cohort_Perms from accounts.sa_utils import auth_dataset_whitelists_for_user -from accounts.dcf_support import refresh_at_dcf, TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired +# from accounts.dcf_support import refresh_at_dcf, TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired logger = logging.getLogger(settings.LOGGER_NAME) @@ -96,25 +96,25 @@ def validate_user(user=None, cohort_id=None, uuids=None): " Please contact this cohort owner to obtain permission." ) - if uuids: - acls_needed = get_acls_by_uuid(uuids) - user_acls = get_user_acls(user) - - logger.info("User ACLs: {}".format(str(user_acls))) - logger.info("ACLs needed: {}".format(str(acls_needed))) - inaccessible = [] - - for acl in acls_needed: - if acl not in user_acls: - inaccessible.append(acl) - - if len(inaccessible): - logger.warn( - "User {} does not have access to one or more programs in this barcode set.".format(user_email)) - raise UserValidationException( - "User {} does not have access to one or more programs in this barcode set.".format(user_email) + - " Please double-check that you have linked the email '{}' to your eRA Commons ID via DCF and are currently signed in." - ) + # if uuids: + # acls_needed = get_acls_by_uuid(uuids) + # user_acls = get_user_acls(user) + # + # logger.info("User ACLs: {}".format(str(user_acls))) + # logger.info("ACLs needed: {}".format(str(acls_needed))) + # inaccessible = [] + # + # for acl in acls_needed: + # if acl not in user_acls: + # inaccessible.append(acl) + # + # if len(inaccessible): + # logger.warn( + # "User {} does not have access to one or more programs in this barcode set.".format(user_email)) + # raise UserValidationException( + # "User {} does not have access to one or more programs in this barcode set.".format(user_email) + + # " Please double-check that you have linked the email '{}' to your eRA Commons ID via DCF and are currently signed in." + # ) return user @@ -123,37 +123,43 @@ def get_user_acls(user): user_acls = auth_dataset_whitelists_for_user(user.id) if not user_acls: - try: - err_msg, expr_str, _ = refresh_at_dcf(user.id) - exception_msg = None - if err_msg: - logger.warn(err_msg) - exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user.email) + \ - " Please visit the web application at and attempt a login to DCF from" + \ - " your Account Settings page." - else: - user_acls = auth_dataset_whitelists_for_user(user.id) - - if not user_acls: - exception_msg = "Couldn't verify user controlled data access for user {}.".format(user.email) + \ - " Please visit the web application at and attempt a login to DCF from" + \ - " your Account Settings page, then verify your controlled dataset access." - - if exception_msg: - raise UserValidationException(exception_msg) - - except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: - if type(e) is RefreshTokenExpired: - raise UserValidationException( - "Unable to refresh your 24 hour access to controlled data. Please log in to Web " - + "Application at and visit your Account Details page to refresh " - + "your controlled dataset access.") - else: - if type(e) is DCFCommFailure: - msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format(user.email) - else: - msg = "There is an internal inconsistency with user tokens for user {}".format(user.email) - raise Exception(msg) + raise UserValidationException("Couldn't verify user controlled data access for user {}.".format(user.email) + + " Please visit the web application at and attempt a login to DCF from" + + " your Account Settings page, then verify your controlled dataset access." + ) + + + # try: + # err_msg, expr_str, _ = refresh_at_dcf(user.id) + # exception_msg = None + # if err_msg: + # logger.warn(err_msg) + # exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user.email) + \ + # " Please visit the web application at and attempt a login to DCF from" + \ + # " your Account Settings page." + # else: + # user_acls = auth_dataset_whitelists_for_user(user.id) + # + # if not user_acls: + # exception_msg = "Couldn't verify user controlled data access for user {}.".format(user.email) + \ + # " Please visit the web application at and attempt a login to DCF from" + \ + # " your Account Settings page, then verify your controlled dataset access." + # + # if exception_msg: + # raise UserValidationException(exception_msg) + # + # except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: + # if type(e) is RefreshTokenExpired: + # raise UserValidationException( + # "Unable to refresh your 24 hour access to controlled data. Please log in to Web " + # + "Application at and visit your Account Details page to refresh " + # + "your controlled dataset access.") + # else: + # if type(e) is DCFCommFailure: + # msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format(user.email) + # else: + # msg = "There is an internal inconsistency with user tokens for user {}".format(user.email) + # raise Exception(msg) return user_acls diff --git a/apiv4/file_routes.py b/apiv4/file_routes.py index acbda41b..46e8ec48 100644 --- a/apiv4/file_routes.py +++ b/apiv4/file_routes.py @@ -27,95 +27,95 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/files/signed_uris//', methods=['GET'], strict_slashes=False) -def signed_uri(file_uuid): - response = None - - request_data = request.get_json() - - try: - user = validate_user(uuids=[file_uuid]) - signed_uris = get_signed_uris(user, file_uuid) - - if signed_uris: - response = jsonify({ - 'code': 200, - 'data': signed_uris, - 'README': '' - }) - response.status_code = 200 - else: - response = jsonify({ - 'code': 404, - 'message': "File UUID {} was not found.".format(file_uuid)}) - response.status_code = 404 - - except UserValidationException as e: - response = jsonify({ - 'code': 403, - 'message': str(e) - }) - response.status_code = 403 - - except Exception as e: - logger.exception(e) - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while attempting to retrieve signed URIs for file UUID {}.'.format(file_uuid) - }) - response.status_code = 500 - - return response - - -@app.route('/apiv4/files/signed_uris/', methods=['POST'], strict_slashes=False) -def signed_uri_list(): - - response = None - - request_data = request.get_json() - - try: - - if 'uuids' not in request_data: - response = jsonify({ - 'code': 400, - 'message': "File UUIDs not provided in data payload." - }) - response.status_code = 400 - else: - user = validate_user() - signed_uris = get_signed_uris(user, request_data['uuids']) - - if signed_uris: - response = jsonify({ - 'code': 200, - 'data': signed_uris, - 'README': '' - }) - response.status_code = 200 - else: - response = jsonify({ - 'code': 404, - 'message': "The provided file UUIDs were not found."}) - response.status_code = 404 - - except UserValidationException as e: - response = jsonify({ - 'code': 403, - 'message': str(e) - }) - response.status_code = 403 - - except Exception as e: - logger.exception(e) - response = jsonify({ - 'code': 500, - 'message': 'Encountered an error while attempting to retrieve signed URIs for these file UUIDs.' - }) - response.status_code = 500 - - return response +# @app.route('/apiv4/files/signed_uris//', methods=['GET'], strict_slashes=False) +# def signed_uri(file_uuid): +# response = None +# +# request_data = request.get_json() +# +# try: +# user = validate_user(uuids=[file_uuid]) +# signed_uris = get_signed_uris(user, file_uuid) +# +# if signed_uris: +# response = jsonify({ +# 'code': 200, +# 'data': signed_uris, +# 'README': '' +# }) +# response.status_code = 200 +# else: +# response = jsonify({ +# 'code': 404, +# 'message': "File UUID {} was not found.".format(file_uuid)}) +# response.status_code = 404 +# +# except UserValidationException as e: +# response = jsonify({ +# 'code': 403, +# 'message': str(e) +# }) +# response.status_code = 403 +# +# except Exception as e: +# logger.exception(e) +# response = jsonify({ +# 'code': 500, +# 'message': 'Encountered an error while attempting to retrieve signed URIs for file UUID {}.'.format(file_uuid) +# }) +# response.status_code = 500 +# +# return response + + +# @app.route('/apiv4/files/signed_uris/', methods=['POST'], strict_slashes=False) +# def signed_uri_list(): +# +# response = None +# +# request_data = request.get_json() +# +# try: +# +# if 'uuids' not in request_data: +# response = jsonify({ +# 'code': 400, +# 'message': "File UUIDs not provided in data payload." +# }) +# response.status_code = 400 +# else: +# user = validate_user() +# signed_uris = get_signed_uris(user, request_data['uuids']) +# +# if signed_uris: +# response = jsonify({ +# 'code': 200, +# 'data': signed_uris, +# 'README': '' +# }) +# response.status_code = 200 +# else: +# response = jsonify({ +# 'code': 404, +# 'message': "The provided file UUIDs were not found."}) +# response.status_code = 404 +# +# except UserValidationException as e: +# response = jsonify({ +# 'code': 403, +# 'message': str(e) +# }) +# response.status_code = 403 +# +# except Exception as e: +# logger.exception(e) +# response = jsonify({ +# 'code': 500, +# 'message': 'Encountered an error while attempting to retrieve signed URIs for these file UUIDs.' +# }) +# response.status_code = 500 +# +# return response @app.route('/apiv4/files/paths//', methods=['GET'], strict_slashes=False) From 836972ee132b25bf57ec77a8a29afc2df4300c3d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 18 Apr 2019 17:04:03 -0700 Subject: [PATCH 157/323] -> GCP validation bugs --- apiv4/user_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 64d75dfb..a0c3d1da 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -69,7 +69,7 @@ def gcp_validation(gcp_id, user): validation = None try: - validation = verify_gcp_for_reg(user, gcp_id) + validation, status = verify_gcp_for_reg(user, gcp_id) if validation: if 'roles' in validation: From c1710a1e19cec14ed50c2e112d475983c44eb787 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Thu, 18 Apr 2019 17:34:19 -0700 Subject: [PATCH 158/323] -> GCP validation bugs --- apiv4/user_routes.py | 4 ++-- apiv4/user_views.py | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index f517e45d..39442084 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -21,7 +21,7 @@ from flask import jsonify, request from apiv4 import app from auth import auth_info, UserValidationException, validate_user, get_user -from user_views import get_user_acls, get_account_details, verify_gcp_for_reg, register_or_refresh_gcp +from user_views import get_user_acls, get_account_details, gcp_validation, gcp_registration from django.conf import settings logger = logging.getLogger(settings.LOGGER_NAME) @@ -104,7 +104,7 @@ def validate_gcp(gcp_id): }) response.status_code = 500 else: - validation = verify_gcp_for_reg(user, gcp_id) + validation = gcp_validation(user, gcp_id) if validation: response_obj = {} diff --git a/apiv4/user_views.py b/apiv4/user_views.py index a0c3d1da..1c3c46f6 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -49,10 +49,9 @@ def get_account_details(user): whitelists = get_user_acls(user) if whitelists: - uads = AuthorizedDataset.objects.filter() - accounts_details = {'dataset_access': whitelists} + uads = AuthorizedDataset.objects.filter(whitelist_id__in=whitelists) + accounts_details = {'dataset_access': [{uad.name: uad.whitelist_id for uad in uads}]} - except UserValidationException as u: logger.warn(u) accounts_details = {'message': str(u)} @@ -65,7 +64,7 @@ def get_account_details(user): return accounts_details -def gcp_validation(gcp_id, user): +def gcp_validation(user, gcp_id): validation = None try: @@ -91,7 +90,7 @@ def gcp_validation(gcp_id, user): return validation -def gcp_registration(gcp_id, user, refresh): +def gcp_registration(user, gcp_id, refresh): registration = None success = False From 5953d7964d6daa9b699a7e8d86ac9a77f91dbf09 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 19 Apr 2019 11:19:19 -0700 Subject: [PATCH 159/323] -> Return structure fix --- apiv4/user_views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 1c3c46f6..4602d5f4 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -50,7 +50,7 @@ def get_account_details(user): if whitelists: uads = AuthorizedDataset.objects.filter(whitelist_id__in=whitelists) - accounts_details = {'dataset_access': [{uad.name: uad.whitelist_id for uad in uads}]} + accounts_details = {'dataset_access': [{'name': uad.name, 'whitelist_id': uad.whitelist_id} for uad in uads]} except UserValidationException as u: logger.warn(u) From c4c53f2c68a74de8304baac8ea1aa5af455139e0 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 19 Apr 2019 12:04:53 -0700 Subject: [PATCH 160/323] -> Remove some debug lines, update logging entries --- settings.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/settings.py b/settings.py index 8a88937a..d546159e 100644 --- a/settings.py +++ b/settings.py @@ -100,7 +100,7 @@ SITE_ID = 3 if IS_APP_ENGINE_FLEX or IS_APP_ENGINE: - print("[STATUS] AppEngine detected.") + print("[STATUS] AppEngine detected - swapping in live site ID.") SITE_ID = 4 @@ -395,8 +395,7 @@ def GET_BQ_COHORT_SETTINGS(): # End django-allauth # ########################## -GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(__file__), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' # Path to privatekey.json -print("Google App Creds location: "+GOOGLE_APPLICATION_CREDENTIALS) +GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(__file__), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), os.environ.get('CLIENT_SECRETS')) if os.environ.get('CLIENT_SECRETS') else '' WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID', '') # Client ID from client_secrets.json IGV_WEB_CLIENT_ID = os.environ.get('IGV_WEB_CLIENT_ID', WEB_CLIENT_ID) @@ -511,6 +510,3 @@ def GET_BQ_COHORT_SETTINGS(): # Explicitly check for known items BLACKLIST_RE = r'((?i)|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i))' - -print("Done with settings.py") - From 8c6ceb56822351f0cb8a171d700cb48687a0b448 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 19 Apr 2019 18:37:59 -0700 Subject: [PATCH 161/323] -> Fix for user validation --- apiv4/auth.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/apiv4/auth.py b/apiv4/auth.py index f369fe36..d4255d4b 100644 --- a/apiv4/auth.py +++ b/apiv4/auth.py @@ -81,10 +81,11 @@ def get_user(user_email=None): return user -def validate_user(user=None, cohort_id=None, uuids=None): - - if not user: - user = get_user() +def validate_user(user_email=None, cohort_id=None, uuids=None): + user = get_user() + + if not user_email: + user_email = user.email try: if cohort_id: From 66fb298347363beb9f9835b9aebd331bb3ec5ae2 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 22 Apr 2019 14:41:22 -0700 Subject: [PATCH 162/323] -> registered_user not registered --- apiv4/user_views.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 4602d5f4..f177a69f 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -72,7 +72,7 @@ def gcp_validation(user, gcp_id): if validation: if 'roles' in validation: - unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered']] + unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered_user']] if len(unregs): validation['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ @@ -106,7 +106,7 @@ def gcp_registration(user, gcp_id, refresh): if status == 200: success = True - unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered']] + unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered_user']] if len(unregs): registration['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ "is intended for use with controlled access data, all users must log in to the ISB-CGC " + \ From 1376a296fb09bc68812f5ab67d92ecd2fa84f476 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 22 Apr 2019 16:35:37 -0700 Subject: [PATCH 163/323] -> Registration weirdness --- apiv4/user_views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index f177a69f..acb3d0de 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -104,6 +104,8 @@ def gcp_registration(user, gcp_id, refresh): registration, status = register_or_refresh_gcp(user, gcp_id, registered_users, refresh) + logger.info("Registration: {}".format(str(registration))) + if status == 200: success = True unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered_user']] From ee96db38c12bed2b7f473de9cb15588bb27daf0f Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 22 Apr 2019 16:53:57 -0700 Subject: [PATCH 164/323] -> Registration weirdness --- apiv4/user_views.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index acb3d0de..36cdf197 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -95,7 +95,7 @@ def gcp_registration(user, gcp_id, refresh): registration = None success = False try: - validation = verify_gcp_for_reg(user, gcp_id, refresh) + validation, validation_status = verify_gcp_for_reg(user, gcp_id, refresh) if validation: if 'roles' in validation: @@ -120,6 +120,7 @@ def gcp_registration(user, gcp_id, refresh): registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') except Exception as e: + logger.error("[ERROR] While registering a GCP:") logger.exception(e) return registration, success From dd7f3b5e91673c2eaf381c647574aa9d8c38c526 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 22 Apr 2019 16:56:38 -0700 Subject: [PATCH 165/323] -> Registration weirdness --- apiv4/user_views.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 36cdf197..65b55c43 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -64,11 +64,11 @@ def get_account_details(user): return accounts_details -def gcp_validation(user, gcp_id): +def gcp_validation(user, gcp_id, refresh=False): validation = None try: - validation, status = verify_gcp_for_reg(user, gcp_id) + validation, status = verify_gcp_for_reg(user, gcp_id, refresh) if validation: if 'roles' in validation: @@ -95,7 +95,7 @@ def gcp_registration(user, gcp_id, refresh): registration = None success = False try: - validation, validation_status = verify_gcp_for_reg(user, gcp_id, refresh) + validation = gcp_validation(user, gcp_id, refresh) if validation: if 'roles' in validation: @@ -118,6 +118,8 @@ def gcp_registration(user, gcp_id, refresh): if 'message' not in registration: registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') + else: + logger.warn("[WARNING] Validation was unsuccessful!") except Exception as e: logger.error("[ERROR] While registering a GCP:") From e5e0ca31102e1993b0963742b4e01fe0a26f366a Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 22 Apr 2019 17:01:33 -0700 Subject: [PATCH 166/323] -> Run registration through main validation process, not a separate one --- apiv4/user_views.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index 65b55c43..fbf319fa 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -83,6 +83,8 @@ def gcp_validation(user, gcp_id, refresh=False): if 'message' not in validation: validation['message'] = "Google Cloud Platform project ID {} was successfully validated for registration.".format(gcp_id) + else: + logger.warn("[WARNING] Validation was unsuccessful!") except Exception as e: logger.exception(e) @@ -101,21 +103,13 @@ def gcp_registration(user, gcp_id, refresh): if 'roles' in validation: registered_users = [x for x, y in validation['roles'].items() if y['registered_user']] - registration, status = register_or_refresh_gcp(user, gcp_id, registered_users, refresh) - logger.info("Registration: {}".format(str(registration))) if status == 200: success = True - unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered_user']] - if len(unregs): - registration['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ - "is intended for use with controlled access data, all users must log in to the ISB-CGC " + \ - "web application at and link their Google Account to their eRA " + \ - "Commons ID. The link to do so is found in Account Settings. Unregistered users: " + \ - "{}".format("; ".join(unregs)) - + if 'notes' in validation: + registration['notes'] = validation['notes'] if 'message' not in registration: registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') else: From e33c549125891686fa718269620b170e4fe1712d Mon Sep 17 00:00:00 2001 From: s-paquett Date: Mon, 22 Apr 2019 17:51:32 -0700 Subject: [PATCH 167/323] -> Wrong var --- apiv4/user_routes.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index 39442084..7555b3b7 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -178,16 +178,16 @@ def register_gcp(gcp_id): response_obj = {} code = None - if 'message' in validation: - response_obj['message'] = validation['message'] - if 'notes' in validation: - response_obj['notes'] = validation['notes'] + if 'message' in registration: + response_obj['message'] = registration['message'] + if 'notes' in registration: + response_obj['notes'] = registration['notes'] if not success: code = 400 else: code = 200 - response_obj['gcp_project_id'] = validation['gcp_id'] + response_obj['gcp_project_id'] = registration['gcp_id'] response_obj['code'] = code response = jsonify(response_obj) From ed80ba9a60ee41d8c768e33484a50b6e89ae1d10 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Wed, 24 Apr 2019 11:02:18 -0700 Subject: [PATCH 168/323] -> GCP reg and unreg --- apiv4/user_routes.py | 77 +++++++++++++++++++++++++++++++++++++++++--- apiv4/user_views.py | 28 ++++++++++++++-- 2 files changed, 98 insertions(+), 7 deletions(-) diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index 7555b3b7..7e83a9da 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -245,16 +245,16 @@ def refresh_gcp(gcp_id): response_obj = {} code = None - if 'message' in validation: - response_obj['message'] = validation['message'] + if 'message' in refresh: + response_obj['message'] = refresh['message'] if 'notes' in validation: - response_obj['notes'] = validation['notes'] + response_obj['notes'] = refresh['notes'] if not success: code = 400 else: code = 200 - response_obj['gcp_project_id'] = validation['gcp_id'] + response_obj['gcp_project_id'] = refresh['gcp_id'] response_obj['code'] = code response = jsonify(response_obj) @@ -286,3 +286,72 @@ def refresh_gcp(gcp_id): response.status_code = 500 return response + + +@app.route('/apiv4/users/gcp/unregister//', methods=['POST'], strict_slashes=False) +def unregister_gcp(gcp_id): + """ + POST: Unregister a Google Cloud Project with ISB-CGC + """ + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + unreg, success = gcp_unregistration(user, gcp_id, False) + + if unreg: + response_obj = {} + code = None + + if 'message' in unreg: + response_obj['message'] = unreg['message'] + if 'notes' in registration: + response_obj['notes'] = unreg['notes'] + + if not success: + code = 400 + else: + code = 200 + response_obj['gcp_project_id'] = unreg['gcp_id'] + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Encountered an error while attempting to unregister Google Cloud Platform project ID {}.".format( + gcp_id) + }) + response.status_code = 500 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to unregister Google Cloud Platform project ID {}.'.format( + gcp_id) + }) + response.status_code = 500 + + return response + diff --git a/apiv4/user_views.py b/apiv4/user_views.py index fbf319fa..d9e31535 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -29,7 +29,7 @@ from django.conf import settings from accounts.sa_utils import auth_dataset_whitelists_for_user -from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg +from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg, unreg_gcp from accounts.sa_utils import auth_dataset_whitelists_for_user from accounts.models import AuthorizedDataset from projects.models import Program @@ -84,9 +84,10 @@ def gcp_validation(user, gcp_id, refresh=False): if 'message' not in validation: validation['message'] = "Google Cloud Platform project ID {} was successfully validated for registration.".format(gcp_id) else: - logger.warn("[WARNING] Validation was unsuccessful!") + logger.warn("[WARNING] Validation of {} by user {} was unsuccessful!".format(gcp_id, user.email)) except Exception as e: + logger.error("[ERROR] While attempting to validate a project for registration:") logger.exception(e) return validation @@ -113,10 +114,31 @@ def gcp_registration(user, gcp_id, refresh): if 'message' not in registration: registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') else: - logger.warn("[WARNING] Validation was unsuccessful!") + logger.warn("[WARNING] Validation of {} by user {} was unsuccessful!".format(gcp_id, user.email)) except Exception as e: logger.error("[ERROR] While registering a GCP:") logger.exception(e) return registration, success + + +def gcp_unregistration(user, gcp_id): + unreg = None + success = False + try: + + unreg, status = unreg_gcp(user, gcp_id) + + if status == 200: + success = True + if 'message' not in unreg: + unreg['message'] = "Google Cloud Platform project ID {} was successfully unregistered.".format(gcp_id) + else: + logger.warn("[WARNING] Unregistration of {} by user {} was unsuccessful!".format(gcp_id, user.email)) + + except Exception as e: + logger.error("[ERROR] While unregistering a GCP:") + logger.exception(e) + + return unreg, success From ecefdf04dd9f53caf569b93d77beb89ee5593822 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 30 Apr 2019 13:26:04 -0700 Subject: [PATCH 169/323] -> Merge to master/dev --- shell/install-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shell/install-deps.sh b/shell/install-deps.sh index 20610958..f56592e4 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -2,7 +2,7 @@ if [ -n "$CI" ]; then export HOME=/home/circleci/${CIRCLE_PROJECT_REPONAME} export HOMEROOT=/home/circleci/${CIRCLE_PROJECT_REPONAME} # Clone dependencies - git clone -b py-3-testing https://github.com/isb-cgc/ISB-CGC-Common.git + git clone -b master https://github.com/isb-cgc/ISB-CGC-Common.git # Remove .pyc files; these can sometimes stick around and if a # model has changed names it will cause various load failures From 75ce63f64bcc021327ac8850a2b5f2ad30f8aaa2 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 30 Apr 2019 13:30:34 -0700 Subject: [PATCH 170/323] -> Merge to master/dev --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a34d28c..0ef45d9a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,4 +87,4 @@ workflows: - build_job filters: branches: - only: api-v4 + only: master From 28f2046a336614a627dc7bf4d691dd02948538d5 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 30 Apr 2019 13:58:07 -0700 Subject: [PATCH 171/323] -> app.yaml fixes --- app.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app.yaml b/app.yaml index d28defce..988d2de8 100644 --- a/app.yaml +++ b/app.yaml @@ -1,5 +1,5 @@ runtime: custom -service: api-v4 +service: mvm env: flex api_version: 1 threadsafe: true @@ -8,7 +8,7 @@ threadsafe: true endpoints_api_service: # The following values are to be replaced by information from the output of # 'gcloud endpoints services deploy openapi-appengine.yaml' command. - name: api-v4-dot-isb-cgc.appspot.com + name: mvm-api-dot-isb-cgc.appspot.com rollout_strategy: managed # [END configuration] @@ -31,4 +31,4 @@ beta_settings: cloud_sql_instances: isb-cgc:us-central1:dev-v3 network: - instance_tag: isb-cgc-api-v4 + instance_tag: isb-cgc-mvm-api From b5d401587ce1965de78cf9908cb095817928f825 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 30 Apr 2019 16:20:49 -0700 Subject: [PATCH 172/323] -> app.yaml fixes --- app.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.yaml b/app.yaml index 988d2de8..1a308784 100644 --- a/app.yaml +++ b/app.yaml @@ -1,5 +1,5 @@ runtime: custom -service: mvm +service: mvm-api env: flex api_version: 1 threadsafe: true From 5889066c3f9c7266ba28361b625d01f1ce4952a1 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 30 Apr 2019 17:37:39 -0700 Subject: [PATCH 173/323] -> Fix multiple import --- apiv4/user_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apiv4/user_views.py b/apiv4/user_views.py index d9e31535..42e27aae 100644 --- a/apiv4/user_views.py +++ b/apiv4/user_views.py @@ -30,7 +30,6 @@ from accounts.sa_utils import auth_dataset_whitelists_for_user from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg, unreg_gcp -from accounts.sa_utils import auth_dataset_whitelists_for_user from accounts.models import AuthorizedDataset from projects.models import Program from auth import get_user_acls, UserValidationException From 3cb2f86b0f0bb8c87257a3b795b536ba517ad8d3 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Fri, 3 May 2019 13:44:53 -0700 Subject: [PATCH 174/323] -> Enable cohort editing --- apiv4/cohorts_routes.py | 108 ++++++++++++++++++++++++---------------- apiv4/cohorts_views.py | 81 ++++++++++++++++-------------- apiv4/user_routes.py | 2 +- 3 files changed, 109 insertions(+), 82 deletions(-) diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py index 80485f08..d90442d9 100644 --- a/apiv4/cohorts_routes.py +++ b/apiv4/cohorts_routes.py @@ -39,6 +39,7 @@ def cohort(cohort_id): user = validate_user(user_info['email'], cohort_id) response = None + code = None if not user: response = jsonify({ @@ -53,11 +54,16 @@ def cohort(cohort_id): cohort_info = edit_cohort(cohort_id) if cohort_info: - response = jsonify({ - 'code': 200, - 'data': cohort_info - }) - response.status_code = 200 + if 'message' in cohort_info: + code = 400 + else: + code = 200 + + response_obj['data'] = cohort_info + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + else: response = jsonify({ 'code': 404, @@ -88,14 +94,15 @@ def cohorts(): GET: Retrieve a user's list of cohorts POST: Add a new cohort """ + + response = None + info = None + code = None try: user_info = auth_info() user = validate_user(user_info['email']) - response = None - info = None - if not user: response = jsonify({ 'code': 500, @@ -109,24 +116,23 @@ def cohorts(): info = create_cohort(user) if info: - if 'msg' in info: - # Presence of a message means something was wrong. - response = jsonify({ - 'code': 400, - 'data': info - }) - response.status_code = 400 + if 'message' in info: + code = 400 else: - response = jsonify({ - 'code': 200, - 'data': info - }) - response.status_code = 200 - # Lack of a valid object means something went wrong on the server + code = 200 + + response_obj['data'] = info + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server else: response = jsonify({ 'code': 500, - 'message': "Error while attempting to create this cohort." + 'message': "Error while attempting to {}.".format( + 'retrieve the cohort list' if request.method == 'GET' else 'create this cohort' + ) }) response.status_code = 500 @@ -157,12 +163,13 @@ def cohort_file_manifest(cohort_id): POST: Retrieve a cohort's file manifest with applied filters """ + response = None + code = None + try: user_info = auth_info() user = validate_user(user_info['email'], cohort_id) - response = None - if not user: response = jsonify({ 'code': 500, @@ -172,11 +179,16 @@ def cohort_file_manifest(cohort_id): else: file_manifest = get_file_manifest(cohort_id, user) if file_manifest: - response = jsonify({ - 'code': 200, - 'data': file_manifest - }) - response.status_code = 200 + # Presence of a message means something went wrong with our request + if 'message' in file_manifest: + code = 400 + else: + code = 200 + + response_obj['data'] = file_manifest + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code else: response = jsonify({ 'code': 500, @@ -207,28 +219,36 @@ def cohort_preview(): """List the samples, cases, and counts a given set of cohort filters would produce""" response = None + code = None - cohort_counts = get_cohort_counts() + try: + cohort_counts = get_cohort_counts() + + if cohort_counts: + # Presence of a message means something went wrong with the filters we received + if 'message' in cohort_counts: + code = 400 + else: + code = 200 - if cohort_counts: - # Presence of a message means something went wrong with the filters we received - if 'msg' in cohort_counts: - response = jsonify({ - 'code': 400, - 'data': cohort_counts - }) - response.status_code = 400 + response_obj['data'] = cohort_counts + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server else: response = jsonify({ - 'code': 200, - 'data': cohort_counts + 'code': 500, + 'message': "Error while attempting to retrieve case and sample counts for these filters." }) - response.status_code = 200 - # Lack of a valid object means something went wrong on the server - else: + response.status_code = 500 + + except Exception as e: + logger.exception(e) response = jsonify({ 'code': 500, - 'message': "Error while attempting to retrieve case and sample counts for these filters." + 'message': 'Encountered an error while attempting to build this cohort preview.' }) response.status_code = 500 diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py index 241bb0e3..a5ae9a9e 100644 --- a/apiv4/cohorts_views.py +++ b/apiv4/cohorts_views.py @@ -133,19 +133,19 @@ def get_cohort_counts(): if 'filters' not in request_data: cohort_counts = { - 'msg': 'No filters were provided; ensure that the request body contains a \'filters\' property.' + 'message': 'No filters were provided; ensure that the request body contains a \'filters\' property.' } else: cohort_counts = get_sample_case_list_bq(None, request_data['filters']) for prog in cohort_counts: if cohort_counts[prog]['case_count'] <= 0: - cohort_counts[prog]['msg'] = "No cases or samples found which meet the filter criteria for this program." + cohort_counts[prog]['message'] = "No cases or samples found which meet the filter criteria for this program." cohort_counts[prog]['provided_filters'] = request_data['filters'][prog] except ValidationError as e: logger.warn('Filters rejected for improper formatting: {}'.format(e)) cohort_counts = { - 'msg': 'Filters were improperly formatted.' + 'message': 'Filters were improperly formatted.' } except Exception as e: logger.exception(e) @@ -161,73 +161,80 @@ def create_cohort(user): if 'name' not in request_data: cohort_info = { - 'msg': 'A name was not provided for this cohort. Cohort was not made.' + 'message': 'A name was not provided for this cohort. The cohort was not made.', + 'code': 400 } return cohort_info if 'filters' not in request_data: cohort_info = { - 'msg': 'Filters were not provided; at least one filter must be provided for a cohort to be valid.' + - ' Cohort was not made.' + 'message': 'Filters were not provided; at least one filter must be provided for a cohort to be valid.' + + ' The cohort was not made.', + 'code': 400 } return cohort_info - name = request_data['name'] - filters = request_data['filters'] - blacklist = re.compile(BLACKLIST_RE, re.UNICODE) - match = blacklist.search(str(name)) + match = blacklist.search(str(request_data['name'])) + + if not match and 'desc' in request_data: + match = blacklist.search(str(request_data['desc'])) + if match: - # XSS risk, log and fail this cohort save - match = blacklist.findall(str(name)) cohort_info = { - 'msg': 'Your cohort\'s name contains invalid characters; please choose another name. ' + - '[Saw {}]'.format(str(match)) + 'message': 'Your cohort\'s name or description contains invalid characters; please edit them and resubmit. ' + + '[Saw {}]'.format(str(match)), + 'code': 400 } - else: - desc = None - if 'description' in request_data: - desc = request_data['description'] - result = make_cohort(user, filters, name, desc) + else: + result = make_cohort(user, **request_data) - if 'msg' in result: + if 'message' in result: cohort_info = result else: cohort_info = get_cohort_info(result['cohort_id']) except ValidationError as e: - logger.warn("Filters rejected for improper formatting: {}".format(e)) + logger.warn("[WARNING] Cohort information rejected for improper formatting: {}".format(e)) cohort_info = { - 'msg': 'Filters were improperly formatted - cohort not created.' + 'message': 'Cohort information was improperly formatted - cohort not edited.', + 'code': 400 } return cohort_info def edit_cohort(cohort_id): - cohort_info = None + result = None + match = None try: request_data = request.get_json() - schema_validate(request_data, COHORT_FILTER_SCHEMA) + if len(request_data.keys()): + schema_validate(request_data, COHORT_FILTER_SCHEMA) + + if 'name' in request_data: + blacklist = re.compile(BLACKLIST_RE, re.UNICODE) + match = blacklist.search(str(request_data['name'])) + + if not match and 'desc' in request_data: + match = blacklist.search(str(request_data['desc'])) - name = request_data['name'] - blacklist = re.compile(BLACKLIST_RE, re.UNICODE) - match = blacklist.search(str(name)) if match: - # XSS risk, log and fail this cohort save - match = blacklist.findall(str(name)) - cohort_info = { - 'msg': 'Your cohort\'s name contains invalid characters; please choose another name.' + - ' [Saw {}]'.format(str(match)) + result = { + 'message': 'Your cohort\'s name or description contains invalid characters; please edit them and resubmit. ' + + '[Saw {}]'.format(str(match)), + 'code': 400 } else: - logger.warn("Make cohort") + result = make_cohort(user, source_id=cohort_id, **request_data) + except ValidationError as e: - logger.warn("Filters rejected for improper formatting: {}".format(e)) - cohort_info = { - 'msg': 'Filters were improperly formatted - cohort not created.' + logger.warn("[WARNING] Cohort information rejected for improper formatting: {}".format(e)) + result = { + 'message': 'Cohort information was improperly formatted - cohort not edited.', + 'code': 400 } - return cohort_info + return result diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py index 7e83a9da..7d8a6e8e 100644 --- a/apiv4/user_routes.py +++ b/apiv4/user_routes.py @@ -60,7 +60,7 @@ def account_details(): response_obj['data'] = account_info response_obj['code'] = code response = jsonify(response_obj) - response.status_code = 200 + response.status_code = code else: response = jsonify({ 'code': 404, From 50e1c82c3fd9ca058b445383ee5c7157fb9b44d6 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 13:24:12 -0700 Subject: [PATCH 175/323] -> Swagger attempt #1 --- apiv4/__init__.py | 6 +- static/favicon-16x16.png | Bin 0 -> 665 bytes static/favicon-32x32.png | Bin 0 -> 628 bytes static/swagger-ui-bundle.js | 93 +++++++++++++++++++++++++ static/swagger-ui-standalone-preset.js | 14 ++++ static/swagger-ui.css | 3 + static/swagger-ui.js | 9 +++ swagger/index.html | 60 ++++++++++++++++ swagger/oauth2-redirect.html | 67 ++++++++++++++++++ 9 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 static/favicon-16x16.png create mode 100644 static/favicon-32x32.png create mode 100644 static/swagger-ui-bundle.js create mode 100644 static/swagger-ui-standalone-preset.js create mode 100644 static/swagger-ui.css create mode 100644 static/swagger-ui.js create mode 100644 swagger/index.html create mode 100644 swagger/oauth2-redirect.html diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 4789b89e..c444e95b 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -23,13 +23,13 @@ from flask_cors import cross_origin from flask_talisman import Talisman -app = Flask(__name__) -Talisman(app, strict_transport_security_max_age=300) - import django django.setup() from django.conf import settings +app = Flask(__name__, static_url_path=settings.STATIC_URL, static_folder='api_static') +Talisman(app, strict_transport_security_max_age=300) + from auth import auth_info from main_routes import * from cohorts_routes import * diff --git a/static/favicon-16x16.png b/static/favicon-16x16.png new file mode 100644 index 0000000000000000000000000000000000000000..8b194e617af1c135e6b37939591d24ac3a5efa18 GIT binary patch literal 665 zcmV;K0%rY*P)}JKSduyL>)s!A4EhTMMEM%Q;aL6%l#xiZiF>S;#Y{N2Zz%pvTGHJduXuC6Lx-)0EGfRy*N{Tv4i8@4oJ41gw zKzThrcRe|7J~(YYIBq{SYCkn-KQm=N8$CrEK1CcqMI1dv9z#VRL_{D)L|`QmF8}}l zJ9JV`Q}p!p_4f7m_U`WQ@apR4;o;!mnU<7}iG_qr zF(e)x9~BG-3IzcG2M4an0002kNkl41`ZiN1i62V%{PM@Ry|IS_+Yc7{bb`MM~xm(7p4|kMHP&!VGuDW4kFixat zXw43VmgwEvB$hXt_u=vZ>+v4i7E}n~eG6;n4Z=zF1n?T*yg<;W6kOfxpC6nao>VR% z?fpr=asSJ&`L*wu^rLJ5Peq*PB0;alL#XazZCBxJLd&giTfw@!hW167F^`7kobi;( ze<<>qNlP|xy7S1zl@lZNIBR7#o9ybJsptO#%}P0hz~sBp00000NkvXXu0mjfUsDF? literal 0 HcmV?d00001 diff --git a/static/favicon-32x32.png b/static/favicon-32x32.png new file mode 100644 index 0000000000000000000000000000000000000000..249737fe44558e679f0b67134e274461d988fa98 GIT binary patch literal 628 zcmV-)0*n2LP)Ma*GM0}OV<074bNCP7P7GVd{iMr*I6y~TMLss@FjvgL~HxU z%Vvj33AwpD(Z4*$Mfx=HaU16axM zt2xG_rloN<$iy9j9I5>>0;if(""+n!==t||4294967295===n)return NaN;t=n}return t<0?C(e)+t:t}function A(){return!0}function O(e,t,n){return(0===e||void 0!==n&&e<=-n)&&(void 0===t||void 0!==n&&t>=n)}function P(e,t){return M(e,t,0)}function T(e,t){return M(e,t,t)}function M(e,t,n){return void 0===e?n:e<0?Math.max(0,t+e):void 0===t?e:Math.min(t,e)}var I=0,j=1,N=2,R="function"==typeof Symbol&&Symbol.iterator,D="@@iterator",L=R||D;function U(e){this.next=e}function q(e,t,n,r){var o=0===e?t:1===e?n:[t,n];return r?r.value=o:r={value:o,done:!1},r}function F(){return{value:void 0,done:!0}}function z(e){return!!H(e)}function B(e){return e&&"function"==typeof e.next}function V(e){var t=H(e);return t&&t.call(e)}function H(e){var t=e&&(R&&e[R]||e[D]);if("function"==typeof t)return t}function W(e){return e&&"number"==typeof e.length}function J(e){return null===e||void 0===e?ie():a(e)?e.toSeq():function(e){var t=se(e)||"object"==typeof e&&new te(e);if(!t)throw new TypeError("Expected Array or iterable object of values, or keyed object: "+e);return t}(e)}function Y(e){return null===e||void 0===e?ie().toKeyedSeq():a(e)?u(e)?e.toSeq():e.fromEntrySeq():ae(e)}function K(e){return null===e||void 0===e?ie():a(e)?u(e)?e.entrySeq():e.toIndexedSeq():ue(e)}function G(e){return(null===e||void 0===e?ie():a(e)?u(e)?e.entrySeq():e:ue(e)).toSetSeq()}U.prototype.toString=function(){return"[Iterator]"},U.KEYS=I,U.VALUES=j,U.ENTRIES=N,U.prototype.inspect=U.prototype.toSource=function(){return this.toString()},U.prototype[L]=function(){return this},t(J,n),J.of=function(){return J(arguments)},J.prototype.toSeq=function(){return this},J.prototype.toString=function(){return this.__toString("Seq {","}")},J.prototype.cacheResult=function(){return!this._cache&&this.__iterateUncached&&(this._cache=this.entrySeq().toArray(),this.size=this._cache.length),this},J.prototype.__iterate=function(e,t){return le(this,e,t,!0)},J.prototype.__iterator=function(e,t){return ce(this,e,t,!0)},t(Y,J),Y.prototype.toKeyedSeq=function(){return this},t(K,J),K.of=function(){return K(arguments)},K.prototype.toIndexedSeq=function(){return this},K.prototype.toString=function(){return this.__toString("Seq [","]")},K.prototype.__iterate=function(e,t){return le(this,e,t,!1)},K.prototype.__iterator=function(e,t){return ce(this,e,t,!1)},t(G,J),G.of=function(){return G(arguments)},G.prototype.toSetSeq=function(){return this},J.isSeq=oe,J.Keyed=Y,J.Set=G,J.Indexed=K;var $,Z,X,Q="@@__IMMUTABLE_SEQ__@@";function ee(e){this._array=e,this.size=e.length}function te(e){var t=Object.keys(e);this._object=e,this._keys=t,this.size=t.length}function ne(e){this._iterable=e,this.size=e.length||e.size}function re(e){this._iterator=e,this._iteratorCache=[]}function oe(e){return!(!e||!e[Q])}function ie(){return $||($=new ee([]))}function ae(e){var t=Array.isArray(e)?new ee(e).fromEntrySeq():B(e)?new re(e).fromEntrySeq():z(e)?new ne(e).fromEntrySeq():"object"==typeof e?new te(e):void 0;if(!t)throw new TypeError("Expected Array or iterable object of [k, v] entries, or keyed object: "+e);return t}function ue(e){var t=se(e);if(!t)throw new TypeError("Expected Array or iterable object of values: "+e);return t}function se(e){return W(e)?new ee(e):B(e)?new re(e):z(e)?new ne(e):void 0}function le(e,t,n,r){var o=e._cache;if(o){for(var i=o.length-1,a=0;a<=i;a++){var u=o[n?i-a:a];if(!1===t(u[1],r?u[0]:a,e))return a+1}return a}return e.__iterateUncached(t,n)}function ce(e,t,n,r){var o=e._cache;if(o){var i=o.length-1,a=0;return new U(function(){var e=o[n?i-a:a];return a++>i?{value:void 0,done:!0}:q(t,r?e[0]:a-1,e[1])})}return e.__iteratorUncached(t,n)}function fe(e,t){return t?function e(t,n,r,o){if(Array.isArray(n))return t.call(o,r,K(n).map(function(r,o){return e(t,r,o,n)}));if(de(n))return t.call(o,r,Y(n).map(function(r,o){return e(t,r,o,n)}));return n}(t,e,"",{"":e}):pe(e)}function pe(e){return Array.isArray(e)?K(e).map(pe).toList():de(e)?Y(e).map(pe).toMap():e}function de(e){return e&&(e.constructor===Object||void 0===e.constructor)}function he(e,t){if(e===t||e!=e&&t!=t)return!0;if(!e||!t)return!1;if("function"==typeof e.valueOf&&"function"==typeof t.valueOf){if((e=e.valueOf())===(t=t.valueOf())||e!=e&&t!=t)return!0;if(!e||!t)return!1}return!("function"!=typeof e.equals||"function"!=typeof t.equals||!e.equals(t))}function ve(e,t){if(e===t)return!0;if(!a(t)||void 0!==e.size&&void 0!==t.size&&e.size!==t.size||void 0!==e.__hash&&void 0!==t.__hash&&e.__hash!==t.__hash||u(e)!==u(t)||s(e)!==s(t)||c(e)!==c(t))return!1;if(0===e.size&&0===t.size)return!0;var n=!l(e);if(c(e)){var r=e.entries();return t.every(function(e,t){var o=r.next().value;return o&&he(o[1],e)&&(n||he(o[0],t))})&&r.next().done}var o=!1;if(void 0===e.size)if(void 0===t.size)"function"==typeof e.cacheResult&&e.cacheResult();else{o=!0;var i=e;e=t,t=i}var f=!0,p=t.__iterate(function(t,r){if(n?!e.has(t):o?!he(t,e.get(r,y)):!he(e.get(r,y),t))return f=!1,!1});return f&&e.size===p}function me(e,t){if(!(this instanceof me))return new me(e,t);if(this._value=e,this.size=void 0===t?1/0:Math.max(0,t),0===this.size){if(Z)return Z;Z=this}}function ge(e,t){if(!e)throw new Error(t)}function ye(e,t,n){if(!(this instanceof ye))return new ye(e,t,n);if(ge(0!==n,"Cannot step a Range by 0"),e=e||0,void 0===t&&(t=1/0),n=void 0===n?1:Math.abs(n),tr?{value:void 0,done:!0}:q(e,o,n[t?r-o++:o++])})},t(te,Y),te.prototype.get=function(e,t){return void 0===t||this.has(e)?this._object[e]:t},te.prototype.has=function(e){return this._object.hasOwnProperty(e)},te.prototype.__iterate=function(e,t){for(var n=this._object,r=this._keys,o=r.length-1,i=0;i<=o;i++){var a=r[t?o-i:i];if(!1===e(n[a],a,this))return i+1}return i},te.prototype.__iterator=function(e,t){var n=this._object,r=this._keys,o=r.length-1,i=0;return new U(function(){var a=r[t?o-i:i];return i++>o?{value:void 0,done:!0}:q(e,a,n[a])})},te.prototype[h]=!0,t(ne,K),ne.prototype.__iterateUncached=function(e,t){if(t)return this.cacheResult().__iterate(e,t);var n=V(this._iterable),r=0;if(B(n))for(var o;!(o=n.next()).done&&!1!==e(o.value,r++,this););return r},ne.prototype.__iteratorUncached=function(e,t){if(t)return this.cacheResult().__iterator(e,t);var n=V(this._iterable);if(!B(n))return new U(F);var r=0;return new U(function(){var t=n.next();return t.done?t:q(e,r++,t.value)})},t(re,K),re.prototype.__iterateUncached=function(e,t){if(t)return this.cacheResult().__iterate(e,t);for(var n,r=this._iterator,o=this._iteratorCache,i=0;i=r.length){var t=n.next();if(t.done)return t;r[o]=t.value}return q(e,o,r[o++])})},t(me,K),me.prototype.toString=function(){return 0===this.size?"Repeat []":"Repeat [ "+this._value+" "+this.size+" times ]"},me.prototype.get=function(e,t){return this.has(e)?this._value:t},me.prototype.includes=function(e){return he(this._value,e)},me.prototype.slice=function(e,t){var n=this.size;return O(e,t,n)?this:new me(this._value,T(t,n)-P(e,n))},me.prototype.reverse=function(){return this},me.prototype.indexOf=function(e){return he(this._value,e)?0:-1},me.prototype.lastIndexOf=function(e){return he(this._value,e)?this.size:-1},me.prototype.__iterate=function(e,t){for(var n=0;n=0&&t=0&&nn?{value:void 0,done:!0}:q(e,i++,a)})},ye.prototype.equals=function(e){return e instanceof ye?this._start===e._start&&this._end===e._end&&this._step===e._step:ve(this,e)},t(be,n),t(_e,be),t(we,be),t(Ee,be),be.Keyed=_e,be.Indexed=we,be.Set=Ee;var xe="function"==typeof Math.imul&&-2===Math.imul(4294967295,2)?Math.imul:function(e,t){var n=65535&(e|=0),r=65535&(t|=0);return n*r+((e>>>16)*r+n*(t>>>16)<<16>>>0)|0};function Se(e){return e>>>1&1073741824|3221225471&e}function Ce(e){if(!1===e||null===e||void 0===e)return 0;if("function"==typeof e.valueOf&&(!1===(e=e.valueOf())||null===e||void 0===e))return 0;if(!0===e)return 1;var t=typeof e;if("number"===t){if(e!=e||e===1/0)return 0;var n=0|e;for(n!==e&&(n^=4294967295*e);e>4294967295;)n^=e/=4294967295;return Se(n)}if("string"===t)return e.length>je?function(e){var t=De[e];void 0===t&&(t=ke(e),Re===Ne&&(Re=0,De={}),Re++,De[e]=t);return t}(e):ke(e);if("function"==typeof e.hashCode)return e.hashCode();if("object"===t)return function(e){var t;if(Te&&void 0!==(t=Pe.get(e)))return t;if(void 0!==(t=e[Ie]))return t;if(!Oe){if(void 0!==(t=e.propertyIsEnumerable&&e.propertyIsEnumerable[Ie]))return t;if(void 0!==(t=function(e){if(e&&e.nodeType>0)switch(e.nodeType){case 1:return e.uniqueID;case 9:return e.documentElement&&e.documentElement.uniqueID}}(e)))return t}t=++Me,1073741824&Me&&(Me=0);if(Te)Pe.set(e,t);else{if(void 0!==Ae&&!1===Ae(e))throw new Error("Non-extensible objects are not allowed as keys.");if(Oe)Object.defineProperty(e,Ie,{enumerable:!1,configurable:!1,writable:!1,value:t});else if(void 0!==e.propertyIsEnumerable&&e.propertyIsEnumerable===e.constructor.prototype.propertyIsEnumerable)e.propertyIsEnumerable=function(){return this.constructor.prototype.propertyIsEnumerable.apply(this,arguments)},e.propertyIsEnumerable[Ie]=t;else{if(void 0===e.nodeType)throw new Error("Unable to set a non-enumerable property on object.");e[Ie]=t}}return t}(e);if("function"==typeof e.toString)return ke(e.toString());throw new Error("Value type "+t+" cannot be hashed.")}function ke(e){for(var t=0,n=0;n=t.length)throw new Error("Missing value for key: "+t[n]);e.set(t[n],t[n+1])}})},Ue.prototype.toString=function(){return this.__toString("Map {","}")},Ue.prototype.get=function(e,t){return this._root?this._root.get(0,void 0,e,t):t},Ue.prototype.set=function(e,t){return Qe(this,e,t)},Ue.prototype.setIn=function(e,t){return this.updateIn(e,y,function(){return t})},Ue.prototype.remove=function(e){return Qe(this,e,y)},Ue.prototype.deleteIn=function(e){return this.updateIn(e,function(){return y})},Ue.prototype.update=function(e,t,n){return 1===arguments.length?e(this):this.updateIn([e],t,n)},Ue.prototype.updateIn=function(e,t,n){n||(n=t,t=void 0);var r=function e(t,n,r,o){var i=t===y;var a=n.next();if(a.done){var u=i?r:t,s=o(u);return s===u?t:s}ge(i||t&&t.set,"invalid keyPath");var l=a.value;var c=i?y:t.get(l,y);var f=e(c,n,r,o);return f===c?t:f===y?t.remove(l):(i?Xe():t).set(l,f)}(this,nn(e),t,n);return r===y?void 0:r},Ue.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=0,this._root=null,this.__hash=void 0,this.__altered=!0,this):Xe()},Ue.prototype.merge=function(){return rt(this,void 0,arguments)},Ue.prototype.mergeWith=function(t){return rt(this,t,e.call(arguments,1))},Ue.prototype.mergeIn=function(t){var n=e.call(arguments,1);return this.updateIn(t,Xe(),function(e){return"function"==typeof e.merge?e.merge.apply(e,n):n[n.length-1]})},Ue.prototype.mergeDeep=function(){return rt(this,ot,arguments)},Ue.prototype.mergeDeepWith=function(t){var n=e.call(arguments,1);return rt(this,it(t),n)},Ue.prototype.mergeDeepIn=function(t){var n=e.call(arguments,1);return this.updateIn(t,Xe(),function(e){return"function"==typeof e.mergeDeep?e.mergeDeep.apply(e,n):n[n.length-1]})},Ue.prototype.sort=function(e){return Pt(Wt(this,e))},Ue.prototype.sortBy=function(e,t){return Pt(Wt(this,t,e))},Ue.prototype.withMutations=function(e){var t=this.asMutable();return e(t),t.wasAltered()?t.__ensureOwner(this.__ownerID):this},Ue.prototype.asMutable=function(){return this.__ownerID?this:this.__ensureOwner(new x)},Ue.prototype.asImmutable=function(){return this.__ensureOwner()},Ue.prototype.wasAltered=function(){return this.__altered},Ue.prototype.__iterator=function(e,t){return new Ke(this,e,t)},Ue.prototype.__iterate=function(e,t){var n=this,r=0;return this._root&&this._root.iterate(function(t){return r++,e(t[1],t[0],n)},t),r},Ue.prototype.__ensureOwner=function(e){return e===this.__ownerID?this:e?Ze(this.size,this._root,e,this.__hash):(this.__ownerID=e,this.__altered=!1,this)},Ue.isMap=qe;var Fe,ze="@@__IMMUTABLE_MAP__@@",Be=Ue.prototype;function Ve(e,t){this.ownerID=e,this.entries=t}function He(e,t,n){this.ownerID=e,this.bitmap=t,this.nodes=n}function We(e,t,n){this.ownerID=e,this.count=t,this.nodes=n}function Je(e,t,n){this.ownerID=e,this.keyHash=t,this.entries=n}function Ye(e,t,n){this.ownerID=e,this.keyHash=t,this.entry=n}function Ke(e,t,n){this._type=t,this._reverse=n,this._stack=e._root&&$e(e._root)}function Ge(e,t){return q(e,t[0],t[1])}function $e(e,t){return{node:e,index:0,__prev:t}}function Ze(e,t,n,r){var o=Object.create(Be);return o.size=e,o._root=t,o.__ownerID=n,o.__hash=r,o.__altered=!1,o}function Xe(){return Fe||(Fe=Ze(0))}function Qe(e,t,n){var r,o;if(e._root){var i=w(b),a=w(_);if(r=et(e._root,e.__ownerID,0,void 0,t,n,i,a),!a.value)return e;o=e.size+(i.value?n===y?-1:1:0)}else{if(n===y)return e;o=1,r=new Ve(e.__ownerID,[[t,n]])}return e.__ownerID?(e.size=o,e._root=r,e.__hash=void 0,e.__altered=!0,e):r?Ze(o,r):Xe()}function et(e,t,n,r,o,i,a,u){return e?e.update(t,n,r,o,i,a,u):i===y?e:(E(u),E(a),new Ye(t,r,[o,i]))}function tt(e){return e.constructor===Ye||e.constructor===Je}function nt(e,t,n,r,o){if(e.keyHash===r)return new Je(t,r,[e.entry,o]);var i,a=(0===n?e.keyHash:e.keyHash>>>n)&g,u=(0===n?r:r>>>n)&g;return new He(t,1<>1&1431655765))+(e>>2&858993459))+(e>>4)&252645135,e+=e>>8,127&(e+=e>>16)}function st(e,t,n,r){var o=r?e:S(e);return o[t]=n,o}Be[ze]=!0,Be.delete=Be.remove,Be.removeIn=Be.deleteIn,Ve.prototype.get=function(e,t,n,r){for(var o=this.entries,i=0,a=o.length;i=lt)return function(e,t,n,r){e||(e=new x);for(var o=new Ye(e,Ce(n),[n,r]),i=0;i>>e)&g),i=this.bitmap;return 0==(i&o)?r:this.nodes[ut(i&o-1)].get(e+v,t,n,r)},He.prototype.update=function(e,t,n,r,o,i,a){void 0===n&&(n=Ce(r));var u=(0===t?n:n>>>t)&g,s=1<=ct)return function(e,t,n,r,o){for(var i=0,a=new Array(m),u=0;0!==n;u++,n>>>=1)a[u]=1&n?t[i++]:void 0;return a[r]=o,new We(e,i+1,a)}(e,p,l,u,h);if(c&&!h&&2===p.length&&tt(p[1^f]))return p[1^f];if(c&&h&&1===p.length&&tt(h))return h;var b=e&&e===this.ownerID,_=c?h?l:l^s:l|s,w=c?h?st(p,f,h,b):function(e,t,n){var r=e.length-1;if(n&&t===r)return e.pop(),e;for(var o=new Array(r),i=0,a=0;a>>e)&g,i=this.nodes[o];return i?i.get(e+v,t,n,r):r},We.prototype.update=function(e,t,n,r,o,i,a){void 0===n&&(n=Ce(r));var u=(0===t?n:n>>>t)&g,s=o===y,l=this.nodes,c=l[u];if(s&&!c)return this;var f=et(c,e,t+v,n,r,o,i,a);if(f===c)return this;var p=this.count;if(c){if(!f&&--p0&&r=0&&e=e.size||t<0)return e.withMutations(function(e){t<0?kt(e,t).set(0,n):kt(e,0,t+1).set(t,n)});t+=e._origin;var r=e._tail,o=e._root,i=w(_);t>=Ot(e._capacity)?r=xt(r,e.__ownerID,0,t,n,i):o=xt(o,e.__ownerID,e._level,t,n,i);if(!i.value)return e;if(e.__ownerID)return e._root=o,e._tail=r,e.__hash=void 0,e.__altered=!0,e;return wt(e._origin,e._capacity,e._level,o,r)}(this,e,t)},pt.prototype.remove=function(e){return this.has(e)?0===e?this.shift():e===this.size-1?this.pop():this.splice(e,1):this},pt.prototype.insert=function(e,t){return this.splice(e,0,t)},pt.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=this._origin=this._capacity=0,this._level=v,this._root=this._tail=null,this.__hash=void 0,this.__altered=!0,this):Et()},pt.prototype.push=function(){var e=arguments,t=this.size;return this.withMutations(function(n){kt(n,0,t+e.length);for(var r=0;r>>t&g;if(r>=this.array.length)return new mt([],e);var o,i=0===r;if(t>0){var a=this.array[r];if((o=a&&a.removeBefore(e,t-v,n))===a&&i)return this}if(i&&!o)return this;var u=St(this,e);if(!i)for(var s=0;s>>t&g;if(o>=this.array.length)return this;if(t>0){var i=this.array[o];if((r=i&&i.removeAfter(e,t-v,n))===i&&o===this.array.length-1)return this}var a=St(this,e);return a.array.splice(o+1),r&&(a.array[o]=r),a};var gt,yt,bt={};function _t(e,t){var n=e._origin,r=e._capacity,o=Ot(r),i=e._tail;return a(e._root,e._level,0);function a(e,u,s){return 0===u?function(e,a){var u=a===o?i&&i.array:e&&e.array,s=a>n?0:n-a,l=r-a;l>m&&(l=m);return function(){if(s===l)return bt;var e=t?--l:s++;return u&&u[e]}}(e,s):function(e,o,i){var u,s=e&&e.array,l=i>n?0:n-i>>o,c=1+(r-i>>o);c>m&&(c=m);return function(){for(;;){if(u){var e=u();if(e!==bt)return e;u=null}if(l===c)return bt;var n=t?--c:l++;u=a(s&&s[n],o-v,i+(n<>>n&g,s=e&&u0){var l=e&&e.array[u],c=xt(l,t,n-v,r,o,i);return c===l?e:((a=St(e,t)).array[u]=c,a)}return s&&e.array[u]===o?e:(E(i),a=St(e,t),void 0===o&&u===a.array.length-1?a.array.pop():a.array[u]=o,a)}function St(e,t){return t&&e&&t===e.ownerID?e:new mt(e?e.array.slice():[],t)}function Ct(e,t){if(t>=Ot(e._capacity))return e._tail;if(t<1<0;)n=n.array[t>>>r&g],r-=v;return n}}function kt(e,t,n){void 0!==t&&(t|=0),void 0!==n&&(n|=0);var r=e.__ownerID||new x,o=e._origin,i=e._capacity,a=o+t,u=void 0===n?i:n<0?i+n:o+n;if(a===o&&u===i)return e;if(a>=u)return e.clear();for(var s=e._level,l=e._root,c=0;a+c<0;)l=new mt(l&&l.array.length?[void 0,l]:[],r),c+=1<<(s+=v);c&&(a+=c,o+=c,u+=c,i+=c);for(var f=Ot(i),p=Ot(u);p>=1<f?new mt([],r):d;if(d&&p>f&&av;y-=v){var b=f>>>y&g;m=m.array[b]=St(m.array[b],r)}m.array[f>>>v&g]=d}if(u=p)a-=p,u-=p,s=v,l=null,h=h&&h.removeBefore(r,0,a);else if(a>o||p>>s&g;if(_!==p>>>s&g)break;_&&(c+=(1<o&&(l=l.removeBefore(r,s,a-c)),l&&pi&&(i=l.size),a(s)||(l=l.map(function(e){return fe(e)})),r.push(l)}return i>e.size&&(e=e.setSize(i)),at(e,t,r)}function Ot(e){return e>>v<=m&&a.size>=2*i.size?(r=(o=a.filter(function(e,t){return void 0!==e&&u!==t})).toKeyedSeq().map(function(e){return e[0]}).flip().toMap(),e.__ownerID&&(r.__ownerID=o.__ownerID=e.__ownerID)):(r=i.remove(t),o=u===a.size-1?a.pop():a.set(u,void 0))}else if(s){if(n===a.get(u)[1])return e;r=i,o=a.set(u,[t,n])}else r=i.set(t,a.size),o=a.set(a.size,[t,n]);return e.__ownerID?(e.size=r.size,e._map=r,e._list=o,e.__hash=void 0,e):Mt(r,o)}function Nt(e,t){this._iter=e,this._useKeys=t,this.size=e.size}function Rt(e){this._iter=e,this.size=e.size}function Dt(e){this._iter=e,this.size=e.size}function Lt(e){this._iter=e,this.size=e.size}function Ut(e){var t=Qt(e);return t._iter=e,t.size=e.size,t.flip=function(){return e},t.reverse=function(){var t=e.reverse.apply(this);return t.flip=function(){return e.reverse()},t},t.has=function(t){return e.includes(t)},t.includes=function(t){return e.has(t)},t.cacheResult=en,t.__iterateUncached=function(t,n){var r=this;return e.__iterate(function(e,n){return!1!==t(n,e,r)},n)},t.__iteratorUncached=function(t,n){if(t===N){var r=e.__iterator(t,n);return new U(function(){var e=r.next();if(!e.done){var t=e.value[0];e.value[0]=e.value[1],e.value[1]=t}return e})}return e.__iterator(t===j?I:j,n)},t}function qt(e,t,n){var r=Qt(e);return r.size=e.size,r.has=function(t){return e.has(t)},r.get=function(r,o){var i=e.get(r,y);return i===y?o:t.call(n,i,r,e)},r.__iterateUncached=function(r,o){var i=this;return e.__iterate(function(e,o,a){return!1!==r(t.call(n,e,o,a),o,i)},o)},r.__iteratorUncached=function(r,o){var i=e.__iterator(N,o);return new U(function(){var o=i.next();if(o.done)return o;var a=o.value,u=a[0];return q(r,u,t.call(n,a[1],u,e),o)})},r}function Ft(e,t){var n=Qt(e);return n._iter=e,n.size=e.size,n.reverse=function(){return e},e.flip&&(n.flip=function(){var t=Ut(e);return t.reverse=function(){return e.flip()},t}),n.get=function(n,r){return e.get(t?n:-1-n,r)},n.has=function(n){return e.has(t?n:-1-n)},n.includes=function(t){return e.includes(t)},n.cacheResult=en,n.__iterate=function(t,n){var r=this;return e.__iterate(function(e,n){return t(e,n,r)},!n)},n.__iterator=function(t,n){return e.__iterator(t,!n)},n}function zt(e,t,n,r){var o=Qt(e);return r&&(o.has=function(r){var o=e.get(r,y);return o!==y&&!!t.call(n,o,r,e)},o.get=function(r,o){var i=e.get(r,y);return i!==y&&t.call(n,i,r,e)?i:o}),o.__iterateUncached=function(o,i){var a=this,u=0;return e.__iterate(function(e,i,s){if(t.call(n,e,i,s))return u++,o(e,r?i:u-1,a)},i),u},o.__iteratorUncached=function(o,i){var a=e.__iterator(N,i),u=0;return new U(function(){for(;;){var i=a.next();if(i.done)return i;var s=i.value,l=s[0],c=s[1];if(t.call(n,c,l,e))return q(o,r?l:u++,c,i)}})},o}function Bt(e,t,n,r){var o=e.size;if(void 0!==t&&(t|=0),void 0!==n&&(n===1/0?n=o:n|=0),O(t,n,o))return e;var i=P(t,o),a=T(n,o);if(i!=i||a!=a)return Bt(e.toSeq().cacheResult(),t,n,r);var u,s=a-i;s==s&&(u=s<0?0:s);var l=Qt(e);return l.size=0===u?u:e.size&&u||void 0,!r&&oe(e)&&u>=0&&(l.get=function(t,n){return(t=k(this,t))>=0&&tu)return{value:void 0,done:!0};var e=o.next();return r||t===j?e:q(t,s-1,t===I?void 0:e.value[1],e)})},l}function Vt(e,t,n,r){var o=Qt(e);return o.__iterateUncached=function(o,i){var a=this;if(i)return this.cacheResult().__iterate(o,i);var u=!0,s=0;return e.__iterate(function(e,i,l){if(!u||!(u=t.call(n,e,i,l)))return s++,o(e,r?i:s-1,a)}),s},o.__iteratorUncached=function(o,i){var a=this;if(i)return this.cacheResult().__iterator(o,i);var u=e.__iterator(N,i),s=!0,l=0;return new U(function(){var e,i,c;do{if((e=u.next()).done)return r||o===j?e:q(o,l++,o===I?void 0:e.value[1],e);var f=e.value;i=f[0],c=f[1],s&&(s=t.call(n,c,i,a))}while(s);return o===N?e:q(o,i,c,e)})},o}function Ht(e,t,n){var r=Qt(e);return r.__iterateUncached=function(r,o){var i=0,u=!1;return function e(s,l){var c=this;s.__iterate(function(o,s){return(!t||l0}function Kt(e,t,r){var o=Qt(e);return o.size=new ee(r).map(function(e){return e.size}).min(),o.__iterate=function(e,t){for(var n,r=this.__iterator(j,t),o=0;!(n=r.next()).done&&!1!==e(n.value,o++,this););return o},o.__iteratorUncached=function(e,o){var i=r.map(function(e){return e=n(e),V(o?e.reverse():e)}),a=0,u=!1;return new U(function(){var n;return u||(n=i.map(function(e){return e.next()}),u=n.some(function(e){return e.done})),u?{value:void 0,done:!0}:q(e,a++,t.apply(null,n.map(function(e){return e.value})))})},o}function Gt(e,t){return oe(e)?t:e.constructor(t)}function $t(e){if(e!==Object(e))throw new TypeError("Expected [K, V] tuple: "+e)}function Zt(e){return Le(e.size),C(e)}function Xt(e){return u(e)?r:s(e)?o:i}function Qt(e){return Object.create((u(e)?Y:s(e)?K:G).prototype)}function en(){return this._iter.cacheResult?(this._iter.cacheResult(),this.size=this._iter.size,this):J.prototype.cacheResult.call(this)}function tn(e,t){return e>t?1:e=0;n--)t={value:arguments[n],next:t};return this.__ownerID?(this.size=e,this._head=t,this.__hash=void 0,this.__altered=!0,this):An(e,t)},En.prototype.pushAll=function(e){if(0===(e=o(e)).size)return this;Le(e.size);var t=this.size,n=this._head;return e.reverse().forEach(function(e){t++,n={value:e,next:n}}),this.__ownerID?(this.size=t,this._head=n,this.__hash=void 0,this.__altered=!0,this):An(t,n)},En.prototype.pop=function(){return this.slice(1)},En.prototype.unshift=function(){return this.push.apply(this,arguments)},En.prototype.unshiftAll=function(e){return this.pushAll(e)},En.prototype.shift=function(){return this.pop.apply(this,arguments)},En.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=0,this._head=void 0,this.__hash=void 0,this.__altered=!0,this):On()},En.prototype.slice=function(e,t){if(O(e,t,this.size))return this;var n=P(e,this.size);if(T(t,this.size)!==this.size)return we.prototype.slice.call(this,e,t);for(var r=this.size-n,o=this._head;n--;)o=o.next;return this.__ownerID?(this.size=r,this._head=o,this.__hash=void 0,this.__altered=!0,this):An(r,o)},En.prototype.__ensureOwner=function(e){return e===this.__ownerID?this:e?An(this.size,this._head,e,this.__hash):(this.__ownerID=e,this.__altered=!1,this)},En.prototype.__iterate=function(e,t){if(t)return this.reverse().__iterate(e);for(var n=0,r=this._head;r&&!1!==e(r.value,n++,this);)r=r.next;return n},En.prototype.__iterator=function(e,t){if(t)return this.reverse().__iterator(e);var n=0,r=this._head;return new U(function(){if(r){var t=r.value;return r=r.next,q(e,n++,t)}return{value:void 0,done:!0}})},En.isStack=xn;var Sn,Cn="@@__IMMUTABLE_STACK__@@",kn=En.prototype;function An(e,t,n,r){var o=Object.create(kn);return o.size=e,o._head=t,o.__ownerID=n,o.__hash=r,o.__altered=!1,o}function On(){return Sn||(Sn=An(0))}function Pn(e,t){var n=function(n){e.prototype[n]=t[n]};return Object.keys(t).forEach(n),Object.getOwnPropertySymbols&&Object.getOwnPropertySymbols(t).forEach(n),e}kn[Cn]=!0,kn.withMutations=Be.withMutations,kn.asMutable=Be.asMutable,kn.asImmutable=Be.asImmutable,kn.wasAltered=Be.wasAltered,n.Iterator=U,Pn(n,{toArray:function(){Le(this.size);var e=new Array(this.size||0);return this.valueSeq().__iterate(function(t,n){e[n]=t}),e},toIndexedSeq:function(){return new Rt(this)},toJS:function(){return this.toSeq().map(function(e){return e&&"function"==typeof e.toJS?e.toJS():e}).__toJS()},toJSON:function(){return this.toSeq().map(function(e){return e&&"function"==typeof e.toJSON?e.toJSON():e}).__toJS()},toKeyedSeq:function(){return new Nt(this,!0)},toMap:function(){return Ue(this.toKeyedSeq())},toObject:function(){Le(this.size);var e={};return this.__iterate(function(t,n){e[n]=t}),e},toOrderedMap:function(){return Pt(this.toKeyedSeq())},toOrderedSet:function(){return mn(u(this)?this.valueSeq():this)},toSet:function(){return sn(u(this)?this.valueSeq():this)},toSetSeq:function(){return new Dt(this)},toSeq:function(){return s(this)?this.toIndexedSeq():u(this)?this.toKeyedSeq():this.toSetSeq()},toStack:function(){return En(u(this)?this.valueSeq():this)},toList:function(){return pt(u(this)?this.valueSeq():this)},toString:function(){return"[Iterable]"},__toString:function(e,t){return 0===this.size?e+t:e+" "+this.toSeq().map(this.__toStringMapper).join(", ")+" "+t},concat:function(){return Gt(this,function(e,t){var n=u(e),o=[e].concat(t).map(function(e){return a(e)?n&&(e=r(e)):e=n?ae(e):ue(Array.isArray(e)?e:[e]),e}).filter(function(e){return 0!==e.size});if(0===o.length)return e;if(1===o.length){var i=o[0];if(i===e||n&&u(i)||s(e)&&s(i))return i}var l=new ee(o);return n?l=l.toKeyedSeq():s(e)||(l=l.toSetSeq()),(l=l.flatten(!0)).size=o.reduce(function(e,t){if(void 0!==e){var n=t.size;if(void 0!==n)return e+n}},0),l}(this,e.call(arguments,0)))},includes:function(e){return this.some(function(t){return he(t,e)})},entries:function(){return this.__iterator(N)},every:function(e,t){Le(this.size);var n=!0;return this.__iterate(function(r,o,i){if(!e.call(t,r,o,i))return n=!1,!1}),n},filter:function(e,t){return Gt(this,zt(this,e,t,!0))},find:function(e,t,n){var r=this.findEntry(e,t);return r?r[1]:n},forEach:function(e,t){return Le(this.size),this.__iterate(t?e.bind(t):e)},join:function(e){Le(this.size),e=void 0!==e?""+e:",";var t="",n=!0;return this.__iterate(function(r){n?n=!1:t+=e,t+=null!==r&&void 0!==r?r.toString():""}),t},keys:function(){return this.__iterator(I)},map:function(e,t){return Gt(this,qt(this,e,t))},reduce:function(e,t,n){var r,o;return Le(this.size),arguments.length<2?o=!0:r=t,this.__iterate(function(t,i,a){o?(o=!1,r=t):r=e.call(n,r,t,i,a)}),r},reduceRight:function(e,t,n){var r=this.toKeyedSeq().reverse();return r.reduce.apply(r,arguments)},reverse:function(){return Gt(this,Ft(this,!0))},slice:function(e,t){return Gt(this,Bt(this,e,t,!0))},some:function(e,t){return!this.every(Nn(e),t)},sort:function(e){return Gt(this,Wt(this,e))},values:function(){return this.__iterator(j)},butLast:function(){return this.slice(0,-1)},isEmpty:function(){return void 0!==this.size?0===this.size:!this.some(function(){return!0})},count:function(e,t){return C(e?this.toSeq().filter(e,t):this)},countBy:function(e,t){return function(e,t,n){var r=Ue().asMutable();return e.__iterate(function(o,i){r.update(t.call(n,o,i,e),0,function(e){return e+1})}),r.asImmutable()}(this,e,t)},equals:function(e){return ve(this,e)},entrySeq:function(){var e=this;if(e._cache)return new ee(e._cache);var t=e.toSeq().map(jn).toIndexedSeq();return t.fromEntrySeq=function(){return e.toSeq()},t},filterNot:function(e,t){return this.filter(Nn(e),t)},findEntry:function(e,t,n){var r=n;return this.__iterate(function(n,o,i){if(e.call(t,n,o,i))return r=[o,n],!1}),r},findKey:function(e,t){var n=this.findEntry(e,t);return n&&n[0]},findLast:function(e,t,n){return this.toKeyedSeq().reverse().find(e,t,n)},findLastEntry:function(e,t,n){return this.toKeyedSeq().reverse().findEntry(e,t,n)},findLastKey:function(e,t){return this.toKeyedSeq().reverse().findKey(e,t)},first:function(){return this.find(A)},flatMap:function(e,t){return Gt(this,function(e,t,n){var r=Xt(e);return e.toSeq().map(function(o,i){return r(t.call(n,o,i,e))}).flatten(!0)}(this,e,t))},flatten:function(e){return Gt(this,Ht(this,e,!0))},fromEntrySeq:function(){return new Lt(this)},get:function(e,t){return this.find(function(t,n){return he(n,e)},void 0,t)},getIn:function(e,t){for(var n,r=this,o=nn(e);!(n=o.next()).done;){var i=n.value;if((r=r&&r.get?r.get(i,y):y)===y)return t}return r},groupBy:function(e,t){return function(e,t,n){var r=u(e),o=(c(e)?Pt():Ue()).asMutable();e.__iterate(function(i,a){o.update(t.call(n,i,a,e),function(e){return(e=e||[]).push(r?[a,i]:i),e})});var i=Xt(e);return o.map(function(t){return Gt(e,i(t))})}(this,e,t)},has:function(e){return this.get(e,y)!==y},hasIn:function(e){return this.getIn(e,y)!==y},isSubset:function(e){return e="function"==typeof e.includes?e:n(e),this.every(function(t){return e.includes(t)})},isSuperset:function(e){return(e="function"==typeof e.isSubset?e:n(e)).isSubset(this)},keyOf:function(e){return this.findKey(function(t){return he(t,e)})},keySeq:function(){return this.toSeq().map(In).toIndexedSeq()},last:function(){return this.toSeq().reverse().first()},lastKeyOf:function(e){return this.toKeyedSeq().reverse().keyOf(e)},max:function(e){return Jt(this,e)},maxBy:function(e,t){return Jt(this,t,e)},min:function(e){return Jt(this,e?Rn(e):Un)},minBy:function(e,t){return Jt(this,t?Rn(t):Un,e)},rest:function(){return this.slice(1)},skip:function(e){return this.slice(Math.max(0,e))},skipLast:function(e){return Gt(this,this.toSeq().reverse().skip(e).reverse())},skipWhile:function(e,t){return Gt(this,Vt(this,e,t,!0))},skipUntil:function(e,t){return this.skipWhile(Nn(e),t)},sortBy:function(e,t){return Gt(this,Wt(this,t,e))},take:function(e){return this.slice(0,Math.max(0,e))},takeLast:function(e){return Gt(this,this.toSeq().reverse().take(e).reverse())},takeWhile:function(e,t){return Gt(this,function(e,t,n){var r=Qt(e);return r.__iterateUncached=function(r,o){var i=this;if(o)return this.cacheResult().__iterate(r,o);var a=0;return e.__iterate(function(e,o,u){return t.call(n,e,o,u)&&++a&&r(e,o,i)}),a},r.__iteratorUncached=function(r,o){var i=this;if(o)return this.cacheResult().__iterator(r,o);var a=e.__iterator(N,o),u=!0;return new U(function(){if(!u)return{value:void 0,done:!0};var e=a.next();if(e.done)return e;var o=e.value,s=o[0],l=o[1];return t.call(n,l,s,i)?r===N?e:q(r,s,l,e):(u=!1,{value:void 0,done:!0})})},r}(this,e,t))},takeUntil:function(e,t){return this.takeWhile(Nn(e),t)},valueSeq:function(){return this.toIndexedSeq()},hashCode:function(){return this.__hash||(this.__hash=function(e){if(e.size===1/0)return 0;var t=c(e),n=u(e),r=t?1:0;return function(e,t){return t=xe(t,3432918353),t=xe(t<<15|t>>>-15,461845907),t=xe(t<<13|t>>>-13,5),t=xe((t=(t+3864292196|0)^e)^t>>>16,2246822507),t=Se((t=xe(t^t>>>13,3266489909))^t>>>16)}(e.__iterate(n?t?function(e,t){r=31*r+qn(Ce(e),Ce(t))|0}:function(e,t){r=r+qn(Ce(e),Ce(t))|0}:t?function(e){r=31*r+Ce(e)|0}:function(e){r=r+Ce(e)|0}),r)}(this))}});var Tn=n.prototype;Tn[f]=!0,Tn[L]=Tn.values,Tn.__toJS=Tn.toArray,Tn.__toStringMapper=Dn,Tn.inspect=Tn.toSource=function(){return this.toString()},Tn.chain=Tn.flatMap,Tn.contains=Tn.includes,Pn(r,{flip:function(){return Gt(this,Ut(this))},mapEntries:function(e,t){var n=this,r=0;return Gt(this,this.toSeq().map(function(o,i){return e.call(t,[i,o],r++,n)}).fromEntrySeq())},mapKeys:function(e,t){var n=this;return Gt(this,this.toSeq().flip().map(function(r,o){return e.call(t,r,o,n)}).flip())}});var Mn=r.prototype;function In(e,t){return t}function jn(e,t){return[t,e]}function Nn(e){return function(){return!e.apply(this,arguments)}}function Rn(e){return function(){return-e.apply(this,arguments)}}function Dn(e){return"string"==typeof e?JSON.stringify(e):String(e)}function Ln(){return S(arguments)}function Un(e,t){return et?-1:0}function qn(e,t){return e^t+2654435769+(e<<6)+(e>>2)|0}return Mn[p]=!0,Mn[L]=Tn.entries,Mn.__toJS=Tn.toObject,Mn.__toStringMapper=function(e,t){return JSON.stringify(t)+": "+Dn(e)},Pn(o,{toKeyedSeq:function(){return new Nt(this,!1)},filter:function(e,t){return Gt(this,zt(this,e,t,!1))},findIndex:function(e,t){var n=this.findEntry(e,t);return n?n[0]:-1},indexOf:function(e){var t=this.keyOf(e);return void 0===t?-1:t},lastIndexOf:function(e){var t=this.lastKeyOf(e);return void 0===t?-1:t},reverse:function(){return Gt(this,Ft(this,!1))},slice:function(e,t){return Gt(this,Bt(this,e,t,!1))},splice:function(e,t){var n=arguments.length;if(t=Math.max(0|t,0),0===n||2===n&&!t)return this;e=P(e,e<0?this.count():this.size);var r=this.slice(0,e);return Gt(this,1===n?r:r.concat(S(arguments,2),this.slice(e+t)))},findLastIndex:function(e,t){var n=this.findLastEntry(e,t);return n?n[0]:-1},first:function(){return this.get(0)},flatten:function(e){return Gt(this,Ht(this,e,!1))},get:function(e,t){return(e=k(this,e))<0||this.size===1/0||void 0!==this.size&&e>this.size?t:this.find(function(t,n){return n===e},void 0,t)},has:function(e){return(e=k(this,e))>=0&&(void 0!==this.size?this.size===1/0||e5e3)return e.textContent;return function(e){for(var n,r,o,i,a,u=e.textContent,s=0,l=u[0],c=1,f=e.innerHTML="",p=0;r=n,n=p<7&&"\\"==n?1:c;){if(c=l,l=u[++s],i=f.length>1,!c||p>8&&"\n"==c||[/\S/.test(c),1,1,!/[$\w]/.test(c),("/"==n||"\n"==n)&&i,'"'==n&&i,"'"==n&&i,u[s-4]+r+n=="--\x3e",r+n=="*/"][p])for(f&&(e.appendChild(a=t.createElement("span")).setAttribute("style",["color: #555; font-weight: bold;","","","color: #555;",""][p?p<3?2:p>6?4:p>3?3:+/^(a(bstract|lias|nd|rguments|rray|s(m|sert)?|uto)|b(ase|egin|ool(ean)?|reak|yte)|c(ase|atch|har|hecked|lass|lone|ompl|onst|ontinue)|de(bugger|cimal|clare|f(ault|er)?|init|l(egate|ete)?)|do|double|e(cho|ls?if|lse(if)?|nd|nsure|num|vent|x(cept|ec|p(licit|ort)|te(nds|nsion|rn)))|f(allthrough|alse|inal(ly)?|ixed|loat|or(each)?|riend|rom|unc(tion)?)|global|goto|guard|i(f|mp(lements|licit|ort)|n(it|clude(_once)?|line|out|stanceof|t(erface|ernal)?)?|s)|l(ambda|et|ock|ong)|m(icrolight|odule|utable)|NaN|n(amespace|ative|ext|ew|il|ot|ull)|o(bject|perator|r|ut|verride)|p(ackage|arams|rivate|rotected|rotocol|ublic)|r(aise|e(adonly|do|f|gister|peat|quire(_once)?|scue|strict|try|turn))|s(byte|ealed|elf|hort|igned|izeof|tatic|tring|truct|ubscript|uper|ynchronized|witch)|t(emplate|hen|his|hrows?|ransient|rue|ry|ype(alias|def|id|name|of))|u(n(checked|def(ined)?|ion|less|signed|til)|se|sing)|v(ar|irtual|oid|olatile)|w(char_t|hen|here|hile|ith)|xor|yield)$/.test(f):0]),a.appendChild(t.createTextNode(f))),o=p&&p<7?p:o,f="",p=11;![1,/[\/{}[(\-+*=<>:;|\\.,?!&@~]/.test(c),/[\])]/.test(c),/[$\w]/.test(c),"/"==c&&o<2&&"<"!=n,'"'==c,"'"==c,c+l+u[s+1]+u[s+2]=="\x3c!--",c+l=="/*",c+l=="//","#"==c][--p];);f+=c}}(e)},t.mapToList=function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"key";var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:l.default.Map();if(!l.default.Map.isMap(t)||!t.size)return l.default.List();Array.isArray(n)||(n=[n]);if(n.length<1)return t.merge(r);var a=l.default.List();var u=n[0];var s=!0;var c=!1;var f=void 0;try{for(var p,d=(0,i.default)(t.entries());!(s=(p=d.next()).done);s=!0){var h=p.value,v=(0,o.default)(h,2),m=v[0],g=v[1],y=e(g,n.slice(1),r.set(u,m));a=l.default.List.isList(y)?a.concat(y):a.push(y)}}catch(e){c=!0,f=e}finally{try{!s&&d.return&&d.return()}finally{if(c)throw f}}return a},t.extractFileNameFromContentDispositionHeader=function(e){var t=void 0;if([/filename\*=[^']+'\w*'"([^"]+)";?/i,/filename\*=[^']+'\w*'([^;]+);?/i,/filename="([^;]*);?"/i,/filename=([^;]*);?/i].some(function(n){return null!==(t=n.exec(e))}),null!==t&&t.length>1)try{return decodeURIComponent(t[1])}catch(e){console.error(e)}return null},t.pascalCase=C,t.pascalCaseFilename=function(e){return C(e.replace(/\.[^./]*$/,""))},t.sanitizeUrl=function(e){if("string"!=typeof e||""===e)return"";return(0,c.sanitizeUrl)(e)},t.getAcceptControllingResponse=function(e){if(!l.default.OrderedMap.isOrderedMap(e))return null;if(!e.size)return null;var t=e.find(function(e,t){return t.startsWith("2")&&(0,u.default)(e.get("content")||{}).length>0}),n=e.get("default")||l.default.OrderedMap(),r=(n.get("content")||l.default.OrderedMap()).keySeq().toJS().length?n:null;return t||r},t.deeplyStripKey=function e(t,n){var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:function(){return!0};if("object"!==(void 0===t?"undefined":(0,s.default)(t))||Array.isArray(t)||null===t||!n)return t;var o=(0,a.default)({},t);(0,u.default)(o).forEach(function(t){t===n&&r(o[t],t)?delete o[t]:o[t]=e(o[t],n,r)});return o},t.stringify=function(e){if("string"==typeof e)return e;e.toJS&&(e=e.toJS());if("object"===(void 0===e?"undefined":(0,s.default)(e))&&null!==e)try{return(0,r.default)(e,null,2)}catch(t){return String(e)}return e.toString()},t.numberToString=function(e){if("number"==typeof e)return e.toString();return e},t.paramToIdentifier=q,t.paramToValue=function(e,t){return q(e,{returnAll:!0}).map(function(e){return t[e]}).filter(function(e){return void 0!==e})[0]};var l=_(n(7)),c=n(572),f=_(n(573)),p=_(n(281)),d=_(n(285)),h=_(n(288)),v=_(n(651)),m=_(n(105)),g=n(194),y=_(n(32)),b=_(n(724));function _(e){return e&&e.__esModule?e:{default:e}}var w="default",E=t.isImmutable=function(e){return l.default.Iterable.isIterable(e)};function x(e){return Array.isArray(e)?e:[e]}function S(e){return!!e&&"object"===(void 0===e?"undefined":(0,s.default)(e))}t.memoize=d.default;function C(e){return(0,p.default)((0,f.default)(e))}t.propChecker=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:[],r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:[];return(0,u.default)(e).length!==(0,u.default)(t).length||((0,v.default)(e,function(e,n){if(r.includes(n))return!1;var o=t[n];return l.default.Iterable.isIterable(e)?!l.default.is(e,o):("object"!==(void 0===e?"undefined":(0,s.default)(e))||"object"!==(void 0===o?"undefined":(0,s.default)(o)))&&e!==o})||n.some(function(n){return!(0,m.default)(e[n],t[n])}))};var k=t.validateMaximum=function(e,t){if(e>t)return"Value must be less than Maximum"},A=t.validateMinimum=function(e,t){if(et)return"Value must be less than MaxLength"},D=t.validateMinLength=function(e,t){if(e.length2&&void 0!==arguments[2]?arguments[2]:{},r=n.isOAS3,o=void 0!==r&&r,i=n.bypassRequiredCheck,a=void 0!==i&&i,u=[],c=e.get("required"),f=o?e.get("schema"):e;if(!f)return u;var p=f.get("maximum"),d=f.get("minimum"),h=f.get("type"),v=f.get("format"),m=f.get("maxLength"),g=f.get("minLength"),b=f.get("pattern");if(h&&(c||t)){var _="string"===h&&t,w="array"===h&&Array.isArray(t)&&t.length,E="array"===h&&l.default.List.isList(t)&&t.count(),x="file"===h&&t instanceof y.default.File,S="boolean"===h&&(t||!1===t),C="number"===h&&(t||0===t),U="integer"===h&&(t||0===t),q=!1;if(o&&"object"===h)if("object"===(void 0===t?"undefined":(0,s.default)(t)))q=!0;else if("string"==typeof t)try{JSON.parse(t),q=!0}catch(e){return u.push("Parameter string value must be valid JSON"),u}var F=[_,w,E,x,S,C,U,q].some(function(e){return!!e});if(c&&!F&&!a)return u.push("Required field is not provided"),u;if(b){var z=L(t,b);z&&u.push(z)}if(m||0===m){var B=R(t,m);B&&u.push(B)}if(g){var V=D(t,g);V&&u.push(V)}if(p||0===p){var H=k(t,p);H&&u.push(H)}if(d||0===d){var W=A(t,d);W&&u.push(W)}if("string"===h){var J=void 0;if(!(J="date-time"===v?j(t):"uuid"===v?N(t):I(t)))return u;u.push(J)}else if("boolean"===h){var Y=M(t);if(!Y)return u;u.push(Y)}else if("number"===h){var K=O(t);if(!K)return u;u.push(K)}else if("integer"===h){var G=P(t);if(!G)return u;u.push(G)}else if("array"===h){var $;if(!E||!t.count())return u;$=f.getIn(["items","type"]),t.forEach(function(e,t){var n=void 0;"number"===$?n=O(e):"integer"===$?n=P(e):"string"===$&&(n=I(e)),n&&u.push({index:t,error:n})})}else if("file"===h){var Z=T(t);if(!Z)return u;u.push(Z)}}return u},t.getSampleSchema=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"",n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};if(/xml/.test(t)){if(!e.xml||!e.xml.name){if(e.xml=e.xml||{},!e.$$ref)return e.type||e.items||e.properties||e.additionalProperties?'\n\x3c!-- XML example cannot be generated; root element name is undefined --\x3e':null;var o=e.$$ref.match(/\S*\/(\S+)$/);e.xml.name=o[1]}return(0,g.memoizedCreateXMLExample)(e,n)}var i=(0,g.memoizedSampleFromSchema)(e,n);return"object"===(void 0===i?"undefined":(0,s.default)(i))?(0,r.default)(i,null,2):i},t.parseSearch=function(){var e={},t=y.default.location.search;if(!t)return{};if(""!=t){var n=t.substr(1).split("&");for(var r in n)n.hasOwnProperty(r)&&(r=n[r].split("="),e[decodeURIComponent(r[0])]=r[1]&&decodeURIComponent(r[1])||"")}return e},t.serializeSearch=function(e){return(0,u.default)(e).map(function(t){return encodeURIComponent(t)+"="+encodeURIComponent(e[t])}).join("&")},t.btoa=function(t){return(t instanceof e?t:new e(t.toString(),"utf-8")).toString("base64")},t.sorters={operationsSorter:{alpha:function(e,t){return e.get("path").localeCompare(t.get("path"))},method:function(e,t){return e.get("method").localeCompare(t.get("method"))}},tagsSorter:{alpha:function(e,t){return e.localeCompare(t)}}},t.buildFormData=function(e){var t=[];for(var n in e){var r=e[n];void 0!==r&&""!==r&&t.push([n,"=",encodeURIComponent(r).replace(/%20/g,"+")].join(""))}return t.join("&")},t.shallowEqualKeys=function(e,t,n){return!!(0,h.default)(n,function(n){return(0,m.default)(e[n],t[n])})};var U=t.createDeepLinkPath=function(e){return"string"==typeof e||e instanceof String?e.trim().replace(/\s/g,"%20"):""};t.escapeDeepLinkPath=function(e){return(0,b.default)(U(e).replace(/%20/g,"_"))},t.getExtensions=function(e){return e.filter(function(e,t){return/^x-/.test(t)})},t.getCommonExtensions=function(e){return e.filter(function(e,t){return/^pattern|maxLength|minLength|maximum|minimum/.test(t)})};function q(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=t.returnAll,r=void 0!==n&&n,o=t.allowHashes,i=void 0===o||o;if(!l.default.Map.isMap(e))throw new Error("paramToIdentifier: received a non-Im.Map parameter as input");var a=e.get("name"),u=e.get("in"),s=[];return e&&e.hashCode&&u&&a&&i&&s.push(u+"."+a+".hash-"+e.hashCode()),u&&a&&s.push(u+"."+a),s.push(a),r?s:s[0]||""}}).call(t,n(54).Buffer)},function(e,t,n){"use strict";var r=n(34);e.exports=r},function(e,t,n){"use strict";e.exports=function(e){for(var t=arguments.length-1,n="Minified React error #"+e+"; visit http://facebook.github.io/react/docs/error-decoder.html?invariant="+e,r=0;r>",i={listOf:function(e){return l(e,"List",r.List.isList)},mapOf:function(e,t){return c(e,t,"Map",r.Map.isMap)},orderedMapOf:function(e,t){return c(e,t,"OrderedMap",r.OrderedMap.isOrderedMap)},setOf:function(e){return l(e,"Set",r.Set.isSet)},orderedSetOf:function(e){return l(e,"OrderedSet",r.OrderedSet.isOrderedSet)},stackOf:function(e){return l(e,"Stack",r.Stack.isStack)},iterableOf:function(e){return l(e,"Iterable",r.Iterable.isIterable)},recordOf:function(e){return u(function(t,n,o,i,u){for(var s=arguments.length,l=Array(s>5?s-5:0),c=5;c6?s-6:0),c=6;c5?l-5:0),f=5;f5?i-5:0),u=5;u key("+c[f]+")"].concat(a));if(d instanceof Error)return d}})).apply(void 0,i);var s})}function f(e){var t=void 0===arguments[1]?"Iterable":arguments[1],n=void 0===arguments[2]?r.Iterable.isIterable:arguments[2];return u(function(r,o,i,u,s){for(var l=arguments.length,c=Array(l>5?l-5:0),f=5;f?@[\]^_`{|}~-])/g;function a(e){return!(e>=55296&&e<=57343)&&(!(e>=64976&&e<=65007)&&(65535!=(65535&e)&&65534!=(65535&e)&&(!(e>=0&&e<=8)&&(11!==e&&(!(e>=14&&e<=31)&&(!(e>=127&&e<=159)&&!(e>1114111)))))))}function u(e){if(e>65535){var t=55296+((e-=65536)>>10),n=56320+(1023&e);return String.fromCharCode(t,n)}return String.fromCharCode(e)}var s=/&([a-z#][a-z0-9]{1,31});/gi,l=/^#((?:x[a-f0-9]{1,8}|[0-9]{1,8}))/i,c=n(417);function f(e,t){var n=0;return o(c,t)?c[t]:35===t.charCodeAt(0)&&l.test(t)&&a(n="x"===t[1].toLowerCase()?parseInt(t.slice(2),16):parseInt(t.slice(1),10))?u(n):e}var p=/[&<>"]/,d=/[&<>"]/g,h={"&":"&","<":"<",">":">",'"':"""};function v(e){return h[e]}t.assign=function(e){return[].slice.call(arguments,1).forEach(function(t){if(t){if("object"!=typeof t)throw new TypeError(t+"must be object");Object.keys(t).forEach(function(n){e[n]=t[n]})}}),e},t.isString=function(e){return"[object String]"===function(e){return Object.prototype.toString.call(e)}(e)},t.has=o,t.unescapeMd=function(e){return e.indexOf("\\")<0?e:e.replace(i,"$1")},t.isValidEntityCode=a,t.fromCodePoint=u,t.replaceEntities=function(e){return e.indexOf("&")<0?e:e.replace(s,f)},t.escapeHtml=function(e){return p.test(e)?e.replace(d,v):e}},function(e,t){e.exports=function(e){return"object"==typeof e?null!==e:"function"==typeof e}},function(e,t,n){var r=n(33),o=n(61),i=n(59),a=n(73),u=n(120),s=function(e,t,n){var l,c,f,p,d=e&s.F,h=e&s.G,v=e&s.S,m=e&s.P,g=e&s.B,y=h?r:v?r[t]||(r[t]={}):(r[t]||{}).prototype,b=h?o:o[t]||(o[t]={}),_=b.prototype||(b.prototype={});for(l in h&&(n=t),n)f=((c=!d&&y&&void 0!==y[l])?y:n)[l],p=g&&c?u(f,r):m&&"function"==typeof f?u(Function.call,f):f,y&&a(y,l,f,e&s.U),b[l]!=f&&i(b,l,p),m&&_[l]!=f&&(_[l]=f)};r.core=o,s.F=1,s.G=2,s.S=4,s.P=8,s.B=16,s.W=32,s.U=64,s.R=128,e.exports=s},function(e,t,n){var r=n(29),o=n(101),i=n(53),a=/"/g,u=function(e,t,n,r){var o=String(i(e)),u="<"+t;return""!==n&&(u+=" "+n+'="'+String(r).replace(a,""")+'"'),u+">"+o+""};e.exports=function(e,t){var n={};n[e]=t(u),r(r.P+r.F*o(function(){var t=""[e]('"');return t!==t.toLowerCase()||t.split('"').length>3}),"String",n)}},function(e,t){var n;n=function(){return this}();try{n=n||Function("return this")()||(0,eval)("this")}catch(e){"object"==typeof window&&(n=window)}e.exports=n},function(e,t,n){"use strict";var r,o=n(91),i=(r=o)&&r.__esModule?r:{default:r};e.exports=function(){var e={location:{},history:{},open:function(){},close:function(){},File:function(){}};if("undefined"==typeof window)return e;try{e=window;var t=!0,n=!1,r=void 0;try{for(var o,a=(0,i.default)(["File","Blob","FormData"]);!(t=(o=a.next()).done);t=!0){var u=o.value;u in window&&(e[u]=window[u])}}catch(e){n=!0,r=e}finally{try{!t&&a.return&&a.return()}finally{if(n)throw r}}}catch(e){console.error(e)}return e}()},function(e,t){var n=e.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=n)},function(e,t,n){"use strict";function r(e){return function(){return e}}var o=function(){};o.thatReturns=r,o.thatReturnsFalse=r(!1),o.thatReturnsTrue=r(!0),o.thatReturnsNull=r(null),o.thatReturnsThis=function(){return this},o.thatReturnsArgument=function(e){return e},e.exports=o},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=i(n(25));t.isOAS3=a,t.isSwagger2=function(e){var t=e.get("swagger");if("string"!=typeof t)return!1;return t.startsWith("2.0")},t.OAS3ComponentWrapFactory=function(e){return function(t,n){return function(i){if(n&&n.specSelectors&&n.specSelectors.specJson){var u=n.specSelectors.specJson();return a(u)?o.default.createElement(e,(0,r.default)({},i,n,{Ori:t})):o.default.createElement(t,i)}return console.warn("OAS3 wrapper: couldn't get spec"),null}}};var o=i(n(0));function i(e){return e&&e.__esModule?e:{default:e}}function a(e){var t=e.get("openapi");return"string"==typeof t&&(t.startsWith("3.0.")&&t.length>4)}},function(e,t,n){var r=n(28);e.exports=function(e){if(!r(e))throw TypeError(e+" is not an object!");return e}},function(e,t,n){var r=n(279),o="object"==typeof self&&self&&self.Object===Object&&self,i=r||o||Function("return this")();e.exports=i},function(e,t){e.exports=function(e){var t=typeof e;return null!=e&&("object"==t||"function"==t)}},function(e,t,n){"use strict";var r=null;e.exports={debugTool:r}},function(e,t,n){var r=n(36),o=n(239),i=n(158),a=Object.defineProperty;t.f=n(44)?Object.defineProperty:function(e,t,n){if(r(e),t=i(t,!0),r(n),o)try{return a(e,t,n)}catch(e){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(e[t]=n.value),e}},function(e,t,n){e.exports={default:n(517),__esModule:!0}},function(e,t,n){e.exports={default:n(518),__esModule:!0}},function(e,t,n){"use strict";var r=n(11),o=n(13),i=n(354),a=n(69),u=n(355),s=n(88),l=n(148),c=n(8),f=[],p=0,d=i.getPooled(),h=!1,v=null;function m(){E.ReactReconcileTransaction&&v||r("123")}var g=[{initialize:function(){this.dirtyComponentsLength=f.length},close:function(){this.dirtyComponentsLength!==f.length?(f.splice(0,this.dirtyComponentsLength),w()):f.length=0}},{initialize:function(){this.callbackQueue.reset()},close:function(){this.callbackQueue.notifyAll()}}];function y(){this.reinitializeTransaction(),this.dirtyComponentsLength=null,this.callbackQueue=i.getPooled(),this.reconcileTransaction=E.ReactReconcileTransaction.getPooled(!0)}function b(e,t){return e._mountOrder-t._mountOrder}function _(e){var t=e.dirtyComponentsLength;t!==f.length&&r("124",t,f.length),f.sort(b),p++;for(var n=0;n + * @license MIT + */ +var r=n(529),o=n(530),i=n(262);function a(){return s.TYPED_ARRAY_SUPPORT?2147483647:1073741823}function u(e,t){if(a()=a())throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+a().toString(16)+" bytes");return 0|e}function h(e,t){if(s.isBuffer(e))return e.length;if("undefined"!=typeof ArrayBuffer&&"function"==typeof ArrayBuffer.isView&&(ArrayBuffer.isView(e)||e instanceof ArrayBuffer))return e.byteLength;"string"!=typeof e&&(e=""+e);var n=e.length;if(0===n)return 0;for(var r=!1;;)switch(t){case"ascii":case"latin1":case"binary":return n;case"utf8":case"utf-8":case void 0:return F(e).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*n;case"hex":return n>>>1;case"base64":return z(e).length;default:if(r)return F(e).length;t=(""+t).toLowerCase(),r=!0}}function v(e,t,n){var r=e[t];e[t]=e[n],e[n]=r}function m(e,t,n,r,o){if(0===e.length)return-1;if("string"==typeof n?(r=n,n=0):n>2147483647?n=2147483647:n<-2147483648&&(n=-2147483648),n=+n,isNaN(n)&&(n=o?0:e.length-1),n<0&&(n=e.length+n),n>=e.length){if(o)return-1;n=e.length-1}else if(n<0){if(!o)return-1;n=0}if("string"==typeof t&&(t=s.from(t,r)),s.isBuffer(t))return 0===t.length?-1:g(e,t,n,r,o);if("number"==typeof t)return t&=255,s.TYPED_ARRAY_SUPPORT&&"function"==typeof Uint8Array.prototype.indexOf?o?Uint8Array.prototype.indexOf.call(e,t,n):Uint8Array.prototype.lastIndexOf.call(e,t,n):g(e,[t],n,r,o);throw new TypeError("val must be string, number or Buffer")}function g(e,t,n,r,o){var i,a=1,u=e.length,s=t.length;if(void 0!==r&&("ucs2"===(r=String(r).toLowerCase())||"ucs-2"===r||"utf16le"===r||"utf-16le"===r)){if(e.length<2||t.length<2)return-1;a=2,u/=2,s/=2,n/=2}function l(e,t){return 1===a?e[t]:e.readUInt16BE(t*a)}if(o){var c=-1;for(i=n;iu&&(n=u-s),i=n;i>=0;i--){for(var f=!0,p=0;po&&(r=o):r=o;var i=t.length;if(i%2!=0)throw new TypeError("Invalid hex string");r>i/2&&(r=i/2);for(var a=0;a>8,o=n%256,i.push(o),i.push(r);return i}(t,e.length-n),e,n,r)}function S(e,t,n){return 0===t&&n===e.length?r.fromByteArray(e):r.fromByteArray(e.slice(t,n))}function C(e,t,n){n=Math.min(e.length,n);for(var r=[],o=t;o239?4:l>223?3:l>191?2:1;if(o+f<=n)switch(f){case 1:l<128&&(c=l);break;case 2:128==(192&(i=e[o+1]))&&(s=(31&l)<<6|63&i)>127&&(c=s);break;case 3:i=e[o+1],a=e[o+2],128==(192&i)&&128==(192&a)&&(s=(15&l)<<12|(63&i)<<6|63&a)>2047&&(s<55296||s>57343)&&(c=s);break;case 4:i=e[o+1],a=e[o+2],u=e[o+3],128==(192&i)&&128==(192&a)&&128==(192&u)&&(s=(15&l)<<18|(63&i)<<12|(63&a)<<6|63&u)>65535&&s<1114112&&(c=s)}null===c?(c=65533,f=1):c>65535&&(c-=65536,r.push(c>>>10&1023|55296),c=56320|1023&c),r.push(c),o+=f}return function(e){var t=e.length;if(t<=k)return String.fromCharCode.apply(String,e);var n="",r=0;for(;rthis.length)return"";if((void 0===n||n>this.length)&&(n=this.length),n<=0)return"";if((n>>>=0)<=(t>>>=0))return"";for(e||(e="utf8");;)switch(e){case"hex":return P(this,t,n);case"utf8":case"utf-8":return C(this,t,n);case"ascii":return A(this,t,n);case"latin1":case"binary":return O(this,t,n);case"base64":return S(this,t,n);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return T(this,t,n);default:if(r)throw new TypeError("Unknown encoding: "+e);e=(e+"").toLowerCase(),r=!0}}.apply(this,arguments)},s.prototype.equals=function(e){if(!s.isBuffer(e))throw new TypeError("Argument must be a Buffer");return this===e||0===s.compare(this,e)},s.prototype.inspect=function(){var e="",n=t.INSPECT_MAX_BYTES;return this.length>0&&(e=this.toString("hex",0,n).match(/.{2}/g).join(" "),this.length>n&&(e+=" ... ")),""},s.prototype.compare=function(e,t,n,r,o){if(!s.isBuffer(e))throw new TypeError("Argument must be a Buffer");if(void 0===t&&(t=0),void 0===n&&(n=e?e.length:0),void 0===r&&(r=0),void 0===o&&(o=this.length),t<0||n>e.length||r<0||o>this.length)throw new RangeError("out of range index");if(r>=o&&t>=n)return 0;if(r>=o)return-1;if(t>=n)return 1;if(t>>>=0,n>>>=0,r>>>=0,o>>>=0,this===e)return 0;for(var i=o-r,a=n-t,u=Math.min(i,a),l=this.slice(r,o),c=e.slice(t,n),f=0;fo)&&(n=o),e.length>0&&(n<0||t<0)||t>this.length)throw new RangeError("Attempt to write outside buffer bounds");r||(r="utf8");for(var i=!1;;)switch(r){case"hex":return y(this,e,t,n);case"utf8":case"utf-8":return b(this,e,t,n);case"ascii":return _(this,e,t,n);case"latin1":case"binary":return w(this,e,t,n);case"base64":return E(this,e,t,n);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return x(this,e,t,n);default:if(i)throw new TypeError("Unknown encoding: "+r);r=(""+r).toLowerCase(),i=!0}},s.prototype.toJSON=function(){return{type:"Buffer",data:Array.prototype.slice.call(this._arr||this,0)}};var k=4096;function A(e,t,n){var r="";n=Math.min(e.length,n);for(var o=t;or)&&(n=r);for(var o="",i=t;in)throw new RangeError("Trying to access beyond buffer length")}function I(e,t,n,r,o,i){if(!s.isBuffer(e))throw new TypeError('"buffer" argument must be a Buffer instance');if(t>o||te.length)throw new RangeError("Index out of range")}function j(e,t,n,r){t<0&&(t=65535+t+1);for(var o=0,i=Math.min(e.length-n,2);o>>8*(r?o:1-o)}function N(e,t,n,r){t<0&&(t=4294967295+t+1);for(var o=0,i=Math.min(e.length-n,4);o>>8*(r?o:3-o)&255}function R(e,t,n,r,o,i){if(n+r>e.length)throw new RangeError("Index out of range");if(n<0)throw new RangeError("Index out of range")}function D(e,t,n,r,i){return i||R(e,0,n,4),o.write(e,t,n,r,23,4),n+4}function L(e,t,n,r,i){return i||R(e,0,n,8),o.write(e,t,n,r,52,8),n+8}s.prototype.slice=function(e,t){var n,r=this.length;if(e=~~e,t=void 0===t?r:~~t,e<0?(e+=r)<0&&(e=0):e>r&&(e=r),t<0?(t+=r)<0&&(t=0):t>r&&(t=r),t0&&(o*=256);)r+=this[e+--t]*o;return r},s.prototype.readUInt8=function(e,t){return t||M(e,1,this.length),this[e]},s.prototype.readUInt16LE=function(e,t){return t||M(e,2,this.length),this[e]|this[e+1]<<8},s.prototype.readUInt16BE=function(e,t){return t||M(e,2,this.length),this[e]<<8|this[e+1]},s.prototype.readUInt32LE=function(e,t){return t||M(e,4,this.length),(this[e]|this[e+1]<<8|this[e+2]<<16)+16777216*this[e+3]},s.prototype.readUInt32BE=function(e,t){return t||M(e,4,this.length),16777216*this[e]+(this[e+1]<<16|this[e+2]<<8|this[e+3])},s.prototype.readIntLE=function(e,t,n){e|=0,t|=0,n||M(e,t,this.length);for(var r=this[e],o=1,i=0;++i=(o*=128)&&(r-=Math.pow(2,8*t)),r},s.prototype.readIntBE=function(e,t,n){e|=0,t|=0,n||M(e,t,this.length);for(var r=t,o=1,i=this[e+--r];r>0&&(o*=256);)i+=this[e+--r]*o;return i>=(o*=128)&&(i-=Math.pow(2,8*t)),i},s.prototype.readInt8=function(e,t){return t||M(e,1,this.length),128&this[e]?-1*(255-this[e]+1):this[e]},s.prototype.readInt16LE=function(e,t){t||M(e,2,this.length);var n=this[e]|this[e+1]<<8;return 32768&n?4294901760|n:n},s.prototype.readInt16BE=function(e,t){t||M(e,2,this.length);var n=this[e+1]|this[e]<<8;return 32768&n?4294901760|n:n},s.prototype.readInt32LE=function(e,t){return t||M(e,4,this.length),this[e]|this[e+1]<<8|this[e+2]<<16|this[e+3]<<24},s.prototype.readInt32BE=function(e,t){return t||M(e,4,this.length),this[e]<<24|this[e+1]<<16|this[e+2]<<8|this[e+3]},s.prototype.readFloatLE=function(e,t){return t||M(e,4,this.length),o.read(this,e,!0,23,4)},s.prototype.readFloatBE=function(e,t){return t||M(e,4,this.length),o.read(this,e,!1,23,4)},s.prototype.readDoubleLE=function(e,t){return t||M(e,8,this.length),o.read(this,e,!0,52,8)},s.prototype.readDoubleBE=function(e,t){return t||M(e,8,this.length),o.read(this,e,!1,52,8)},s.prototype.writeUIntLE=function(e,t,n,r){(e=+e,t|=0,n|=0,r)||I(this,e,t,n,Math.pow(2,8*n)-1,0);var o=1,i=0;for(this[t]=255&e;++i=0&&(i*=256);)this[t+o]=e/i&255;return t+n},s.prototype.writeUInt8=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,1,255,0),s.TYPED_ARRAY_SUPPORT||(e=Math.floor(e)),this[t]=255&e,t+1},s.prototype.writeUInt16LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,65535,0),s.TYPED_ARRAY_SUPPORT?(this[t]=255&e,this[t+1]=e>>>8):j(this,e,t,!0),t+2},s.prototype.writeUInt16BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,65535,0),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>8,this[t+1]=255&e):j(this,e,t,!1),t+2},s.prototype.writeUInt32LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,4294967295,0),s.TYPED_ARRAY_SUPPORT?(this[t+3]=e>>>24,this[t+2]=e>>>16,this[t+1]=e>>>8,this[t]=255&e):N(this,e,t,!0),t+4},s.prototype.writeUInt32BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,4294967295,0),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>24,this[t+1]=e>>>16,this[t+2]=e>>>8,this[t+3]=255&e):N(this,e,t,!1),t+4},s.prototype.writeIntLE=function(e,t,n,r){if(e=+e,t|=0,!r){var o=Math.pow(2,8*n-1);I(this,e,t,n,o-1,-o)}var i=0,a=1,u=0;for(this[t]=255&e;++i>0)-u&255;return t+n},s.prototype.writeIntBE=function(e,t,n,r){if(e=+e,t|=0,!r){var o=Math.pow(2,8*n-1);I(this,e,t,n,o-1,-o)}var i=n-1,a=1,u=0;for(this[t+i]=255&e;--i>=0&&(a*=256);)e<0&&0===u&&0!==this[t+i+1]&&(u=1),this[t+i]=(e/a>>0)-u&255;return t+n},s.prototype.writeInt8=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,1,127,-128),s.TYPED_ARRAY_SUPPORT||(e=Math.floor(e)),e<0&&(e=255+e+1),this[t]=255&e,t+1},s.prototype.writeInt16LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,32767,-32768),s.TYPED_ARRAY_SUPPORT?(this[t]=255&e,this[t+1]=e>>>8):j(this,e,t,!0),t+2},s.prototype.writeInt16BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,32767,-32768),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>8,this[t+1]=255&e):j(this,e,t,!1),t+2},s.prototype.writeInt32LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,2147483647,-2147483648),s.TYPED_ARRAY_SUPPORT?(this[t]=255&e,this[t+1]=e>>>8,this[t+2]=e>>>16,this[t+3]=e>>>24):N(this,e,t,!0),t+4},s.prototype.writeInt32BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,2147483647,-2147483648),e<0&&(e=4294967295+e+1),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>24,this[t+1]=e>>>16,this[t+2]=e>>>8,this[t+3]=255&e):N(this,e,t,!1),t+4},s.prototype.writeFloatLE=function(e,t,n){return D(this,e,t,!0,n)},s.prototype.writeFloatBE=function(e,t,n){return D(this,e,t,!1,n)},s.prototype.writeDoubleLE=function(e,t,n){return L(this,e,t,!0,n)},s.prototype.writeDoubleBE=function(e,t,n){return L(this,e,t,!1,n)},s.prototype.copy=function(e,t,n,r){if(n||(n=0),r||0===r||(r=this.length),t>=e.length&&(t=e.length),t||(t=0),r>0&&r=this.length)throw new RangeError("sourceStart out of bounds");if(r<0)throw new RangeError("sourceEnd out of bounds");r>this.length&&(r=this.length),e.length-t=0;--o)e[o+t]=this[o+n];else if(i<1e3||!s.TYPED_ARRAY_SUPPORT)for(o=0;o>>=0,n=void 0===n?this.length:n>>>0,e||(e=0),"number"==typeof e)for(i=t;i55295&&n<57344){if(!o){if(n>56319){(t-=3)>-1&&i.push(239,191,189);continue}if(a+1===r){(t-=3)>-1&&i.push(239,191,189);continue}o=n;continue}if(n<56320){(t-=3)>-1&&i.push(239,191,189),o=n;continue}n=65536+(o-55296<<10|n-56320)}else o&&(t-=3)>-1&&i.push(239,191,189);if(o=null,n<128){if((t-=1)<0)break;i.push(n)}else if(n<2048){if((t-=2)<0)break;i.push(n>>6|192,63&n|128)}else if(n<65536){if((t-=3)<0)break;i.push(n>>12|224,n>>6&63|128,63&n|128)}else{if(!(n<1114112))throw new Error("Invalid code point");if((t-=4)<0)break;i.push(n>>18|240,n>>12&63|128,n>>6&63|128,63&n|128)}}return i}function z(e){return r.toByteArray(function(e){if((e=function(e){return e.trim?e.trim():e.replace(/^\s+|\s+$/g,"")}(e).replace(U,"")).length<2)return"";for(;e.length%4!=0;)e+="=";return e}(e))}function B(e,t,n,r){for(var o=0;o=t.length||o>=e.length);++o)t[o+n]=e[o];return o}}).call(t,n(31))},function(e,t,n){var r=n(278);e.exports=function(e){return null==e?"":r(e)}},function(e,t){var n,r,o=e.exports={};function i(){throw new Error("setTimeout has not been defined")}function a(){throw new Error("clearTimeout has not been defined")}function u(e){if(n===setTimeout)return setTimeout(e,0);if((n===i||!n)&&setTimeout)return n=setTimeout,setTimeout(e,0);try{return n(e,0)}catch(t){try{return n.call(null,e,0)}catch(t){return n.call(this,e,0)}}}!function(){try{n="function"==typeof setTimeout?setTimeout:i}catch(e){n=i}try{r="function"==typeof clearTimeout?clearTimeout:a}catch(e){r=a}}();var s,l=[],c=!1,f=-1;function p(){c&&s&&(c=!1,s.length?l=s.concat(l):f=-1,l.length&&d())}function d(){if(!c){var e=u(p);c=!0;for(var t=l.length;t;){for(s=l,l=[];++f1)for(var n=1;n1?t-1:0),r=1;r2?n-2:0),o=2;o1){for(var h=Array(d),v=0;v1){for(var g=Array(m),y=0;y=0||Object.prototype.hasOwnProperty.call(e,r)&&(n[r]=e[r]);return n}},function(e,t,n){"use strict";function r(e){return void 0===e||null===e}e.exports.isNothing=r,e.exports.isObject=function(e){return"object"==typeof e&&null!==e},e.exports.toArray=function(e){return Array.isArray(e)?e:r(e)?[]:[e]},e.exports.repeat=function(e,t){var n,r="";for(n=0;n=t.length?{value:void 0,done:!0}:(e=r(t,n),this._i+=e.length,{value:e,done:!1})})},function(e,t){var n={}.toString;e.exports=function(e){return n.call(e).slice(8,-1)}},function(e,t,n){e.exports=!n(101)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(e,t){e.exports=function(e){try{return!!e()}catch(e){return!0}}},function(e,t){e.exports={}},function(e,t,n){var r=n(119),o=Math.min;e.exports=function(e){return e>0?o(r(e),9007199254740991):0}},function(e,t,n){"use strict";e.exports=function(e){for(var t=arguments.length-1,n="Minified React error #"+e+"; visit http://facebook.github.io/react/docs/error-decoder.html?invariant="+e,r=0;r0?o(r(e),9007199254740991):0}},function(e,t){var n=0,r=Math.random();e.exports=function(e){return"Symbol(".concat(void 0===e?"":e,")_",(++n+r).toString(36))}},function(e,t,n){var r=n(60),o=n(460),i=n(461),a=Object.defineProperty;t.f=n(100)?Object.defineProperty:function(e,t,n){if(r(e),t=i(t,!0),r(n),o)try{return a(e,t,n)}catch(e){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(e[t]=n.value),e}},function(e,t){var n={}.hasOwnProperty;e.exports=function(e,t){return n.call(e,t)}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t,n){var r=n(121);e.exports=function(e,t,n){if(r(e),void 0===t)return e;switch(n){case 1:return function(n){return e.call(t,n)};case 2:return function(n,r){return e.call(t,n,r)};case 3:return function(n,r,o){return e.call(t,n,r,o)}}return function(){return e.apply(t,arguments)}}},function(e,t){e.exports=function(e){if("function"!=typeof e)throw TypeError(e+" is not a function!");return e}},function(e,t,n){var r=n(466),o=n(53);e.exports=function(e){return r(o(e))}},function(e,t,n){"use strict";var r=n(59),o=n(73),i=n(101),a=n(53),u=n(18);e.exports=function(e,t,n){var s=u(e),l=n(a,s,""[e]),c=l[0],f=l[1];i(function(){var t={};return t[s]=function(){return 7},7!=""[e](t)})&&(o(String.prototype,e,c),r(RegExp.prototype,s,2==t?function(e,t){return f.call(e,this,t)}:function(e){return f.call(e,this)}))}},function(e,t,n){var r=n(116)("meta"),o=n(28),i=n(52),a=n(40).f,u=0,s=Object.isExtensible||function(){return!0},l=!n(51)(function(){return s(Object.preventExtensions({}))}),c=function(e){a(e,r,{value:{i:"O"+ ++u,w:{}}})},f=e.exports={KEY:r,NEED:!1,fastKey:function(e,t){if(!o(e))return"symbol"==typeof e?e:("string"==typeof e?"S":"P")+e;if(!i(e,r)){if(!s(e))return"F";if(!t)return"E";c(e)}return e[r].i},getWeak:function(e,t){if(!i(e,r)){if(!s(e))return!0;if(!t)return!1;c(e)}return e[r].w},onFreeze:function(e){return l&&f.NEED&&s(e)&&!i(e,r)&&c(e),e}}},function(e,t){t.f={}.propertyIsEnumerable},function(e,t,n){"use strict";var r={};e.exports=r},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.CLEAR_BY=t.CLEAR=t.NEW_AUTH_ERR=t.NEW_SPEC_ERR_BATCH=t.NEW_SPEC_ERR=t.NEW_THROWN_ERR_BATCH=t.NEW_THROWN_ERR=void 0,t.newThrownErr=function(e){return{type:a,payload:(0,i.default)(e)}},t.newThrownErrBatch=function(e){return{type:u,payload:e}},t.newSpecErr=function(e){return{type:s,payload:e}},t.newSpecErrBatch=function(e){return{type:l,payload:e}},t.newAuthErr=function(e){return{type:c,payload:e}},t.clear=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return{type:f,payload:e}},t.clearBy=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(){return!0};return{type:p,payload:e}};var r,o=n(180),i=(r=o)&&r.__esModule?r:{default:r};var a=t.NEW_THROWN_ERR="err_new_thrown_err",u=t.NEW_THROWN_ERR_BATCH="err_new_thrown_err_batch",s=t.NEW_SPEC_ERR="err_new_spec_err",l=t.NEW_SPEC_ERR_BATCH="err_new_spec_err_batch",c=t.NEW_AUTH_ERR="err_new_auth_err",f=t.CLEAR="err_clear",p=t.CLEAR_BY="err_clear_by"},function(e,t,n){var r=n(62),o=n(47),i="[object Symbol]";e.exports=function(e){return"symbol"==typeof e||o(e)&&r(e)==i}},function(e,t,n){var r=n(63)(Object,"create");e.exports=r},function(e,t,n){var r=n(601),o=n(602),i=n(603),a=n(604),u=n(605);function s(e){var t=-1,n=null==e?0:e.length;for(this.clear();++t-1&&e%1==0&&e1&&void 0!==arguments[1]?arguments[1]:"";if(u.List.isList(e))return e.some(function(e){return u.Map.isMap(e)&&e.get("in")===t})},t.parametersIncludeType=T,t.contentTypeValues=function(e,t){t=t||[];var n=d(e).getIn(["paths"].concat((0,o.default)(t)),(0,u.fromJS)({})),r=e.getIn(["meta","paths"].concat((0,o.default)(t)),(0,u.fromJS)({})),i=M(e,t),a=n.get("parameters")||new u.List,s=r.get("consumes_value")?r.get("consumes_value"):T(a,"file")?"multipart/form-data":T(a,"formData")?"application/x-www-form-urlencoded":void 0;return(0,u.fromJS)({requestContentType:s,responseContentType:i})},t.currentProducesFor=M,t.producesOptionsFor=function(e,t){t=t||[];var n=d(e),i=n.getIn(["paths"].concat((0,o.default)(t)),null);if(null===i)return;var a=t,u=(0,r.default)(a,1)[0],s=i.get("produces",null),l=n.getIn(["paths",u,"produces"],null),c=n.getIn(["produces"],null);return s||l||c},t.consumesOptionsFor=function(e,t){t=t||[];var n=d(e),i=n.getIn(["paths"].concat((0,o.default)(t)),null);if(null===i)return;var a=t,u=(0,r.default)(a,1)[0],s=i.get("consumes",null),l=n.getIn(["paths",u,"consumes"],null),c=n.getIn(["consumes"],null);return s||l||c};var i=n(58),a=n(9),u=n(7);function s(e){return e&&e.__esModule?e:{default:e}}var l=["get","put","post","delete","options","head","patch","trace"],c=function(e){return e||(0,u.Map)()},f=(t.lastError=(0,i.createSelector)(c,function(e){return e.get("lastError")}),t.url=(0,i.createSelector)(c,function(e){return e.get("url")}),t.specStr=(0,i.createSelector)(c,function(e){return e.get("spec")||""}),t.specSource=(0,i.createSelector)(c,function(e){return e.get("specSource")||"not-editor"}),t.specJson=(0,i.createSelector)(c,function(e){return e.get("json",(0,u.Map)())})),p=(t.specResolved=(0,i.createSelector)(c,function(e){return e.get("resolved",(0,u.Map)())}),t.specResolvedSubtree=function(e,t){return e.getIn(["resolvedSubtrees"].concat((0,o.default)(t)),void 0)},function e(t,n){return u.Map.isMap(t)&&u.Map.isMap(n)?n.get("$$ref")?n:(0,u.OrderedMap)().mergeWith(e,t,n):n}),d=t.specJsonWithResolvedSubtrees=(0,i.createSelector)(c,function(e){return(0,u.OrderedMap)().mergeWith(p,e.get("json"),e.get("resolvedSubtrees"))}),h=t.spec=function(e){return f(e)},v=(t.isOAS3=(0,i.createSelector)(h,function(){return!1}),t.info=(0,i.createSelector)(h,function(e){return j(e&&e.get("info"))})),m=(t.externalDocs=(0,i.createSelector)(h,function(e){return j(e&&e.get("externalDocs"))}),t.version=(0,i.createSelector)(v,function(e){return e&&e.get("version")})),g=(t.semver=(0,i.createSelector)(m,function(e){return/v?([0-9]*)\.([0-9]*)\.([0-9]*)/i.exec(e).slice(1)}),t.paths=(0,i.createSelector)(d,function(e){return e.get("paths")})),y=t.operations=(0,i.createSelector)(g,function(e){if(!e||e.size<1)return(0,u.List)();var t=(0,u.List)();return e&&e.forEach?(e.forEach(function(e,n){if(!e||!e.forEach)return{};e.forEach(function(e,r){l.indexOf(r)<0||(t=t.push((0,u.fromJS)({path:n,method:r,operation:e,id:r+"-"+n})))})}),t):(0,u.List)()}),b=t.consumes=(0,i.createSelector)(h,function(e){return(0,u.Set)(e.get("consumes"))}),_=t.produces=(0,i.createSelector)(h,function(e){return(0,u.Set)(e.get("produces"))}),w=(t.security=(0,i.createSelector)(h,function(e){return e.get("security",(0,u.List)())}),t.securityDefinitions=(0,i.createSelector)(h,function(e){return e.get("securityDefinitions")}),t.findDefinition=function(e,t){var n=e.getIn(["resolvedSubtrees","definitions",t],null),r=e.getIn(["json","definitions",t],null);return n||r||null},t.definitions=(0,i.createSelector)(h,function(e){var t=e.get("definitions");return u.Map.isMap(t)?t:(0,u.Map)()}),t.basePath=(0,i.createSelector)(h,function(e){return e.get("basePath")}),t.host=(0,i.createSelector)(h,function(e){return e.get("host")}),t.schemes=(0,i.createSelector)(h,function(e){return e.get("schemes",(0,u.Map)())}),t.operationsWithRootInherited=(0,i.createSelector)(y,b,_,function(e,t,n){return e.map(function(e){return e.update("operation",function(e){if(e){if(!u.Map.isMap(e))return;return e.withMutations(function(e){return e.get("consumes")||e.update("consumes",function(e){return(0,u.Set)(e).merge(t)}),e.get("produces")||e.update("produces",function(e){return(0,u.Set)(e).merge(n)}),e})}return(0,u.Map)()})})})),E=t.tags=(0,i.createSelector)(h,function(e){var t=e.get("tags",(0,u.List)());return u.List.isList(t)?t.filter(function(e){return u.Map.isMap(e)}):(0,u.List)()}),x=t.tagDetails=function(e,t){return(E(e)||(0,u.List)()).filter(u.Map.isMap).find(function(e){return e.get("name")===t},(0,u.Map)())},S=t.operationsWithTags=(0,i.createSelector)(w,E,function(e,t){return e.reduce(function(e,t){var n=(0,u.Set)(t.getIn(["operation","tags"]));return n.count()<1?e.update("default",(0,u.List)(),function(e){return e.push(t)}):n.reduce(function(e,n){return e.update(n,(0,u.List)(),function(e){return e.push(t)})},e)},t.reduce(function(e,t){return e.set(t.get("name"),(0,u.List)())},(0,u.OrderedMap)()))}),C=(t.taggedOperations=function(e){return function(t){var n=(0,t.getConfigs)(),r=n.tagsSorter,o=n.operationsSorter;return S(e).sortBy(function(e,t){return t},function(e,t){var n="function"==typeof r?r:a.sorters.tagsSorter[r];return n?n(e,t):null}).map(function(t,n){var r="function"==typeof o?o:a.sorters.operationsSorter[o],i=r?t.sort(r):t;return(0,u.Map)({tagDetails:x(e,n),operations:i})})}},t.responses=(0,i.createSelector)(c,function(e){return e.get("responses",(0,u.Map)())})),k=t.requests=(0,i.createSelector)(c,function(e){return e.get("requests",(0,u.Map)())}),A=t.mutatedRequests=(0,i.createSelector)(c,function(e){return e.get("mutatedRequests",(0,u.Map)())}),O=(t.responseFor=function(e,t,n){return C(e).getIn([t,n],null)},t.requestFor=function(e,t,n){return k(e).getIn([t,n],null)},t.mutatedRequestFor=function(e,t,n){return A(e).getIn([t,n],null)},t.allowTryItOutFor=function(){return!0},t.parameterWithMetaByIdentity=function(e,t,n){var r=d(e).getIn(["paths"].concat((0,o.default)(t),["parameters"]),(0,u.OrderedMap)()),i=e.getIn(["meta","paths"].concat((0,o.default)(t),["parameters"]),(0,u.OrderedMap)());return r.map(function(e){var t=i.get(n.get("in")+"."+n.get("name")),r=i.get(n.get("in")+"."+n.get("name")+".hash-"+n.hashCode());return(0,u.OrderedMap)().merge(e,t,r)}).find(function(e){return e.get("in")===n.get("in")&&e.get("name")===n.get("name")},(0,u.OrderedMap)())}),P=(t.parameterInclusionSettingFor=function(e,t,n,r){var i=r+"."+n;return e.getIn(["meta","paths"].concat((0,o.default)(t),["parameter_inclusions",i]),!1)},t.parameterWithMeta=function(e,t,n,r){var i=d(e).getIn(["paths"].concat((0,o.default)(t),["parameters"]),(0,u.OrderedMap)()).find(function(e){return e.get("in")===r&&e.get("name")===n},(0,u.OrderedMap)());return O(e,t,i)},t.operationWithMeta=function(e,t,n){var r=d(e).getIn(["paths",t,n],(0,u.OrderedMap)()),o=e.getIn(["meta","paths",t,n],(0,u.OrderedMap)()),i=r.get("parameters",(0,u.List)()).map(function(r){return O(e,[t,n],r)});return(0,u.OrderedMap)().merge(r,o).set("parameters",i)});t.hasHost=(0,i.createSelector)(h,function(e){var t=e.get("host");return"string"==typeof t&&t.length>0&&"/"!==t[0]});function T(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"";if(u.List.isList(e))return e.some(function(e){return u.Map.isMap(e)&&e.get("type")===t})}function M(e,t){t=t||[];var n=d(e).getIn(["paths"].concat((0,o.default)(t)),null);if(null!==n){var r=e.getIn(["meta","paths"].concat((0,o.default)(t),["produces_value"]),null),i=n.getIn(["produces",0],null);return r||i||"application/json"}}var I=t.operationScheme=function(e,t,n){var r=e.get("url").match(/^([a-z][a-z0-9+\-.]*):/),o=Array.isArray(r)?r[1]:null;return e.getIn(["scheme",t,n])||e.getIn(["scheme","_defaultScheme"])||o||""};t.canExecuteScheme=function(e,t,n){return["http","https"].indexOf(I(e,t,n))>-1},t.validateBeforeExecute=function(e,t){t=t||[];var n=!0;return e.getIn(["meta","paths"].concat((0,o.default)(t),["parameters"]),(0,u.fromJS)([])).forEach(function(e){var t=e.get("errors");t&&t.count()&&(n=!1)}),n};function j(e){return u.Map.isMap(e)?e:new u.Map}},function(e,t,n){var r=n(49),o=n(329),i=n(330),a=n(36),u=n(115),s=n(165),l={},c={};(t=e.exports=function(e,t,n,f,p){var d,h,v,m,g=p?function(){return e}:s(e),y=r(n,f,t?2:1),b=0;if("function"!=typeof g)throw TypeError(e+" is not iterable!");if(i(g)){for(d=u(e.length);d>b;b++)if((m=t?y(a(h=e[b])[0],h[1]):y(e[b]))===l||m===c)return m}else for(v=g.call(e);!(h=v.next()).done;)if((m=o(v,y,h.value,t))===l||m===c)return m}).BREAK=l,t.RETURN=c},function(e,t,n){"use strict";var r=n(86);e.exports=r.DEFAULT=new r({include:[n(108)],explicit:[n(758),n(759),n(760)]})},function(e,t,n){var r=n(344),o=n(105),i=Object.prototype.hasOwnProperty;e.exports=function(e,t,n){var a=e[t];i.call(e,t)&&o(a,n)&&(void 0!==n||t in e)||r(e,t,n)}},function(e,t,n){"use strict";var r=n(11),o=(n(8),{}),i={reinitializeTransaction:function(){this.transactionWrappers=this.getTransactionWrappers(),this.wrapperInitData?this.wrapperInitData.length=0:this.wrapperInitData=[],this._isInTransaction=!1},_isInTransaction:!1,getTransactionWrappers:null,isInTransaction:function(){return!!this._isInTransaction},perform:function(e,t,n,o,i,a,u,s){var l,c;this.isInTransaction()&&r("27");try{this._isInTransaction=!0,l=!0,this.initializeAll(0),c=e.call(t,n,o,i,a,u,s),l=!1}finally{try{if(l)try{this.closeAll(0)}catch(e){}else this.closeAll(0)}finally{this._isInTransaction=!1}}return c},initializeAll:function(e){for(var t=this.transactionWrappers,n=e;n]/,s=n(219)(function(e,t){if(e.namespaceURI!==i.svg||"innerHTML"in e)e.innerHTML=t;else{(r=r||document.createElement("div")).innerHTML=""+t+"";for(var n=r.firstChild;n.firstChild;)e.appendChild(n.firstChild)}});if(o.canUseDOM){var l=document.createElement("div");l.innerHTML=" ",""===l.innerHTML&&(s=function(e,t){if(e.parentNode&&e.parentNode.replaceChild(e,e),a.test(t)||"<"===t[0]&&u.test(t)){e.innerHTML=String.fromCharCode(65279)+t;var n=e.firstChild;1===n.data.length?e.removeChild(n):n.deleteData(0,1)}else e.innerHTML=t}),l=null}e.exports=s},function(e,t,n){"use strict";var r=/["'&<>]/;e.exports=function(e){return"boolean"==typeof e||"number"==typeof e?""+e:function(e){var t,n=""+e,o=r.exec(n);if(!o)return n;var i="",a=0,u=0;for(a=o.index;adocument.F=Object<\/script>"),e.close(),s=e.F;r--;)delete s.prototype[i[r]];return s()};e.exports=Object.create||function(e,t){var n;return null!==e?(u.prototype=r(e),n=new u,u.prototype=null,n[a]=e):n=s(),void 0===t?n:o(n,t)}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t,n){var r=n(163)("keys"),o=n(116);e.exports=function(e){return r[e]||(r[e]=o(e))}},function(e,t,n){var r=n(21),o=r["__core-js_shared__"]||(r["__core-js_shared__"]={});e.exports=function(e){return o[e]||(o[e]={})}},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t,n){var r=n(166),o=n(19)("iterator"),i=n(70);e.exports=n(15).getIteratorMethod=function(e){if(void 0!=e)return e[o]||e["@@iterator"]||i[r(e)]}},function(e,t,n){var r=n(93),o=n(19)("toStringTag"),i="Arguments"==r(function(){return arguments}());e.exports=function(e){var t,n,a;return void 0===e?"Undefined":null===e?"Null":"string"==typeof(n=function(e,t){try{return e[t]}catch(e){}}(t=Object(e),o))?n:i?r(t):"Object"==(a=r(t))&&"function"==typeof t.callee?"Arguments":a}},function(e,t,n){var r=n(99),o=n(18)("toStringTag"),i="Arguments"==r(function(){return arguments}());e.exports=function(e){var t,n,a;return void 0===e?"Undefined":null===e?"Null":"string"==typeof(n=function(e,t){try{return e[t]}catch(e){}}(t=Object(e),o))?n:i?r(t):"Object"==(a=r(t))&&"function"==typeof t.callee?"Arguments":a}},function(e,t){var n=0,r=Math.random();e.exports=function(e){return"Symbol(".concat(void 0===e?"":e,")_",(++n+r).toString(36))}},function(e,t,n){var r=n(74),o=n(33).document,i=r(o)&&r(o.createElement);e.exports=function(e){return i?o.createElement(e):{}}},function(e,t,n){var r=n(243)("keys"),o=n(168);e.exports=function(e){return r[e]||(r[e]=o(e))}},function(e,t,n){var r=n(117).f,o=n(118),i=n(18)("toStringTag");e.exports=function(e,t,n){e&&!o(e=n?e:e.prototype,i)&&r(e,i,{configurable:!0,value:t})}},function(e,t,n){"use strict";var r=n(121);e.exports.f=function(e){return new function(e){var t,n;this.promise=new e(function(e,r){if(void 0!==t||void 0!==n)throw TypeError("Bad Promise constructor");t=e,n=r}),this.resolve=r(t),this.reject=r(n)}(e)}},function(e,t,n){var r=n(257),o=n(53);e.exports=function(e,t,n){if(r(t))throw TypeError("String#"+n+" doesn't accept regex!");return String(o(e))}},function(e,t,n){var r=n(18)("match");e.exports=function(e){var t=/./;try{"/./"[e](t)}catch(n){try{return t[r]=!1,!"/./"[e](t)}catch(e){}}return!0}},function(e,t,n){t.f=n(19)},function(e,t,n){var r=n(21),o=n(15),i=n(114),a=n(175),u=n(40).f;e.exports=function(e){var t=o.Symbol||(o.Symbol=i?{}:r.Symbol||{});"_"==e.charAt(0)||e in t||u(t,e,{value:a.f(e)})}},function(e,t){t.f=Object.getOwnPropertySymbols},function(e,t){},function(e,t,n){"use strict";(function(t){ +/*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +function n(e){return e instanceof t||e instanceof Date||e instanceof RegExp}function r(e){if(e instanceof t){var n=t.alloc?t.alloc(e.length):new t(e.length);return e.copy(n),n}if(e instanceof Date)return new Date(e.getTime());if(e instanceof RegExp)return new RegExp(e);throw new Error("Unexpected situation")}function o(e,t){return"__proto__"===t?void 0:e[t]}var i=e.exports=function(){if(arguments.length<1||"object"!=typeof arguments[0])return!1;if(arguments.length<2)return arguments[0];var e,t,a=arguments[0];return Array.prototype.slice.call(arguments,1).forEach(function(u){"object"!=typeof u||null===u||Array.isArray(u)||Object.keys(u).forEach(function(s){return t=o(a,s),(e=o(u,s))===a?void 0:"object"!=typeof e||null===e?void(a[s]=e):Array.isArray(e)?void(a[s]=function e(t){var o=[];return t.forEach(function(t,a){"object"==typeof t&&null!==t?Array.isArray(t)?o[a]=e(t):n(t)?o[a]=r(t):o[a]=i({},t):o[a]=t}),o}(e)):n(e)?void(a[s]=r(e)):"object"!=typeof t||null===t||Array.isArray(t)?void(a[s]=i({},e)):void(a[s]=i(t,e))})}),a}}).call(t,n(54).Buffer)},function(e,t,n){"use strict";e.exports=function(e){return"object"==typeof e?function e(t,n){var r;r=Array.isArray(t)?[]:{};n.push(t);Object.keys(t).forEach(function(o){var i=t[o];"function"!=typeof i&&(i&&"object"==typeof i?-1!==n.indexOf(t[o])?r[o]="[Circular]":r[o]=e(t[o],n.slice(0)):r[o]=i)});"string"==typeof t.name&&(r.name=t.name);"string"==typeof t.message&&(r.message=t.message);"string"==typeof t.stack&&(r.stack=t.stack);return r}(e,[]):"function"==typeof e?"[Function: "+(e.name||"anonymous")+"]":e}},function(e,t,n){"use strict";var r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e};function o(e){return null===e?"null":void 0===e?"undefined":"object"===(void 0===e?"undefined":r(e))?Array.isArray(e)?"array":"object":void 0===e?"undefined":r(e)}function i(e){return"object"===o(e)?u(e):"array"===o(e)?a(e):e}function a(e){return e.map(i)}function u(e){var t={};for(var n in e)e.hasOwnProperty(n)&&(t[n]=i(e[n]));return t}function s(e){for(var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],n={arrayBehaviour:(arguments.length>2&&void 0!==arguments[2]?arguments[2]:{}).arrayBehaviour||"replace"},r=t.map(function(e){return e||{}}),i=e||{},l=0;l1?t-1:0),r=1;r-1&&e%1==0&&e<=n}},function(e,t){e.exports=function(e){return function(t){return e(t)}}},function(e,t,n){(function(e){var r=n(279),o="object"==typeof t&&t&&!t.nodeType&&t,i=o&&"object"==typeof e&&e&&!e.nodeType&&e,a=i&&i.exports===o&&r.process,u=function(){try{var e=i&&i.require&&i.require("util").types;return e||a&&a.binding&&a.binding("util")}catch(e){}}();e.exports=u}).call(t,n(134)(e))},function(e,t,n){var r=n(24),o=n(128),i=/\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/,a=/^\w*$/;e.exports=function(e,t){if(r(e))return!1;var n=typeof e;return!("number"!=n&&"symbol"!=n&&"boolean"!=n&&null!=e&&!o(e))||a.test(e)||!i.test(e)||null!=t&&e in Object(t)}},function(e,t){e.exports=function(e){return e}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.memoizedSampleFromSchema=t.memoizedCreateXMLExample=t.sampleXmlFromSchema=t.inferSchema=t.sampleFromSchema=void 0,t.createXMLExample=p;var r=n(9),o=u(n(657)),i=u(n(670)),a=u(n(181));function u(e){return e&&e.__esModule?e:{default:e}}var s={string:function(){return"string"},string_email:function(){return"user@example.com"},"string_date-time":function(){return(new Date).toISOString()},string_date:function(){return(new Date).toISOString().substring(0,10)},string_uuid:function(){return"3fa85f64-5717-4562-b3fc-2c963f66afa6"},string_hostname:function(){return"example.com"},string_ipv4:function(){return"198.51.100.42"},string_ipv6:function(){return"2001:0db8:5b96:0000:0000:426f:8e17:642a"},number:function(){return 0},number_float:function(){return 0},integer:function(){return 0},boolean:function(e){return"boolean"!=typeof e.default||e.default}},l=function(e){var t=e=(0,r.objectify)(e),n=t.type,o=t.format,i=s[n+"_"+o]||s[n];return(0,r.isFunc)(i)?i(e):"Unknown Type: "+e.type},c=t.sampleFromSchema=function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},o=(0,r.objectify)(t),i=o.type,a=o.example,u=o.properties,s=o.additionalProperties,c=o.items,f=n.includeReadOnly,p=n.includeWriteOnly;if(void 0!==a)return(0,r.deeplyStripKey)(a,"$$ref",function(e){return"string"==typeof e&&e.indexOf("#")>-1});if(!i)if(u)i="object";else{if(!c)return;i="array"}if("object"===i){var d=(0,r.objectify)(u),h={};for(var v in d)d[v]&&d[v].deprecated||d[v]&&d[v].readOnly&&!f||d[v]&&d[v].writeOnly&&!p||(h[v]=e(d[v],n));if(!0===s)h.additionalProp1={};else if(s)for(var m=(0,r.objectify)(s),g=e(m,n),y=1;y<4;y++)h["additionalProp"+y]=g;return h}return"array"===i?Array.isArray(c.anyOf)?c.anyOf.map(function(t){return e(t,n)}):Array.isArray(c.oneOf)?c.oneOf.map(function(t){return e(t,n)}):[e(c,n)]:t.enum?t.default?t.default:(0,r.normalizeArray)(t.enum)[0]:"file"!==i?l(t):void 0},f=(t.inferSchema=function(e){return e.schema&&(e=e.schema),e.properties&&(e.type="object"),e},t.sampleXmlFromSchema=function e(t){var n,o=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},i=(0,a.default)({},(0,r.objectify)(t)),u=i.type,s=i.properties,c=i.additionalProperties,f=i.items,p=i.example,d=o.includeReadOnly,h=o.includeWriteOnly,v=i.default,m={},g={},y=t.xml,b=y.name,_=y.prefix,w=y.namespace,E=i.enum,x=void 0;if(!u)if(s||c)u="object";else{if(!f)return;u="array"}(b=b||"notagname",n=(_?_+":":"")+b,w)&&(g[_?"xmlns:"+_:"xmlns"]=w);if("array"===u&&f){if(f.xml=f.xml||y||{},f.xml.name=f.xml.name||y.name,y.wrapped)return m[n]=[],Array.isArray(p)?p.forEach(function(t){f.example=t,m[n].push(e(f,o))}):Array.isArray(v)?v.forEach(function(t){f.default=t,m[n].push(e(f,o))}):m[n]=[e(f,o)],g&&m[n].push({_attr:g}),m;var S=[];return Array.isArray(p)?(p.forEach(function(t){f.example=t,S.push(e(f,o))}),S):Array.isArray(v)?(v.forEach(function(t){f.default=t,S.push(e(f,o))}),S):e(f,o)}if("object"===u){var C=(0,r.objectify)(s);for(var k in m[n]=[],p=p||{},C)if(C.hasOwnProperty(k)&&(!C[k].readOnly||d)&&(!C[k].writeOnly||h))if(C[k].xml=C[k].xml||{},C[k].xml.attribute){var A=Array.isArray(C[k].enum)&&C[k].enum[0],O=C[k].example,P=C[k].default;g[C[k].xml.name||k]=void 0!==O&&O||void 0!==p[k]&&p[k]||void 0!==P&&P||A||l(C[k])}else{C[k].xml.name=C[k].xml.name||k,void 0===C[k].example&&void 0!==p[k]&&(C[k].example=p[k]);var T=e(C[k]);Array.isArray(T)?m[n]=m[n].concat(T):m[n].push(T)}return!0===c?m[n].push({additionalProp:"Anything can be here"}):c&&m[n].push({additionalProp:l(c)}),g&&m[n].push({_attr:g}),m}return x=void 0!==p?p:void 0!==v?v:Array.isArray(E)?E[0]:l(t),m[n]=g?[{_attr:g},x]:x,m});function p(e,t){var n=f(e,t);if(n)return(0,o.default)(n,{declaration:!0,indent:"\t"})}t.memoizedCreateXMLExample=(0,i.default)(p),t.memoizedSampleFromSchema=(0,i.default)(c)},function(e,t){function n(){this._events=this._events||{},this._maxListeners=this._maxListeners||void 0}function r(e){return"function"==typeof e}function o(e){return"object"==typeof e&&null!==e}function i(e){return void 0===e}e.exports=n,n.EventEmitter=n,n.prototype._events=void 0,n.prototype._maxListeners=void 0,n.defaultMaxListeners=10,n.prototype.setMaxListeners=function(e){if("number"!=typeof e||e<0||isNaN(e))throw TypeError("n must be a positive number");return this._maxListeners=e,this},n.prototype.emit=function(e){var t,n,a,u,s,l;if(this._events||(this._events={}),"error"===e&&(!this._events.error||o(this._events.error)&&!this._events.error.length)){if((t=arguments[1])instanceof Error)throw t;var c=new Error('Uncaught, unspecified "error" event. ('+t+")");throw c.context=t,c}if(i(n=this._events[e]))return!1;if(r(n))switch(arguments.length){case 1:n.call(this);break;case 2:n.call(this,arguments[1]);break;case 3:n.call(this,arguments[1],arguments[2]);break;default:u=Array.prototype.slice.call(arguments,1),n.apply(this,u)}else if(o(n))for(u=Array.prototype.slice.call(arguments,1),a=(l=n.slice()).length,s=0;s0&&this._events[e].length>a&&(this._events[e].warned=!0,console.error("(node) warning: possible EventEmitter memory leak detected. %d listeners added. Use emitter.setMaxListeners() to increase limit.",this._events[e].length),"function"==typeof console.trace&&console.trace()),this},n.prototype.on=n.prototype.addListener,n.prototype.once=function(e,t){if(!r(t))throw TypeError("listener must be a function");var n=!1;function o(){this.removeListener(e,o),n||(n=!0,t.apply(this,arguments))}return o.listener=t,this.on(e,o),this},n.prototype.removeListener=function(e,t){var n,i,a,u;if(!r(t))throw TypeError("listener must be a function");if(!this._events||!this._events[e])return this;if(a=(n=this._events[e]).length,i=-1,n===t||r(n.listener)&&n.listener===t)delete this._events[e],this._events.removeListener&&this.emit("removeListener",e,t);else if(o(n)){for(u=a;u-- >0;)if(n[u]===t||n[u].listener&&n[u].listener===t){i=u;break}if(i<0)return this;1===n.length?(n.length=0,delete this._events[e]):n.splice(i,1),this._events.removeListener&&this.emit("removeListener",e,t)}return this},n.prototype.removeAllListeners=function(e){var t,n;if(!this._events)return this;if(!this._events.removeListener)return 0===arguments.length?this._events={}:this._events[e]&&delete this._events[e],this;if(0===arguments.length){for(t in this._events)"removeListener"!==t&&this.removeAllListeners(t);return this.removeAllListeners("removeListener"),this._events={},this}if(r(n=this._events[e]))this.removeListener(e,n);else if(n)for(;n.length;)this.removeListener(e,n[n.length-1]);return delete this._events[e],this},n.prototype.listeners=function(e){return this._events&&this._events[e]?r(this._events[e])?[this._events[e]]:this._events[e].slice():[]},n.prototype.listenerCount=function(e){if(this._events){var t=this._events[e];if(r(t))return 1;if(t)return t.length}return 0},n.listenerCount=function(e,t){return e.listenerCount(t)}},function(e,t,n){(t=e.exports=n(306)).Stream=t,t.Readable=t,t.Writable=n(197),t.Duplex=n(65),t.Transform=n(311),t.PassThrough=n(665)},function(e,t,n){"use strict";(function(t,r,o){var i=n(140);function a(e){var t=this;this.next=null,this.entry=null,this.finish=function(){!function(e,t,n){var r=e.entry;e.entry=null;for(;r;){var o=r.callback;t.pendingcb--,o(n),r=r.next}t.corkedRequestsFree?t.corkedRequestsFree.next=e:t.corkedRequestsFree=e}(t,e)}}e.exports=y;var u,s=!t.browser&&["v0.10","v0.9."].indexOf(t.version.slice(0,5))>-1?r:i.nextTick;y.WritableState=g;var l=n(106);l.inherits=n(81);var c={deprecate:n(664)},f=n(307),p=n(141).Buffer,d=o.Uint8Array||function(){};var h,v=n(308);function m(){}function g(e,t){u=u||n(65),e=e||{};var r=t instanceof u;this.objectMode=!!e.objectMode,r&&(this.objectMode=this.objectMode||!!e.writableObjectMode);var o=e.highWaterMark,l=e.writableHighWaterMark,c=this.objectMode?16:16384;this.highWaterMark=o||0===o?o:r&&(l||0===l)?l:c,this.highWaterMark=Math.floor(this.highWaterMark),this.finalCalled=!1,this.needDrain=!1,this.ending=!1,this.ended=!1,this.finished=!1,this.destroyed=!1;var f=!1===e.decodeStrings;this.decodeStrings=!f,this.defaultEncoding=e.defaultEncoding||"utf8",this.length=0,this.writing=!1,this.corked=0,this.sync=!0,this.bufferProcessing=!1,this.onwrite=function(e){!function(e,t){var n=e._writableState,r=n.sync,o=n.writecb;if(function(e){e.writing=!1,e.writecb=null,e.length-=e.writelen,e.writelen=0}(n),t)!function(e,t,n,r,o){--t.pendingcb,n?(i.nextTick(o,r),i.nextTick(S,e,t),e._writableState.errorEmitted=!0,e.emit("error",r)):(o(r),e._writableState.errorEmitted=!0,e.emit("error",r),S(e,t))}(e,n,r,t,o);else{var a=E(n);a||n.corked||n.bufferProcessing||!n.bufferedRequest||w(e,n),r?s(_,e,n,a,o):_(e,n,a,o)}}(t,e)},this.writecb=null,this.writelen=0,this.bufferedRequest=null,this.lastBufferedRequest=null,this.pendingcb=0,this.prefinished=!1,this.errorEmitted=!1,this.bufferedRequestCount=0,this.corkedRequestsFree=new a(this)}function y(e){if(u=u||n(65),!(h.call(y,this)||this instanceof u))return new y(e);this._writableState=new g(e,this),this.writable=!0,e&&("function"==typeof e.write&&(this._write=e.write),"function"==typeof e.writev&&(this._writev=e.writev),"function"==typeof e.destroy&&(this._destroy=e.destroy),"function"==typeof e.final&&(this._final=e.final)),f.call(this)}function b(e,t,n,r,o,i,a){t.writelen=r,t.writecb=a,t.writing=!0,t.sync=!0,n?e._writev(o,t.onwrite):e._write(o,i,t.onwrite),t.sync=!1}function _(e,t,n,r){n||function(e,t){0===t.length&&t.needDrain&&(t.needDrain=!1,e.emit("drain"))}(e,t),t.pendingcb--,r(),S(e,t)}function w(e,t){t.bufferProcessing=!0;var n=t.bufferedRequest;if(e._writev&&n&&n.next){var r=t.bufferedRequestCount,o=new Array(r),i=t.corkedRequestsFree;i.entry=n;for(var u=0,s=!0;n;)o[u]=n,n.isBuf||(s=!1),n=n.next,u+=1;o.allBuffers=s,b(e,t,!0,t.length,o,"",i.finish),t.pendingcb++,t.lastBufferedRequest=null,i.next?(t.corkedRequestsFree=i.next,i.next=null):t.corkedRequestsFree=new a(t),t.bufferedRequestCount=0}else{for(;n;){var l=n.chunk,c=n.encoding,f=n.callback;if(b(e,t,!1,t.objectMode?1:l.length,l,c,f),n=n.next,t.bufferedRequestCount--,t.writing)break}null===n&&(t.lastBufferedRequest=null)}t.bufferedRequest=n,t.bufferProcessing=!1}function E(e){return e.ending&&0===e.length&&null===e.bufferedRequest&&!e.finished&&!e.writing}function x(e,t){e._final(function(n){t.pendingcb--,n&&e.emit("error",n),t.prefinished=!0,e.emit("prefinish"),S(e,t)})}function S(e,t){var n=E(t);return n&&(!function(e,t){t.prefinished||t.finalCalled||("function"==typeof e._final?(t.pendingcb++,t.finalCalled=!0,i.nextTick(x,e,t)):(t.prefinished=!0,e.emit("prefinish")))}(e,t),0===t.pendingcb&&(t.finished=!0,e.emit("finish"))),n}l.inherits(y,f),g.prototype.getBuffer=function(){for(var e=this.bufferedRequest,t=[];e;)t.push(e),e=e.next;return t},function(){try{Object.defineProperty(g.prototype,"buffer",{get:c.deprecate(function(){return this.getBuffer()},"_writableState.buffer is deprecated. Use _writableState.getBuffer instead.","DEP0003")})}catch(e){}}(),"function"==typeof Symbol&&Symbol.hasInstance&&"function"==typeof Function.prototype[Symbol.hasInstance]?(h=Function.prototype[Symbol.hasInstance],Object.defineProperty(y,Symbol.hasInstance,{value:function(e){return!!h.call(this,e)||this===y&&(e&&e._writableState instanceof g)}})):h=function(e){return e instanceof this},y.prototype.pipe=function(){this.emit("error",new Error("Cannot pipe, not readable"))},y.prototype.write=function(e,t,n){var r,o=this._writableState,a=!1,u=!o.objectMode&&(r=e,p.isBuffer(r)||r instanceof d);return u&&!p.isBuffer(e)&&(e=function(e){return p.from(e)}(e)),"function"==typeof t&&(n=t,t=null),u?t="buffer":t||(t=o.defaultEncoding),"function"!=typeof n&&(n=m),o.ended?function(e,t){var n=new Error("write after end");e.emit("error",n),i.nextTick(t,n)}(this,n):(u||function(e,t,n,r){var o=!0,a=!1;return null===n?a=new TypeError("May not write null values to stream"):"string"==typeof n||void 0===n||t.objectMode||(a=new TypeError("Invalid non-string/buffer chunk")),a&&(e.emit("error",a),i.nextTick(r,a),o=!1),o}(this,o,e,n))&&(o.pendingcb++,a=function(e,t,n,r,o,i){if(!n){var a=function(e,t,n){e.objectMode||!1===e.decodeStrings||"string"!=typeof t||(t=p.from(t,n));return t}(t,r,o);r!==a&&(n=!0,o="buffer",r=a)}var u=t.objectMode?1:r.length;t.length+=u;var s=t.length-1))throw new TypeError("Unknown encoding: "+e);return this._writableState.defaultEncoding=e,this},Object.defineProperty(y.prototype,"writableHighWaterMark",{enumerable:!1,get:function(){return this._writableState.highWaterMark}}),y.prototype._write=function(e,t,n){n(new Error("_write() is not implemented"))},y.prototype._writev=null,y.prototype.end=function(e,t,n){var r=this._writableState;"function"==typeof e?(n=e,e=null,t=null):"function"==typeof t&&(n=t,t=null),null!==e&&void 0!==e&&this.write(e,t),r.corked&&(r.corked=1,this.uncork()),r.ending||r.finished||function(e,t,n){t.ending=!0,S(e,t),n&&(t.finished?i.nextTick(n):e.once("finish",n));t.ended=!0,e.writable=!1}(this,r,n)},Object.defineProperty(y.prototype,"destroyed",{get:function(){return void 0!==this._writableState&&this._writableState.destroyed},set:function(e){this._writableState&&(this._writableState.destroyed=e)}}),y.prototype.destroy=v.destroy,y.prototype._undestroy=v.undestroy,y.prototype._destroy=function(e,t){this.end(),t(e)}}).call(t,n(56),n(309).setImmediate,n(31))},function(e,t,n){"use strict";e.exports=function(e){return"function"==typeof e}},function(e,t,n){"use strict";e.exports=n(691)()?Array.from:n(692)},function(e,t,n){"use strict";var r=n(705),o=n(67),i=n(82),a=Array.prototype.indexOf,u=Object.prototype.hasOwnProperty,s=Math.abs,l=Math.floor;e.exports=function(e){var t,n,c,f;if(!r(e))return a.apply(this,arguments);for(n=o(i(this).length),c=arguments[1],t=c=isNaN(c)?0:c>=0?l(c):o(this.length)-l(s(c));t1&&void 0!==arguments[1])||arguments[1];return e=(0,r.normalizeArray)(e),{type:u,payload:{thing:e,shown:t}}},t.changeMode=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"";return e=(0,r.normalizeArray)(e),{type:a,payload:{thing:e,mode:t}}};var r=n(9),o=t.UPDATE_LAYOUT="layout_update_layout",i=t.UPDATE_FILTER="layout_update_filter",a=t.UPDATE_MODE="layout_update_mode",u=t.SHOW="layout_show"},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.execute=t.executeRequest=t.logRequest=t.setMutatedRequest=t.setRequest=t.setResponse=t.updateEmptyParamInclusion=t.validateParams=t.invalidateResolvedSubtreeCache=t.updateResolvedSubtree=t.requestResolvedSubtree=t.resolveSpec=t.parseToJson=t.SET_SCHEME=t.UPDATE_RESOLVED_SUBTREE=t.UPDATE_RESOLVED=t.UPDATE_OPERATION_META_VALUE=t.CLEAR_VALIDATE_PARAMS=t.CLEAR_REQUEST=t.CLEAR_RESPONSE=t.LOG_REQUEST=t.SET_MUTATED_REQUEST=t.SET_REQUEST=t.SET_RESPONSE=t.VALIDATE_PARAMS=t.UPDATE_EMPTY_PARAM_INCLUSION=t.UPDATE_PARAM=t.UPDATE_JSON=t.UPDATE_URL=t.UPDATE_SPEC=void 0;var r=b(n(25)),o=b(n(84)),i=b(n(23)),a=b(n(42)),u=b(n(204)),s=b(n(338)),l=b(n(339)),c=b(n(45));t.updateSpec=function(e){var t=L(e).replace(/\t/g," ");if("string"==typeof e)return{type:_,payload:t}},t.updateResolved=function(e){return{type:N,payload:e}},t.updateUrl=function(e){return{type:w,payload:e}},t.updateJsonSpec=function(e){return{type:E,payload:e}},t.changeParam=function(e,t,n,r,o){return{type:x,payload:{path:e,value:r,paramName:t,paramIn:n,isXml:o}}},t.changeParamByIdentity=function(e,t,n,r){return{type:x,payload:{path:e,param:t,value:n,isXml:r}}},t.clearValidateParams=function(e){return{type:I,payload:{pathMethod:e}}},t.changeConsumesValue=function(e,t){return{type:j,payload:{path:e,value:t,key:"consumes_value"}}},t.changeProducesValue=function(e,t){return{type:j,payload:{path:e,value:t,key:"produces_value"}}},t.clearResponse=function(e,t){return{type:T,payload:{path:e,method:t}}},t.clearRequest=function(e,t){return{type:M,payload:{path:e,method:t}}},t.setScheme=function(e,t,n){return{type:D,payload:{scheme:e,path:t,method:n}}};var f=b(n(208)),p=n(7),d=b(n(210)),h=b(n(180)),v=b(n(342)),m=b(n(764)),g=b(n(766)),y=n(9);function b(e){return e&&e.__esModule?e:{default:e}}var _=t.UPDATE_SPEC="spec_update_spec",w=t.UPDATE_URL="spec_update_url",E=t.UPDATE_JSON="spec_update_json",x=t.UPDATE_PARAM="spec_update_param",S=t.UPDATE_EMPTY_PARAM_INCLUSION="spec_update_empty_param_inclusion",C=t.VALIDATE_PARAMS="spec_validate_param",k=t.SET_RESPONSE="spec_set_response",A=t.SET_REQUEST="spec_set_request",O=t.SET_MUTATED_REQUEST="spec_set_mutated_request",P=t.LOG_REQUEST="spec_log_request",T=t.CLEAR_RESPONSE="spec_clear_response",M=t.CLEAR_REQUEST="spec_clear_request",I=t.CLEAR_VALIDATE_PARAMS="spec_clear_validate_param",j=t.UPDATE_OPERATION_META_VALUE="spec_update_operation_meta_value",N=t.UPDATE_RESOLVED="spec_update_resolved",R=t.UPDATE_RESOLVED_SUBTREE="spec_update_resolved_subtree",D=t.SET_SCHEME="set_scheme",L=function(e){return(0,v.default)(e)?e:""};t.parseToJson=function(e){return function(t){var n=t.specActions,r=t.specSelectors,o=t.errActions,i=r.specStr,a=null;try{e=e||i(),o.clear({source:"parser"}),a=f.default.safeLoad(e)}catch(e){return console.error(e),o.newSpecErr({source:"parser",level:"error",message:e.reason,line:e.mark&&e.mark.line?e.mark.line+1:void 0})}return a&&"object"===(void 0===a?"undefined":(0,c.default)(a))?n.updateJsonSpec(a):{}}};var U=!1,q=(t.resolveSpec=function(e,t){return function(n){var r=n.specActions,o=n.specSelectors,i=n.errActions,a=n.fn,u=a.fetch,s=a.resolve,l=a.AST,c=void 0===l?{}:l,f=n.getConfigs;U||(console.warn("specActions.resolveSpec is deprecated since v3.10.0 and will be removed in v4.0.0; use requestResolvedSubtree instead!"),U=!0);var p=f(),d=p.modelPropertyMacro,h=p.parameterMacro,v=p.requestInterceptor,m=p.responseInterceptor;void 0===e&&(e=o.specJson()),void 0===t&&(t=o.url());var g=c.getLineNumberForPath?c.getLineNumberForPath:function(){},y=o.specStr();return s({fetch:u,spec:e,baseDoc:t,modelPropertyMacro:d,parameterMacro:h,requestInterceptor:v,responseInterceptor:m}).then(function(e){var t=e.spec,n=e.errors;if(i.clear({type:"thrown"}),Array.isArray(n)&&n.length>0){var o=n.map(function(e){return console.error(e),e.line=e.fullPath?g(y,e.fullPath):null,e.path=e.fullPath?e.fullPath.join("."):null,e.level="error",e.type="thrown",e.source="resolver",Object.defineProperty(e,"message",{enumerable:!0,value:e.message}),e});i.newThrownErrBatch(o)}return r.updateResolved(t)})}},[]),F=(0,m.default)((0,l.default)(s.default.mark(function e(){var t,n,r,o,i,a,c,f,d,h,v,m,y,b,_,w,E;return s.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:if(t=q.system){e.next=4;break}return console.error("debResolveSubtrees: don't have a system to operate on, aborting."),e.abrupt("return");case 4:if(n=t.errActions,r=t.errSelectors,o=t.fn,i=o.resolveSubtree,a=o.AST,c=void 0===a?{}:a,f=t.specSelectors,d=t.specActions,i){e.next=8;break}return console.error("Error: Swagger-Client did not provide a `resolveSubtree` method, doing nothing."),e.abrupt("return");case 8:return h=c.getLineNumberForPath?c.getLineNumberForPath:function(){},v=f.specStr(),m=t.getConfigs(),y=m.modelPropertyMacro,b=m.parameterMacro,_=m.requestInterceptor,w=m.responseInterceptor,e.prev=11,e.next=14,q.reduce(function(){var e=(0,l.default)(s.default.mark(function e(t,o){var a,u,l,c,p,d,m;return s.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,t;case 2:return a=e.sent,u=a.resultMap,l=a.specWithCurrentSubtrees,e.next=7,i(l,o,{baseDoc:f.url(),modelPropertyMacro:y,parameterMacro:b,requestInterceptor:_,responseInterceptor:w});case 7:return c=e.sent,p=c.errors,d=c.spec,r.allErrors().size&&n.clearBy(function(e){return"thrown"!==e.get("type")||"resolver"!==e.get("source")||!e.get("fullPath").every(function(e,t){return e===o[t]||void 0===o[t]})}),Array.isArray(p)&&p.length>0&&(m=p.map(function(e){return e.line=e.fullPath?h(v,e.fullPath):null,e.path=e.fullPath?e.fullPath.join("."):null,e.level="error",e.type="thrown",e.source="resolver",Object.defineProperty(e,"message",{enumerable:!0,value:e.message}),e}),n.newThrownErrBatch(m)),(0,g.default)(u,o,d),(0,g.default)(l,o,d),e.abrupt("return",{resultMap:u,specWithCurrentSubtrees:l});case 15:case"end":return e.stop()}},e,void 0)}));return function(t,n){return e.apply(this,arguments)}}(),u.default.resolve({resultMap:(f.specResolvedSubtree([])||(0,p.Map)()).toJS(),specWithCurrentSubtrees:f.specJson().toJS()}));case 14:E=e.sent,delete q.system,q=[],e.next=22;break;case 19:e.prev=19,e.t0=e.catch(11),console.error(e.t0);case 22:d.updateResolvedSubtree([],E.resultMap);case 23:case"end":return e.stop()}},e,void 0,[[11,19]])})),35);t.requestResolvedSubtree=function(e){return function(t){q.map(function(e){return e.join("@@")}).indexOf(e.join("@@"))>-1||(q.push(e),q.system=t,F())}};t.updateResolvedSubtree=function(e,t){return{type:R,payload:{path:e,value:t}}},t.invalidateResolvedSubtreeCache=function(){return{type:R,payload:{path:[],value:(0,p.Map)()}}},t.validateParams=function(e,t){return{type:C,payload:{pathMethod:e,isOAS3:t}}},t.updateEmptyParamInclusion=function(e,t,n,r){return{type:S,payload:{pathMethod:e,paramName:t,paramIn:n,includeEmptyValue:r}}};t.setResponse=function(e,t,n){return{payload:{path:e,method:t,res:n},type:k}},t.setRequest=function(e,t,n){return{payload:{path:e,method:t,req:n},type:A}},t.setMutatedRequest=function(e,t,n){return{payload:{path:e,method:t,req:n},type:O}},t.logRequest=function(e){return{payload:e,type:P}},t.executeRequest=function(e){return function(t){var n=t.fn,r=t.specActions,o=t.specSelectors,u=t.getConfigs,s=t.oas3Selectors,l=e.pathName,c=e.method,f=e.operation,p=u(),v=p.requestInterceptor,m=p.responseInterceptor,g=f.toJS();if(f&&f.get("parameters")&&f.get("parameters").filter(function(e){return e&&!0===e.get("allowEmptyValue")}).forEach(function(t){if(o.parameterInclusionSettingFor([l,c],t.get("name"),t.get("in"))){e.parameters=e.parameters||{};var n=(0,y.paramToValue)(t,e.parameters);(!n||n&&0===n.size)&&(e.parameters[t.get("name")]="")}}),e.contextUrl=(0,d.default)(o.url()).toString(),g&&g.operationId?e.operationId=g.operationId:g&&l&&c&&(e.operationId=n.opId(g,l,c)),o.isOAS3()){var b=l+":"+c;e.server=s.selectedServer(b)||s.selectedServer();var _=s.serverVariables({server:e.server,namespace:b}).toJS(),w=s.serverVariables({server:e.server}).toJS();e.serverVariables=(0,a.default)(_).length?_:w,e.requestContentType=s.requestContentType(l,c),e.responseContentType=s.responseContentType(l,c)||"*/*";var E=s.requestBodyValue(l,c);(0,y.isJSONObject)(E)?e.requestBody=JSON.parse(E):E&&E.toJS?e.requestBody=E.toJS():e.requestBody=E}var x=(0,i.default)({},e);x=n.buildRequest(x),r.setRequest(e.pathName,e.method,x);e.requestInterceptor=function(t){var n=v.apply(this,[t]),o=(0,i.default)({},n);return r.setMutatedRequest(e.pathName,e.method,o),n},e.responseInterceptor=m;var S=Date.now();return n.execute(e).then(function(t){t.duration=Date.now()-S,r.setResponse(e.pathName,e.method,t)}).catch(function(t){return r.setResponse(e.pathName,e.method,{error:!0,err:(0,h.default)(t)})})}};t.execute=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e.path,n=e.method,i=(0,o.default)(e,["path","method"]);return function(e){var o=e.fn.fetch,a=e.specSelectors,u=e.specActions,s=a.specJsonWithResolvedSubtrees().toJS(),l=a.operationScheme(t,n),c=a.contentTypeValues([t,n]).toJS(),f=c.requestContentType,p=c.responseContentType,d=/xml/i.test(f),h=a.parameterValues([t,n],d).toJS();return u.executeRequest((0,r.default)({},i,{fetch:o,spec:s,pathName:t,method:n,parameters:h,requestContentType:f,scheme:l,responseContentType:p}))}}},function(e,t,n){e.exports={default:n(733),__esModule:!0}},function(e,t){e.exports=function(e,t,n,r){if(!(e instanceof t)||void 0!==r&&r in e)throw TypeError(n+": incorrect invocation!");return e}},function(e,t,n){"use strict";var r=n(94);e.exports.f=function(e){return new function(e){var t,n;this.promise=new e(function(e,r){if(void 0!==t||void 0!==n)throw TypeError("Bad Promise constructor");t=e,n=r}),this.resolve=r(t),this.reject=r(n)}(e)}},function(e,t,n){var r=n(50);e.exports=function(e,t,n){for(var o in t)n&&e[o]?e[o]=t[o]:r(e,o,t[o]);return e}},function(e,t,n){"use strict";var r=n(742);e.exports=r},function(e,t,n){"use strict";var r=n(86);e.exports=new r({explicit:[n(745),n(746),n(747)]})},function(e,t,n){"use strict";(function(t){var r=n(762),o=n(763),i=/^([a-z][a-z0-9.+-]*:)?(\/\/)?([\S\s]*)/i,a=/^[A-Za-z][A-Za-z0-9+-.]*:\/\//,u=[["#","hash"],["?","query"],function(e){return e.replace("\\","/")},["/","pathname"],["@","auth",1],[NaN,"host",void 0,1,1],[/:(\d+)$/,"port",void 0,1],[NaN,"hostname",void 0,1,1]],s={hash:1,query:1};function l(e){var n,r=t&&t.location||{},o={},i=typeof(e=e||r);if("blob:"===e.protocol)o=new f(unescape(e.pathname),{});else if("string"===i)for(n in o=new f(e,{}),s)delete o[n];else if("object"===i){for(n in e)n in s||(o[n]=e[n]);void 0===o.slashes&&(o.slashes=a.test(e.href))}return o}function c(e){var t=i.exec(e);return{protocol:t[1]?t[1].toLowerCase():"",slashes:!!t[2],rest:t[3]}}function f(e,t,n){if(!(this instanceof f))return new f(e,t,n);var i,a,s,p,d,h,v=u.slice(),m=typeof t,g=this,y=0;for("object"!==m&&"string"!==m&&(n=t,t=null),n&&"function"!=typeof n&&(n=o.parse),t=l(t),i=!(a=c(e||"")).protocol&&!a.slashes,g.slashes=a.slashes||i&&t.slashes,g.protocol=a.protocol||t.protocol||"",e=a.rest,a.slashes||(v[3]=[/(.*)/,"pathname"]);y-1||r("96",e),!l.plugins[n]){t.extractEvents||r("97",e),l.plugins[n]=t;var a=t.eventTypes;for(var s in a)u(a[s],t,s)||r("98",s,e)}}}function u(e,t,n){l.eventNameDispatchConfigs.hasOwnProperty(n)&&r("99",n),l.eventNameDispatchConfigs[n]=e;var o=e.phasedRegistrationNames;if(o){for(var i in o){if(o.hasOwnProperty(i))s(o[i],t,n)}return!0}return!!e.registrationName&&(s(e.registrationName,t,n),!0)}function s(e,t,n){l.registrationNameModules[e]&&r("100",e),l.registrationNameModules[e]=t,l.registrationNameDependencies[e]=t.eventTypes[n].dependencies}var l={plugins:[],eventNameDispatchConfigs:{},registrationNameModules:{},registrationNameDependencies:{},possibleRegistrationNames:null,injectEventPluginOrder:function(e){o&&r("101"),o=Array.prototype.slice.call(e),a()},injectEventPluginsByName:function(e){var t=!1;for(var n in e)if(e.hasOwnProperty(n)){var o=e[n];i.hasOwnProperty(n)&&i[n]===o||(i[n]&&r("102",n),i[n]=o,t=!0)}t&&a()},getPluginModuleForEvent:function(e){var t=e.dispatchConfig;if(t.registrationName)return l.registrationNameModules[t.registrationName]||null;if(void 0!==t.phasedRegistrationNames){var n=t.phasedRegistrationNames;for(var r in n)if(n.hasOwnProperty(r)){var o=l.registrationNameModules[n[r]];if(o)return o}}return null},_resetEventPlugins:function(){for(var e in o=null,i)i.hasOwnProperty(e)&&delete i[e];l.plugins.length=0;var t=l.eventNameDispatchConfigs;for(var n in t)t.hasOwnProperty(n)&&delete t[n];var r=l.registrationNameModules;for(var a in r)r.hasOwnProperty(a)&&delete r[a]}};e.exports=l},function(e,t,n){"use strict";var r,o,i=n(11),a=n(213);n(8),n(10);function u(e,t,n,r){var o=e.type||"unknown-event";e.currentTarget=s.getNodeFromInstance(r),t?a.invokeGuardedCallbackWithCatch(o,n,e):a.invokeGuardedCallback(o,n,e),e.currentTarget=null}var s={isEndish:function(e){return"topMouseUp"===e||"topTouchEnd"===e||"topTouchCancel"===e},isMoveish:function(e){return"topMouseMove"===e||"topTouchMove"===e},isStartish:function(e){return"topMouseDown"===e||"topTouchStart"===e},executeDirectDispatch:function(e){var t=e._dispatchListeners,n=e._dispatchInstances;Array.isArray(t)&&i("103"),e.currentTarget=t?s.getNodeFromInstance(n):null;var r=t?t(e):null;return e.currentTarget=null,e._dispatchListeners=null,e._dispatchInstances=null,r},executeDispatchesInOrder:function(e,t){var n=e._dispatchListeners,r=e._dispatchInstances;if(Array.isArray(n))for(var o=0;o0&&r.length<20?n+" (keys: "+r.join(", ")+")":n}function s(e,t){var n=o.get(e);return n||null}var l={isMounted:function(e){var t=o.get(e);return!!t&&!!t._renderedComponent},enqueueCallback:function(e,t,n){l.validateCallback(t,n);var r=s(e);if(!r)return null;r._pendingCallbacks?r._pendingCallbacks.push(t):r._pendingCallbacks=[t],a(r)},enqueueCallbackInternal:function(e,t){e._pendingCallbacks?e._pendingCallbacks.push(t):e._pendingCallbacks=[t],a(e)},enqueueForceUpdate:function(e){var t=s(e);t&&(t._pendingForceUpdate=!0,a(t))},enqueueReplaceState:function(e,t,n){var r=s(e);r&&(r._pendingStateQueue=[t],r._pendingReplaceState=!0,void 0!==n&&null!==n&&(l.validateCallback(n,"replaceState"),r._pendingCallbacks?r._pendingCallbacks.push(n):r._pendingCallbacks=[n]),a(r))},enqueueSetState:function(e,t){var n=s(e);n&&((n._pendingStateQueue||(n._pendingStateQueue=[])).push(t),a(n))},enqueueElementInternal:function(e,t,n){e._pendingElement=t,e._context=n,a(e)},validateCallback:function(e,t){e&&"function"!=typeof e&&r("122",t,u(e))}};e.exports=l},function(e,t,n){"use strict";n(13);var r=n(34),o=(n(10),r);e.exports=o},function(e,t,n){"use strict";e.exports=function(e){var t,n=e.keyCode;return"charCode"in e?0===(t=e.charCode)&&13===n&&(t=13):t=n,t>=32||13===t?t:0}},function(e,t,n){var r=n(62),o=n(229),i=n(47),a="[object Object]",u=Function.prototype,s=Object.prototype,l=u.toString,c=s.hasOwnProperty,f=l.call(Object);e.exports=function(e){if(!i(e)||r(e)!=a)return!1;var t=o(e);if(null===t)return!0;var n=c.call(t,"constructor")&&t.constructor;return"function"==typeof n&&n instanceof n&&l.call(n)==f}},function(e,t,n){var r=n(298)(Object.getPrototypeOf,Object);e.exports=r},function(e,t,n){var r=n(292);e.exports=function(e){var t=new e.constructor(e.byteLength);return new r(t).set(new r(e)),t}},function(e,t){var n=this&&this.__extends||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n]);function r(){this.constructor=e}e.prototype=null===t?Object.create(t):(r.prototype=t.prototype,new r)},r=Object.prototype.hasOwnProperty; +/*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017 Joachim Wester + * MIT license + */function o(e,t){return r.call(e,t)}function i(e){if(Array.isArray(e)){for(var t=new Array(e.length),n=0;n=48&&t<=57))return!1;n++}return!0},t.escapePathComponent=a,t.unescapePathComponent=function(e){return e.replace(/~1/g,"/").replace(/~0/g,"~")},t._getPathRecursive=u,t.getPath=function(e,t){if(e===t)return"/";var n=u(e,t);if(""===n)throw new Error("Object not found in root");return"/"+n},t.hasUndefined=function e(t){if(void 0===t)return!0;if(t)if(Array.isArray(t)){for(var n=0,r=t.length;nw;w++)if((p||w in y)&&(m=b(v=y[w],w,g),e))if(n)E[w]=m;else if(m)switch(e){case 3:return!0;case 5:return v;case 6:return w;case 2:E.push(v)}else if(c)return!1;return f?-1:l||c?c:E}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.authorizeRequest=t.authorizeAccessCodeWithBasicAuthentication=t.authorizeAccessCodeWithFormParams=t.authorizeApplication=t.authorizePassword=t.preAuthorizeImplicit=t.CONFIGURE_AUTH=t.VALIDATE=t.AUTHORIZE_OAUTH2=t.PRE_AUTHORIZE_OAUTH2=t.LOGOUT=t.AUTHORIZE=t.SHOW_AUTH_POPUP=void 0;var r=l(n(45)),o=l(n(23)),i=l(n(41));t.showDefinitions=function(e){return{type:c,payload:e}},t.authorize=function(e){return{type:f,payload:e}},t.logout=function(e){return{type:p,payload:e}},t.authorizeOauth2=function(e){return{type:d,payload:e}},t.configureAuth=function(e){return{type:h,payload:e}};var a=l(n(210)),u=l(n(32)),s=n(9);function l(e){return e&&e.__esModule?e:{default:e}}var c=t.SHOW_AUTH_POPUP="show_popup",f=t.AUTHORIZE="authorize",p=t.LOGOUT="logout",d=(t.PRE_AUTHORIZE_OAUTH2="pre_authorize_oauth2",t.AUTHORIZE_OAUTH2="authorize_oauth2"),h=(t.VALIDATE="validate",t.CONFIGURE_AUTH="configure_auth");t.preAuthorizeImplicit=function(e){return function(t){var n=t.authActions,r=t.errActions,o=e.auth,a=e.token,s=e.isValid,l=o.schema,c=o.name,f=l.get("flow");delete u.default.swaggerUIRedirectOauth2,"accessCode"===f||s||r.newAuthErr({authId:c,source:"auth",level:"warning",message:"Authorization may be unsafe, passed state was changed in server Passed state wasn't returned from auth server"}),a.error?r.newAuthErr({authId:c,source:"auth",level:"error",message:(0,i.default)(a)}):n.authorizeOauth2({auth:o,token:a})}};t.authorizePassword=function(e){return function(t){var n=t.authActions,r=e.schema,i=e.name,a=e.username,u=e.password,l=e.passwordType,c=e.clientId,f=e.clientSecret,p={grant_type:"password",scope:e.scopes.join(" "),username:a,password:u},d={};switch(l){case"request-body":!function(e,t,n){t&&(0,o.default)(e,{client_id:t});n&&(0,o.default)(e,{client_secret:n})}(p,c,f);break;case"basic":d.Authorization="Basic "+(0,s.btoa)(c+":"+f);break;default:console.warn("Warning: invalid passwordType "+l+" was passed, not including client id and secret")}return n.authorizeRequest({body:(0,s.buildFormData)(p),url:r.get("tokenUrl"),name:i,headers:d,query:{},auth:e})}};t.authorizeApplication=function(e){return function(t){var n=t.authActions,r=e.schema,o=e.scopes,i=e.name,a=e.clientId,u=e.clientSecret,l={Authorization:"Basic "+(0,s.btoa)(a+":"+u)},c={grant_type:"client_credentials",scope:o.join(" ")};return n.authorizeRequest({body:(0,s.buildFormData)(c),name:i,url:r.get("tokenUrl"),auth:e,headers:l})}},t.authorizeAccessCodeWithFormParams=function(e){var t=e.auth,n=e.redirectUrl;return function(e){var r=e.authActions,o=t.schema,i=t.name,a=t.clientId,u=t.clientSecret,l={grant_type:"authorization_code",code:t.code,client_id:a,client_secret:u,redirect_uri:n};return r.authorizeRequest({body:(0,s.buildFormData)(l),name:i,url:o.get("tokenUrl"),auth:t})}},t.authorizeAccessCodeWithBasicAuthentication=function(e){var t=e.auth,n=e.redirectUrl;return function(e){var r=e.authActions,o=t.schema,i=t.name,a=t.clientId,u=t.clientSecret,l={Authorization:"Basic "+(0,s.btoa)(a+":"+u)},c={grant_type:"authorization_code",code:t.code,client_id:a,redirect_uri:n};return r.authorizeRequest({body:(0,s.buildFormData)(c),name:i,url:o.get("tokenUrl"),auth:t,headers:l})}},t.authorizeRequest=function(e){return function(t){var n=t.fn,u=t.getConfigs,s=t.authActions,l=t.errActions,c=t.oas3Selectors,f=t.specSelectors,p=t.authSelectors,d=e.body,h=e.query,v=void 0===h?{}:h,m=e.headers,g=void 0===m?{}:m,y=e.name,b=e.url,_=e.auth,w=(p.getConfigs()||{}).additionalQueryStringParams,E=void 0;E=f.isOAS3()?(0,a.default)(b,c.selectedServer(),!0):(0,a.default)(b,f.url(),!0),"object"===(void 0===w?"undefined":(0,r.default)(w))&&(E.query=(0,o.default)({},E.query,w));var x=E.toString(),S=(0,o.default)({Accept:"application/json, text/plain, */*","Content-Type":"application/x-www-form-urlencoded","X-Requested-With":"XMLHttpRequest"},g);n.fetch({url:x,method:"post",headers:S,query:v,body:d,requestInterceptor:u().requestInterceptor,responseInterceptor:u().responseInterceptor}).then(function(e){var t=JSON.parse(e.data),n=t&&(t.error||""),r=t&&(t.parseError||"");e.ok?n||r?l.newAuthErr({authId:y,level:"error",source:"auth",message:(0,i.default)(t)}):s.authorizeOauth2({auth:_,token:t}):l.newAuthErr({authId:y,level:"error",source:"auth",message:e.statusText})}).catch(function(e){var t=new Error(e).message;if(e.response&&e.response.data){var n=e.response.data;try{var r="string"==typeof n?JSON.parse(n):n;r.error&&(t+=", error: "+r.error),r.error_description&&(t+=", description: "+r.error_description)}catch(e){}}l.newAuthErr({authId:y,level:"error",source:"auth",message:t})})}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.parseYamlConfig=void 0;var r,o=n(208),i=(r=o)&&r.__esModule?r:{default:r};t.parseYamlConfig=function(e,t){try{return i.default.safeLoad(e)}catch(e){return t&&t.errActions.newThrownErr(new Error(e)),{}}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.loaded=t.TOGGLE_CONFIGS=t.UPDATE_CONFIGS=void 0;var r,o=n(22),i=(r=o)&&r.__esModule?r:{default:r};t.update=function(e,t){return{type:a,payload:(0,i.default)({},e,t)}},t.toggle=function(e){return{type:u,payload:e}};var a=t.UPDATE_CONFIGS="configs_update",u=t.TOGGLE_CONFIGS="configs_toggle";t.loaded=function(){return function(){}}},function(e,t,n){"use strict";function r(e,t,n,r,o){this.src=e,this.env=r,this.options=n,this.parser=t,this.tokens=o,this.pos=0,this.posMax=this.src.length,this.level=0,this.pending="",this.pendingLevel=0,this.cache=[],this.isInLabel=!1,this.linkLevel=0,this.linkContent="",this.labelUnmatchedScopes=0}r.prototype.pushPending=function(){this.tokens.push({type:"text",content:this.pending,level:this.pendingLevel}),this.pending=""},r.prototype.push=function(e){this.pending&&this.pushPending(),this.tokens.push(e),this.pendingLevel=this.level},r.prototype.cacheSet=function(e,t){for(var n=this.cache.length;n<=e;n++)this.cache.push(0);this.cache[e]=t},r.prototype.cacheGet=function(e){return es;)r(u,n=t[s++])&&(~i(l,n)||l.push(n));return l}},function(e,t,n){var r=n(21).document;e.exports=r&&r.documentElement},function(e,t,n){var r=n(52),o=n(72),i=n(162)("IE_PROTO"),a=Object.prototype;e.exports=Object.getPrototypeOf||function(e){return e=o(e),r(e,i)?e[i]:"function"==typeof e.constructor&&e instanceof e.constructor?e.constructor.prototype:e instanceof Object?a:null}},function(e,t,n){var r=n(33),o=r["__core-js_shared__"]||(r["__core-js_shared__"]={});e.exports=function(e){return o[e]||(o[e]={})}},function(e,t){e.exports=function(e,t){return{enumerable:!(1&e),configurable:!(2&e),writable:!(4&e),value:t}}},function(e,t,n){"use strict";var r=n(246)(!0);n(247)(String,"String",function(e){this._t=String(e),this._i=0},function(){var e,t=this._t,n=this._i;return n>=t.length?{value:void 0,done:!0}:(e=r(t,n),this._i+=e.length,{value:e,done:!1})})},function(e,t,n){var r=n(119),o=n(53);e.exports=function(e){return function(t,n){var i,a,u=String(o(t)),s=r(n),l=u.length;return s<0||s>=l?e?"":void 0:(i=u.charCodeAt(s))<55296||i>56319||s+1===l||(a=u.charCodeAt(s+1))<56320||a>57343?e?u.charAt(s):i:e?u.slice(s,s+2):a-56320+(i-55296<<10)+65536}}},function(e,t,n){"use strict";var r=n(248),o=n(29),i=n(73),a=n(59),u=n(102),s=n(462),l=n(171),c=n(468),f=n(18)("iterator"),p=!([].keys&&"next"in[].keys()),d=function(){return this};e.exports=function(e,t,n,h,v,m,g){s(n,t,h);var y,b,_,w=function(e){if(!p&&e in C)return C[e];switch(e){case"keys":case"values":return function(){return new n(this,e)}}return function(){return new n(this,e)}},E=t+" Iterator",x="values"==v,S=!1,C=e.prototype,k=C[f]||C["@@iterator"]||v&&C[v],A=k||w(v),O=v?x?w("entries"):A:void 0,P="Array"==t&&C.entries||k;if(P&&(_=c(P.call(new e)))!==Object.prototype&&_.next&&(l(_,E,!0),r||"function"==typeof _[f]||a(_,f,d)),x&&k&&"values"!==k.name&&(S=!0,A=function(){return k.call(this)}),r&&!g||!p&&!S&&C[f]||a(C,f,A),u[t]=A,u[E]=d,v)if(y={values:x?A:w("values"),keys:m?A:w("keys"),entries:O},g)for(b in y)b in C||i(C,b,y[b]);else o(o.P+o.F*(p||S),t,y);return y}},function(e,t){e.exports=!1},function(e,t,n){var r=n(465),o=n(251);e.exports=Object.keys||function(e){return r(e,o)}},function(e,t,n){var r=n(119),o=Math.max,i=Math.min;e.exports=function(e,t){return(e=r(e))<0?o(e+t,0):i(e,t)}},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t,n){var r=n(33).document;e.exports=r&&r.documentElement},function(e,t,n){var r=n(60),o=n(121),i=n(18)("species");e.exports=function(e,t){var n,a=r(e).constructor;return void 0===a||void 0==(n=r(a)[i])?t:o(n)}},function(e,t,n){var r,o,i,a=n(120),u=n(480),s=n(252),l=n(169),c=n(33),f=c.process,p=c.setImmediate,d=c.clearImmediate,h=c.MessageChannel,v=c.Dispatch,m=0,g={},y=function(){var e=+this;if(g.hasOwnProperty(e)){var t=g[e];delete g[e],t()}},b=function(e){y.call(e.data)};p&&d||(p=function(e){for(var t=[],n=1;arguments.length>n;)t.push(arguments[n++]);return g[++m]=function(){u("function"==typeof e?e:Function(e),t)},r(m),m},d=function(e){delete g[e]},"process"==n(99)(f)?r=function(e){f.nextTick(a(y,e,1))}:v&&v.now?r=function(e){v.now(a(y,e,1))}:h?(i=(o=new h).port2,o.port1.onmessage=b,r=a(i.postMessage,i,1)):c.addEventListener&&"function"==typeof postMessage&&!c.importScripts?(r=function(e){c.postMessage(e+"","*")},c.addEventListener("message",b,!1)):r="onreadystatechange"in l("script")?function(e){s.appendChild(l("script")).onreadystatechange=function(){s.removeChild(this),y.call(e)}}:function(e){setTimeout(a(y,e,1),0)}),e.exports={set:p,clear:d}},function(e,t){e.exports=function(e){try{return{e:!1,v:e()}}catch(e){return{e:!0,v:e}}}},function(e,t,n){var r=n(60),o=n(74),i=n(172);e.exports=function(e,t){if(r(e),o(t)&&t.constructor===e)return t;var n=i.f(e);return(0,n.resolve)(t),n.promise}},function(e,t,n){var r=n(74),o=n(99),i=n(18)("match");e.exports=function(e){var t;return r(e)&&(void 0!==(t=e[i])?!!t:"RegExp"==o(e))}},function(e,t,n){var r=n(20),o=n(15),i=n(51);e.exports=function(e,t){var n=(o.Object||{})[e]||Object[e],a={};a[e]=t(n),r(r.S+r.F*i(function(){n(1)}),"Object",a)}},function(e,t,n){var r=n(93);e.exports=Array.isArray||function(e){return"Array"==r(e)}},function(e,t,n){var r=n(240),o=n(164).concat("length","prototype");t.f=Object.getOwnPropertyNames||function(e){return r(e,o)}},function(e,t,n){var r=n(125),o=n(95),i=n(71),a=n(158),u=n(52),s=n(239),l=Object.getOwnPropertyDescriptor;t.f=n(44)?l:function(e,t){if(e=i(e),t=a(t,!0),s)try{return l(e,t)}catch(e){}if(u(e,t))return o(!r.f.call(e,t),e[t])}},function(e,t){var n={}.toString;e.exports=Array.isArray||function(e){return"[object Array]"==n.call(e)}},function(e,t,n){e.exports={default:n(532),__esModule:!0}},function(e,t,n){"use strict";var r=n(96),o=n(177),i=n(125),a=n(72),u=n(155),s=Object.assign;e.exports=!s||n(51)(function(){var e={},t={},n=Symbol(),r="abcdefghijklmnopqrst";return e[n]=7,r.split("").forEach(function(e){t[e]=e}),7!=s({},e)[n]||Object.keys(s({},t)).join("")!=r})?function(e,t){for(var n=a(e),s=arguments.length,l=1,c=o.f,f=i.f;s>l;)for(var p,d=u(arguments[l++]),h=c?r(d).concat(c(d)):r(d),v=h.length,m=0;v>m;)f.call(d,p=h[m++])&&(n[p]=d[p]);return n}:s},function(e,t,n){"use strict";var r=n(104),o=n(13),i=n(266),a=(n(267),n(126));n(8),n(536);function u(e,t,n){this.props=e,this.context=t,this.refs=a,this.updater=n||i}function s(e,t,n){this.props=e,this.context=t,this.refs=a,this.updater=n||i}function l(){}u.prototype.isReactComponent={},u.prototype.setState=function(e,t){"object"!=typeof e&&"function"!=typeof e&&null!=e&&r("85"),this.updater.enqueueSetState(this,e),t&&this.updater.enqueueCallback(this,t,"setState")},u.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this),e&&this.updater.enqueueCallback(this,e,"forceUpdate")},l.prototype=u.prototype,s.prototype=new l,s.prototype.constructor=s,o(s.prototype,u.prototype),s.prototype.isPureReactComponent=!0,e.exports={Component:u,PureComponent:s}},function(e,t,n){"use strict";n(10);var r={isMounted:function(e){return!1},enqueueCallback:function(e,t){},enqueueForceUpdate:function(e){},enqueueReplaceState:function(e,t){},enqueueSetState:function(e,t){}};e.exports=r},function(e,t,n){"use strict";var r=!1;e.exports=r},function(e,t,n){"use strict";var r="function"==typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103;e.exports=r},function(e,t,n){"use strict";var r=n(544);e.exports=function(e){return r(e,!1)}},function(e,t,n){"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(272),o=n(561),i=n(562),a=n(563),u=n(276);n(275);n.d(t,"createStore",function(){return r.b}),n.d(t,"combineReducers",function(){return o.a}),n.d(t,"bindActionCreators",function(){return i.a}),n.d(t,"applyMiddleware",function(){return a.a}),n.d(t,"compose",function(){return u.a})},function(e,t,n){"use strict";n.d(t,"a",function(){return i}),t.b=function e(t,n,a){var u;"function"==typeof n&&void 0===a&&(a=n,n=void 0);if(void 0!==a){if("function"!=typeof a)throw new Error("Expected the enhancer to be a function.");return a(e)(t,n)}if("function"!=typeof t)throw new Error("Expected the reducer to be a function.");var s=t;var l=n;var c=[];var f=c;var p=!1;function d(){f===c&&(f=c.slice())}function h(){return l}function v(e){if("function"!=typeof e)throw new Error("Expected listener to be a function.");var t=!0;return d(),f.push(e),function(){if(t){t=!1,d();var n=f.indexOf(e);f.splice(n,1)}}}function m(e){if(!r.a(e))throw new Error("Actions must be plain objects. Use custom middleware for async actions.");if(void 0===e.type)throw new Error('Actions may not have an undefined "type" property. Have you misspelled a constant?');if(p)throw new Error("Reducers may not dispatch actions.");try{p=!0,l=s(l,e)}finally{p=!1}for(var t=c=f,n=0;no?0:o+t),(n=n>o?o:n)<0&&(n+=o),o=t>n?0:n-t>>>0,t>>>=0;for(var i=Array(o);++rp))return!1;var h=c.get(e);if(h&&c.get(t))return h==t;var v=-1,m=!0,g=n&u?new r:void 0;for(c.set(e,t),c.set(t,e);++v0?("string"==typeof t||a.objectMode||Object.getPrototypeOf(t)===l.prototype||(t=function(e){return l.from(e)}(t)),r?a.endEmitted?e.emit("error",new Error("stream.unshift() after end event")):w(e,a,t,!0):a.ended?e.emit("error",new Error("stream.push() after EOF")):(a.reading=!1,a.decoder&&!n?(t=a.decoder.write(t),a.objectMode||0!==t.length?w(e,a,t,!1):k(e,a)):w(e,a,t,!1))):r||(a.reading=!1));return function(e){return!e.ended&&(e.needReadable||e.lengtht.highWaterMark&&(t.highWaterMark=function(e){return e>=E?e=E:(e--,e|=e>>>1,e|=e>>>2,e|=e>>>4,e|=e>>>8,e|=e>>>16,e++),e}(e)),e<=t.length?e:t.ended?t.length:(t.needReadable=!0,0))}function S(e){var t=e._readableState;t.needReadable=!1,t.emittedReadable||(d("emitReadable",t.flowing),t.emittedReadable=!0,t.sync?o.nextTick(C,e):C(e))}function C(e){d("emit readable"),e.emit("readable"),T(e)}function k(e,t){t.readingMore||(t.readingMore=!0,o.nextTick(A,e,t))}function A(e,t){for(var n=t.length;!t.reading&&!t.flowing&&!t.ended&&t.length=t.length?(n=t.decoder?t.buffer.join(""):1===t.buffer.length?t.buffer.head.data:t.buffer.concat(t.length),t.buffer.clear()):n=function(e,t,n){var r;ei.length?i.length:e;if(a===i.length?o+=i:o+=i.slice(0,e),0===(e-=a)){a===i.length?(++r,n.next?t.head=n.next:t.head=t.tail=null):(t.head=n,n.data=i.slice(a));break}++r}return t.length-=r,o}(e,t):function(e,t){var n=l.allocUnsafe(e),r=t.head,o=1;r.data.copy(n),e-=r.data.length;for(;r=r.next;){var i=r.data,a=e>i.length?i.length:e;if(i.copy(n,n.length-e,0,a),0===(e-=a)){a===i.length?(++o,r.next?t.head=r.next:t.head=t.tail=null):(t.head=r,r.data=i.slice(a));break}++o}return t.length-=o,n}(e,t);return r}(e,t.buffer,t.decoder),n);var n}function I(e){var t=e._readableState;if(t.length>0)throw new Error('"endReadable()" called on non-empty stream');t.endEmitted||(t.ended=!0,o.nextTick(j,t,e))}function j(e,t){e.endEmitted||0!==e.length||(e.endEmitted=!0,t.readable=!1,t.emit("end"))}function N(e,t){for(var n=0,r=e.length;n=t.highWaterMark||t.ended))return d("read: emitReadable",t.length,t.ended),0===t.length&&t.ended?I(this):S(this),null;if(0===(e=x(e,t))&&t.ended)return 0===t.length&&I(this),null;var r,o=t.needReadable;return d("need readable",o),(0===t.length||t.length-e0?M(e,t):null)?(t.needReadable=!0,e=0):t.length-=e,0===t.length&&(t.ended||(t.needReadable=!0),n!==e&&t.ended&&I(this)),null!==r&&this.emit("data",r),r},b.prototype._read=function(e){this.emit("error",new Error("_read() is not implemented"))},b.prototype.pipe=function(e,t){var n=this,i=this._readableState;switch(i.pipesCount){case 0:i.pipes=e;break;case 1:i.pipes=[i.pipes,e];break;default:i.pipes.push(e)}i.pipesCount+=1,d("pipe count=%d opts=%j",i.pipesCount,t);var s=(!t||!1!==t.end)&&e!==r.stdout&&e!==r.stderr?c:b;function l(t,r){d("onunpipe"),t===n&&r&&!1===r.hasUnpiped&&(r.hasUnpiped=!0,d("cleanup"),e.removeListener("close",g),e.removeListener("finish",y),e.removeListener("drain",f),e.removeListener("error",m),e.removeListener("unpipe",l),n.removeListener("end",c),n.removeListener("end",b),n.removeListener("data",v),p=!0,!i.awaitDrain||e._writableState&&!e._writableState.needDrain||f())}function c(){d("onend"),e.end()}i.endEmitted?o.nextTick(s):n.once("end",s),e.on("unpipe",l);var f=function(e){return function(){var t=e._readableState;d("pipeOnDrain",t.awaitDrain),t.awaitDrain&&t.awaitDrain--,0===t.awaitDrain&&u(e,"data")&&(t.flowing=!0,T(e))}}(n);e.on("drain",f);var p=!1;var h=!1;function v(t){d("ondata"),h=!1,!1!==e.write(t)||h||((1===i.pipesCount&&i.pipes===e||i.pipesCount>1&&-1!==N(i.pipes,e))&&!p&&(d("false write response, pause",n._readableState.awaitDrain),n._readableState.awaitDrain++,h=!0),n.pause())}function m(t){d("onerror",t),b(),e.removeListener("error",m),0===u(e,"error")&&e.emit("error",t)}function g(){e.removeListener("finish",y),b()}function y(){d("onfinish"),e.removeListener("close",g),b()}function b(){d("unpipe"),n.unpipe(e)}return n.on("data",v),function(e,t,n){if("function"==typeof e.prependListener)return e.prependListener(t,n);e._events&&e._events[t]?a(e._events[t])?e._events[t].unshift(n):e._events[t]=[n,e._events[t]]:e.on(t,n)}(e,"error",m),e.once("close",g),e.once("finish",y),e.emit("pipe",n),i.flowing||(d("pipe resume"),n.resume()),e},b.prototype.unpipe=function(e){var t=this._readableState,n={hasUnpiped:!1};if(0===t.pipesCount)return this;if(1===t.pipesCount)return e&&e!==t.pipes?this:(e||(e=t.pipes),t.pipes=null,t.pipesCount=0,t.flowing=!1,e&&e.emit("unpipe",this,n),this);if(!e){var r=t.pipes,o=t.pipesCount;t.pipes=null,t.pipesCount=0,t.flowing=!1;for(var i=0;i=0&&(e._idleTimeoutId=setTimeout(function(){e._onTimeout&&e._onTimeout()},t))},n(663),t.setImmediate="undefined"!=typeof self&&self.setImmediate||void 0!==e&&e.setImmediate||this&&this.setImmediate,t.clearImmediate="undefined"!=typeof self&&self.clearImmediate||void 0!==e&&e.clearImmediate||this&&this.clearImmediate}).call(t,n(31))},function(e,t,n){"use strict";var r=n(141).Buffer,o=r.isEncoding||function(e){switch((e=""+e)&&e.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function i(e){var t;switch(this.encoding=function(e){var t=function(e){if(!e)return"utf8";for(var t;;)switch(e){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return e;default:if(t)return;e=(""+e).toLowerCase(),t=!0}}(e);if("string"!=typeof t&&(r.isEncoding===o||!o(e)))throw new Error("Unknown encoding: "+e);return t||e}(e),this.encoding){case"utf16le":this.text=s,this.end=l,t=4;break;case"utf8":this.fillLast=u,t=4;break;case"base64":this.text=c,this.end=f,t=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=r.allocUnsafe(t)}function a(e){return e<=127?0:e>>5==6?2:e>>4==14?3:e>>3==30?4:e>>6==2?-1:-2}function u(e){var t=this.lastTotal-this.lastNeed,n=function(e,t,n){if(128!=(192&t[0]))return e.lastNeed=0,"�";if(e.lastNeed>1&&t.length>1){if(128!=(192&t[1]))return e.lastNeed=1,"�";if(e.lastNeed>2&&t.length>2&&128!=(192&t[2]))return e.lastNeed=2,"�"}}(this,e);return void 0!==n?n:this.lastNeed<=e.length?(e.copy(this.lastChar,t,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(e.copy(this.lastChar,t,0,e.length),void(this.lastNeed-=e.length))}function s(e,t){if((e.length-t)%2==0){var n=e.toString("utf16le",t);if(n){var r=n.charCodeAt(n.length-1);if(r>=55296&&r<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=e[e.length-2],this.lastChar[1]=e[e.length-1],n.slice(0,-1)}return n}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=e[e.length-1],e.toString("utf16le",t,e.length-1)}function l(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed){var n=this.lastTotal-this.lastNeed;return t+this.lastChar.toString("utf16le",0,n)}return t}function c(e,t){var n=(e.length-t)%3;return 0===n?e.toString("base64",t):(this.lastNeed=3-n,this.lastTotal=3,1===n?this.lastChar[0]=e[e.length-1]:(this.lastChar[0]=e[e.length-2],this.lastChar[1]=e[e.length-1]),e.toString("base64",t,e.length-n))}function f(e){var t=e&&e.length?this.write(e):"";return this.lastNeed?t+this.lastChar.toString("base64",0,3-this.lastNeed):t}function p(e){return e.toString(this.encoding)}function d(e){return e&&e.length?this.write(e):""}t.StringDecoder=i,i.prototype.write=function(e){if(0===e.length)return"";var t,n;if(this.lastNeed){if(void 0===(t=this.fillLast(e)))return"";n=this.lastNeed,this.lastNeed=0}else n=0;return n=0)return o>0&&(e.lastNeed=o-1),o;if(--r=0)return o>0&&(e.lastNeed=o-2),o;if(--r=0)return o>0&&(2===o?o=0:e.lastNeed=o-3),o;return 0}(this,e,t);if(!this.lastNeed)return e.toString("utf8",t);this.lastTotal=n;var r=e.length-(n-this.lastNeed);return e.copy(this.lastChar,0,r),e.toString("utf8",t,r)},i.prototype.fillLast=function(e){if(this.lastNeed<=e.length)return e.copy(this.lastChar,this.lastTotal-this.lastNeed,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal);e.copy(this.lastChar,this.lastTotal-this.lastNeed,0,e.length),this.lastNeed-=e.length}},function(e,t,n){"use strict";e.exports=i;var r=n(65),o=n(106);function i(e){if(!(this instanceof i))return new i(e);r.call(this,e),this._transformState={afterTransform:function(e,t){var n=this._transformState;n.transforming=!1;var r=n.writecb;if(!r)return this.emit("error",new Error("write callback called multiple times"));n.writechunk=null,n.writecb=null,null!=t&&this.push(t),r(e);var o=this._readableState;o.reading=!1,(o.needReadable||o.length=0?n&&o?o-1:o:1:!1!==e&&r(e)}},function(e,t,n){"use strict";e.exports=n(679)()?Object.assign:n(680)},function(e,t,n){"use strict";var r,o,i,a,u,s=n(67),l=function(e,t){return t};try{Object.defineProperty(l,"length",{configurable:!0,writable:!1,enumerable:!1,value:1})}catch(e){}1===l.length?(r={configurable:!0,writable:!1,enumerable:!1},o=Object.defineProperty,e.exports=function(e,t){return t=s(t),e.length===t?e:(r.value=t,o(e,"length",r))}):(a=n(317),u=[],i=function(e){var t,n=0;if(u[e])return u[e];for(t=[];e--;)t.push("a"+(++n).toString(36));return new Function("fn","return function ("+t.join(", ")+") { return fn.apply(this, arguments); };")},e.exports=function(e,t){var n;if(t=s(t),e.length===t)return e;n=i(t)(e);try{a(n,e)}catch(e){}return n})},function(e,t,n){"use strict";var r=n(82),o=Object.defineProperty,i=Object.getOwnPropertyDescriptor,a=Object.getOwnPropertyNames,u=Object.getOwnPropertySymbols;e.exports=function(e,t){var n,s=Object(r(t));if(e=Object(r(e)),a(s).forEach(function(r){try{o(e,r,i(t,r))}catch(e){n=e}}),"function"==typeof u&&u(s).forEach(function(r){try{o(e,r,i(t,r))}catch(e){n=e}}),void 0!==n)throw n;return e}},function(e,t,n){"use strict";var r=n(57),o=n(142),i=Function.prototype.call;e.exports=function(e,t){var n={},a=arguments[2];return r(t),o(e,function(e,r,o,u){n[r]=i.call(t,a,e,r,o,u)}),n}},function(e,t){e.exports=function(e){return!!e&&("object"==typeof e||"function"==typeof e)&&"function"==typeof e.then}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e){return{statePlugins:{err:{reducers:(0,i.default)(e),actions:a,selectors:u}}}};var r,o=n(321),i=(r=o)&&r.__esModule?r:{default:r},a=s(n(127)),u=s(n(325));function s(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=s(n(22)),o=s(n(23));t.default=function(e){var t;return t={},(0,r.default)(t,i.NEW_THROWN_ERR,function(t,n){var r=n.payload,i=(0,o.default)(l,r,{type:"thrown"});return t.update("errors",function(e){return(e||(0,a.List)()).push((0,a.fromJS)(i))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_THROWN_ERR_BATCH,function(t,n){var r=n.payload;return r=r.map(function(e){return(0,a.fromJS)((0,o.default)(l,e,{type:"thrown"}))}),t.update("errors",function(e){return(e||(0,a.List)()).concat((0,a.fromJS)(r))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_SPEC_ERR,function(t,n){var r=n.payload,o=(0,a.fromJS)(r);return o=o.set("type","spec"),t.update("errors",function(e){return(e||(0,a.List)()).push((0,a.fromJS)(o)).sortBy(function(e){return e.get("line")})}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_SPEC_ERR_BATCH,function(t,n){var r=n.payload;return r=r.map(function(e){return(0,a.fromJS)((0,o.default)(l,e,{type:"spec"}))}),t.update("errors",function(e){return(e||(0,a.List)()).concat((0,a.fromJS)(r))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_AUTH_ERR,function(t,n){var r=n.payload,i=(0,a.fromJS)((0,o.default)({},r));return i=i.set("type","auth"),t.update("errors",function(e){return(e||(0,a.List)()).push((0,a.fromJS)(i))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.CLEAR,function(e,t){var n=t.payload;if(!n||!e.get("errors"))return e;var r=e.get("errors").filter(function(e){return e.keySeq().every(function(t){var r=e.get(t),o=n[t];return!o||r!==o})});return e.merge({errors:r})}),(0,r.default)(t,i.CLEAR_BY,function(e,t){var n=t.payload;if(!n||"function"!=typeof n)return e;var r=e.get("errors").filter(function(e){return n(e)});return e.merge({errors:r})}),t};var i=n(127),a=n(7),u=s(n(322));function s(e){return e&&e.__esModule?e:{default:e}}var l={line:0,level:"error",message:"Unknown error"}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){var n={jsSpec:t.specSelectors.specJson().toJS()};return(0,i.default)(u,function(e,t){try{var r=t.transform(e,n);return r.filter(function(e){return!!e})}catch(t){return console.error("Transformer error:",t),e}},e).filter(function(e){return!!e}).map(function(e){return!e.get("line")&&e.get("path"),e})};var r,o=n(727),i=(r=o)&&r.__esModule?r:{default:r};function a(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}var u=[a(n(323)),a(n(324))]},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.transform=function(e){return e.map(function(e){var t=e.get("message").indexOf("is not of a type(s)");if(t>-1){var n=e.get("message").slice(t+"is not of a type(s)".length).split(",");return e.set("message",e.get("message").slice(0,t)+function(e){return e.reduce(function(e,t,n,r){return n===r.length-1&&r.length>1?e+"or "+t:r[n+1]&&r.length>2?e+t+", ":r[n+1]?e+t+" ":e+t},"should be a")}(n))}return e})}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.transform=function(e,t){t.jsSpec;return e};var r,o=n(138);(r=o)&&r.__esModule,n(7)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.lastError=t.allErrors=void 0;var r=n(7),o=n(58),i=t.allErrors=(0,o.createSelector)(function(e){return e},function(e){return e.get("errors",(0,r.List)())});t.lastError=(0,o.createSelector)(i,function(e){return e.last()})},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{statePlugins:{layout:{reducers:i.default,actions:a,selectors:u}}}};var r,o=n(327),i=(r=o)&&r.__esModule?r:{default:r},a=s(n(202)),u=s(n(328));function s(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r,o,i=n(22),a=(r=i)&&r.__esModule?r:{default:r},u=n(7),s=n(202);t.default=(o={},(0,a.default)(o,s.UPDATE_LAYOUT,function(e,t){return e.set("layout",t.payload)}),(0,a.default)(o,s.UPDATE_FILTER,function(e,t){return e.set("filter",t.payload)}),(0,a.default)(o,s.SHOW,function(e,t){var n=t.payload.shown,r=(0,u.fromJS)(t.payload.thing);return e.update("shown",(0,u.fromJS)({}),function(e){return e.set(r,n)})}),(0,a.default)(o,s.UPDATE_MODE,function(e,t){var n=t.payload.thing,r=t.payload.mode;return e.setIn(["modes"].concat(n),(r||"")+"")}),o)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.showSummary=t.whatMode=t.isShown=t.currentFilter=t.current=void 0;var r,o=n(83),i=(r=o)&&r.__esModule?r:{default:r},a=n(58),u=n(9),s=n(7);t.current=function(e){return e.get("layout")},t.currentFilter=function(e){return e.get("filter")};var l=t.isShown=function(e,t,n){return t=(0,u.normalizeArray)(t),e.get("shown",(0,s.fromJS)({})).get((0,s.fromJS)(t),n)};t.whatMode=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"";return t=(0,u.normalizeArray)(t),e.getIn(["modes"].concat((0,i.default)(t)),n)},t.showSummary=(0,a.createSelector)(function(e){return e},function(e){return!l(e,"editor")})},function(e,t,n){var r=n(36);e.exports=function(e,t,n,o){try{return o?t(r(n)[0],n[1]):t(n)}catch(t){var i=e.return;throw void 0!==i&&r(i.call(e)),t}}},function(e,t,n){var r=n(70),o=n(19)("iterator"),i=Array.prototype;e.exports=function(e){return void 0!==e&&(r.Array===e||i[o]===e)}},function(e,t,n){var r=n(19)("iterator"),o=!1;try{var i=[7][r]();i.return=function(){o=!0},Array.from(i,function(){throw 2})}catch(e){}e.exports=function(e,t){if(!t&&!o)return!1;var n=!1;try{var i=[7],a=i[r]();a.next=function(){return{done:n=!0}},i[r]=function(){return a},e(i)}catch(e){}return n}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{statePlugins:{spec:{wrapActions:s,reducers:i.default,actions:a,selectors:u}}}};var r,o=n(333),i=(r=o)&&r.__esModule?r:{default:r},a=l(n(203)),u=l(n(144)),s=l(n(346));function l(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r,o=p(n(22)),i=p(n(23)),a=p(n(83)),u=n(7),s=n(9),l=p(n(32)),c=n(144),f=n(203);function p(e){return e&&e.__esModule?e:{default:e}}t.default=(r={},(0,o.default)(r,f.UPDATE_SPEC,function(e,t){return"string"==typeof t.payload?e.set("spec",t.payload):e}),(0,o.default)(r,f.UPDATE_URL,function(e,t){return e.set("url",t.payload+"")}),(0,o.default)(r,f.UPDATE_JSON,function(e,t){return e.set("json",(0,s.fromJSOrdered)(t.payload))}),(0,o.default)(r,f.UPDATE_RESOLVED,function(e,t){return e.setIn(["resolved"],(0,s.fromJSOrdered)(t.payload))}),(0,o.default)(r,f.UPDATE_RESOLVED_SUBTREE,function(e,t){var n=t.payload,r=n.value,o=n.path;return e.setIn(["resolvedSubtrees"].concat((0,a.default)(o)),(0,s.fromJSOrdered)(r))}),(0,o.default)(r,f.UPDATE_PARAM,function(e,t){var n=t.payload,r=n.path,o=n.paramName,i=n.paramIn,u=n.param,l=n.value,c=n.isXml,f=u?(0,s.paramToIdentifier)(u):i+"."+o,p=c?"value_xml":"value";return e.setIn(["meta","paths"].concat((0,a.default)(r),["parameters",f,p]),l)}),(0,o.default)(r,f.UPDATE_EMPTY_PARAM_INCLUSION,function(e,t){var n=t.payload,r=n.pathMethod,o=n.paramName,i=n.paramIn,u=n.includeEmptyValue;if(!o||!i)return console.warn("Warning: UPDATE_EMPTY_PARAM_INCLUSION could not generate a paramKey."),e;var s=i+"."+o;return e.setIn(["meta","paths"].concat((0,a.default)(r),["parameter_inclusions",s]),u)}),(0,o.default)(r,f.VALIDATE_PARAMS,function(e,t){var n=t.payload,r=n.pathMethod,o=n.isOAS3,i=(0,c.specJsonWithResolvedSubtrees)(e).getIn(["paths"].concat((0,a.default)(r))),l=(0,c.parameterValues)(e,r).toJS();return e.updateIn(["meta","paths"].concat((0,a.default)(r),["parameters"]),(0,u.fromJS)({}),function(t){return i.get("parameters",(0,u.List)()).reduce(function(t,n){var i=(0,s.paramToValue)(n,l),a=(0,c.parameterInclusionSettingFor)(e,r,n.get("name"),n.get("in")),f=(0,s.validateParam)(n,i,{bypassRequiredCheck:a,isOAS3:o});return t.setIn([(0,s.paramToIdentifier)(n),"errors"],(0,u.fromJS)(f))},t)})}),(0,o.default)(r,f.CLEAR_VALIDATE_PARAMS,function(e,t){var n=t.payload.pathMethod;return e.updateIn(["meta","paths"].concat((0,a.default)(n),["parameters"]),(0,u.fromJS)([]),function(e){return e.map(function(e){return e.set("errors",(0,u.fromJS)([]))})})}),(0,o.default)(r,f.SET_RESPONSE,function(e,t){var n=t.payload,r=n.res,o=n.path,a=n.method,u=void 0;(u=r.error?(0,i.default)({error:!0,name:r.err.name,message:r.err.message,statusCode:r.err.statusCode},r.err.response):r).headers=u.headers||{};var c=e.setIn(["responses",o,a],(0,s.fromJSOrdered)(u));return l.default.Blob&&r.data instanceof l.default.Blob&&(c=c.setIn(["responses",o,a,"text"],r.data)),c}),(0,o.default)(r,f.SET_REQUEST,function(e,t){var n=t.payload,r=n.req,o=n.path,i=n.method;return e.setIn(["requests",o,i],(0,s.fromJSOrdered)(r))}),(0,o.default)(r,f.SET_MUTATED_REQUEST,function(e,t){var n=t.payload,r=n.req,o=n.path,i=n.method;return e.setIn(["mutatedRequests",o,i],(0,s.fromJSOrdered)(r))}),(0,o.default)(r,f.UPDATE_OPERATION_META_VALUE,function(e,t){var n=t.payload,r=n.path,o=n.value,i=n.key,s=["paths"].concat((0,a.default)(r)),l=["meta","paths"].concat((0,a.default)(r));return e.getIn(["json"].concat((0,a.default)(s)))||e.getIn(["resolved"].concat((0,a.default)(s)))||e.getIn(["resolvedSubtrees"].concat((0,a.default)(s)))?e.setIn([].concat((0,a.default)(l),[i]),(0,u.fromJS)(o)):e}),(0,o.default)(r,f.CLEAR_RESPONSE,function(e,t){var n=t.payload,r=n.path,o=n.method;return e.deleteIn(["responses",r,o])}),(0,o.default)(r,f.CLEAR_REQUEST,function(e,t){var n=t.payload,r=n.path,o=n.method;return e.deleteIn(["requests",r,o])}),(0,o.default)(r,f.SET_SCHEME,function(e,t){var n=t.payload,r=n.scheme,o=n.path,i=n.method;return o&&i?e.setIn(["scheme",o,i],r):o||i?void 0:e.setIn(["scheme","_defaultScheme"],r)}),r)},function(e,t,n){var r=n(36),o=n(94),i=n(19)("species");e.exports=function(e,t){var n,a=r(e).constructor;return void 0===a||void 0==(n=r(a)[i])?t:o(n)}},function(e,t,n){var r,o,i,a=n(49),u=n(735),s=n(241),l=n(157),c=n(21),f=c.process,p=c.setImmediate,d=c.clearImmediate,h=c.MessageChannel,v=c.Dispatch,m=0,g={},y=function(){var e=+this;if(g.hasOwnProperty(e)){var t=g[e];delete g[e],t()}},b=function(e){y.call(e.data)};p&&d||(p=function(e){for(var t=[],n=1;arguments.length>n;)t.push(arguments[n++]);return g[++m]=function(){u("function"==typeof e?e:Function(e),t)},r(m),m},d=function(e){delete g[e]},"process"==n(93)(f)?r=function(e){f.nextTick(a(y,e,1))}:v&&v.now?r=function(e){v.now(a(y,e,1))}:h?(i=(o=new h).port2,o.port1.onmessage=b,r=a(i.postMessage,i,1)):c.addEventListener&&"function"==typeof postMessage&&!c.importScripts?(r=function(e){c.postMessage(e+"","*")},c.addEventListener("message",b,!1)):r="onreadystatechange"in l("script")?function(e){s.appendChild(l("script")).onreadystatechange=function(){s.removeChild(this),y.call(e)}}:function(e){setTimeout(a(y,e,1),0)}),e.exports={set:p,clear:d}},function(e,t){e.exports=function(e){try{return{e:!1,v:e()}}catch(e){return{e:!0,v:e}}}},function(e,t,n){var r=n(36),o=n(28),i=n(206);e.exports=function(e,t){if(r(e),o(t)&&t.constructor===e)return t;var n=i.f(e);return(0,n.resolve)(t),n.promise}},function(e,t,n){e.exports=n(740)},function(e,t,n){"use strict";t.__esModule=!0;var r,o=n(204),i=(r=o)&&r.__esModule?r:{default:r};t.default=function(e){return function(){var t=e.apply(this,arguments);return new i.default(function(e,n){return function r(o,a){try{var u=t[o](a),s=u.value}catch(e){return void n(e)}if(!u.done)return i.default.resolve(s).then(function(e){r("next",e)},function(e){r("throw",e)});e(s)}("next")})}}},function(e,t,n){"use strict";var r=n(86);e.exports=new r({include:[n(341)]})},function(e,t,n){"use strict";var r=n(86);e.exports=new r({include:[n(209)],implicit:[n(748),n(749),n(750),n(751)]})},function(e,t,n){var r=n(62),o=n(24),i=n(47),a="[object String]";e.exports=function(e){return"string"==typeof e||!o(e)&&i(e)&&r(e)==a}},function(e,t,n){var r=n(147),o=n(79),i=n(135),a=n(38),u=n(80);e.exports=function(e,t,n,s){if(!a(e))return e;for(var l=-1,c=(t=o(t,e)).length,f=c-1,p=e;null!=p&&++l.":"function"==typeof t?" Instead of passing a class like Foo, pass React.createElement(Foo) or .":null!=t&&void 0!==t.props?" This may be caused by unintentionally loading two independent copies of React.":"");var i,u=a.createElement(D,{child:t});if(e){var s=p.get(e);i=s._processChildContext(s._context)}else i=g;var l=N(n);if(l){var c=l._currentElement.props.child;if(_(c,t)){var f=l._renderedComponent.getPublicInstance(),d=o&&function(){o.call(f)};return L._updateRootComponent(l,u,i,n,d),f}L.unmountComponentAtNode(n)}var h=A(n),m=h&&!!O(h),y=I(n),b=m&&!l&&!y,w=L._renderNewRootComponent(u,n,b,i)._renderedComponent.getPublicInstance();return o&&o.call(w),w},render:function(e,t,n){return L._renderSubtreeIntoContainer(null,e,t,n)},unmountComponentAtNode:function(e){j(e)||r("40");var t=N(e);if(!t){I(e),1===e.nodeType&&e.hasAttribute(E);return!1}return delete k[t._instance.rootID],m.batchedUpdates(M,t,e,!1),!0},_mountImageIntoNode:function(e,t,n,i,a){if(j(t)||r("41"),i){var u=A(t);if(d.canReuseMarkup(e,u))return void s.precacheNode(n,u);var l=u.getAttribute(d.CHECKSUM_ATTR_NAME);u.removeAttribute(d.CHECKSUM_ATTR_NAME);var c=u.outerHTML;u.setAttribute(d.CHECKSUM_ATTR_NAME,l);var f=e,p=function(e,t){for(var n=Math.min(e.length,t.length),r=0;r1?r-1:0),a=1;a=o&&(t=console)[e].apply(t,i)}return i.warn=i.bind(null,"warn"),i.error=i.bind(null,"error"),i.info=i.bind(null,"info"),i.debug=i.bind(null,"debug"),{rootInjects:{log:i}}}},function(e,t,n){"use strict";var r,o=n(387),i=(r=o)&&r.__esModule?r:{default:r},a=function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}(n(393));e.exports=function(e){var t=e.configs,n=e.getConfigs;return{fn:{fetch:i.default.makeHttp(t.preFetch,t.postFetch),buildRequest:i.default.buildRequest,execute:i.default.execute,resolve:i.default.resolve,resolveSubtree:function(e,t,r){for(var o=arguments.length,a=Array(o>3?o-3:0),u=3;u2&&void 0!==arguments[2]?arguments[2]:"",r=(arguments.length>3&&void 0!==arguments[3]?arguments[3]:{}).v2OperationIdCompatibilityMode;return e&&"object"===(void 0===e?"undefined":(0,c.default)(e))?(e.operationId||"").replace(/\s/g,"").length?h(e.operationId):i(t,n,{v2OperationIdCompatibilityMode:r}):null}function i(e,t){if((arguments.length>2&&void 0!==arguments[2]?arguments[2]:{}).v2OperationIdCompatibilityMode){var n=(t.toLowerCase()+"_"+e).replace(/[\s!@#$%^&*()_+=[{\]};:<>|.\/?,\\'""-]/g,"_");return(n=n||e.substring(1)+"_"+t).replace(/((_){2,})/g,"_").replace(/^(_)*/g,"").replace(/([_])*$/g,"")}return""+d(t)+h(e)}function a(e,t){return d(t)+"-"+e}function u(e,t){return s(e,t,!0)||null}function s(e,t,n){if(!e||"object"!==(void 0===e?"undefined":(0,c.default)(e))||!e.paths||"object"!==(0,c.default)(e.paths))return null;var r=e.paths;for(var o in r)for(var i in r[o])if("PARAMETERS"!==i.toUpperCase()){var a=r[o][i];if(a&&"object"===(void 0===a?"undefined":(0,c.default)(a))){var u={spec:e,pathName:o,method:i.toUpperCase(),operation:a},s=t(u);if(n&&s)return u}}}Object.defineProperty(t,"__esModule",{value:!0});var l=r(n(18)),c=r(n(1));t.isOAS3=function(e){var t=e.openapi;return!!t&&(0,p.default)(t,"3")},t.isSwagger2=function(e){var t=e.swagger;return!!t&&(0,p.default)(t,"2")},t.opId=o,t.idFromPathMethod=i,t.legacyIdFromPathMethod=a,t.getOperationRaw=function(e,t){return e&&e.paths?u(e,function(e){var n=e.pathName,r=e.method,i=e.operation;if(!i||"object"!==(void 0===i?"undefined":(0,c.default)(i)))return!1;var u=i.operationId;return[o(i,n,r),a(n,r),u].some(function(e){return e&&e===t})}):null},t.findOperation=u,t.eachOperation=s,t.normalizeSwagger=function(e){var t=e.spec,n=t.paths,r={};if(!n||t.$$normalized)return e;for(var i in n){var a=n[i];if((0,f.default)(a)){var u=a.parameters;for(var s in a)!function(e){var n=a[e];if(!(0,f.default)(n))return"continue";var s=o(n,i,e);if(s){r[s]?r[s].push(n):r[s]=[n];var c=r[s];if(c.length>1)c.forEach(function(e,t){e.__originalOperationId=e.__originalOperationId||e.operationId,e.operationId=""+s+(t+1)});else if(void 0!==n.operationId){var p=c[0];p.__originalOperationId=p.__originalOperationId||n.operationId,p.operationId=s}}if("parameters"!==e){var d=[],h={};for(var v in t)"produces"!==v&&"consumes"!==v&&"security"!==v||(h[v]=t[v],d.push(h));if(u&&(h.parameters=u,d.push(h)),d.length){var m=!0,g=!1,y=void 0;try{for(var b,_=(0,l.default)(d);!(m=(b=_.next()).done);m=!0){var w=b.value;for(var E in w)if(n[E]){if("parameters"===E){var x=!0,S=!1,C=void 0;try{for(var k,A=(0,l.default)(w[E]);!(x=(k=A.next()).done);x=!0)!function(){var e=k.value;n[E].some(function(t){return t.name&&t.name===e.name||t.$ref&&t.$ref===e.$ref||t.$$ref&&t.$$ref===e.$$ref||t===e})||n[E].push(e)}()}catch(e){S=!0,C=e}finally{try{!x&&A.return&&A.return()}finally{if(S)throw C}}}}else n[E]=w[E]}}catch(e){g=!0,y=e}finally{try{!m&&_.return&&_.return()}finally{if(g)throw y}}}}}(s)}}return t.$$normalized=!0,e};var f=r(n(47)),p=r(n(14)),d=function(e){return String.prototype.toLowerCase.call(e)},h=function(e){return e.replace(/[^\w]/gi,"_")}},function(e,t){e.exports=n(893)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e,t){var n=(arguments.length>2&&void 0!==arguments[2]?arguments[2]:{}).loadSpec,r=void 0!==n&&n,o={ok:e.ok,url:e.url||t,status:e.status,statusText:e.statusText,headers:i(e.headers)},a=o.headers["content-type"],u=r||_(a);return(u?e.text:e.blob||e.buffer).call(e).then(function(e){if(o.text=e,o.data=e,u)try{var t=function(e,t){return t&&(0===t.indexOf("application/json")||t.indexOf("+json")>0)?JSON.parse(e):g.default.safeLoad(e)}(e,a);o.body=t,o.obj=t}catch(e){o.parseError=e}return o})}function i(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t={};return"function"==typeof e.forEach?(e.forEach(function(e,n){void 0!==t[n]?(t[n]=Array.isArray(t[n])?t[n]:[t[n]],t[n].push(e)):t[n]=e}),t):t}function a(e,t){return t||"undefined"==typeof navigator||(t=navigator),t&&"ReactNative"===t.product?!(!e||"object"!==(void 0===e?"undefined":(0,h.default)(e))||"string"!=typeof e.uri):"undefined"!=typeof File?e instanceof File:null!==e&&"object"===(void 0===e?"undefined":(0,h.default)(e))&&"function"==typeof e.pipe}function u(e,t){var n=e.collectionFormat,r=e.allowEmptyValue,o="object"===(void 0===e?"undefined":(0,h.default)(e))?e.value:e;if(void 0===o&&r)return"";if(a(o)||"boolean"==typeof o)return o;var i=encodeURIComponent;return t&&(i=(0,y.default)(o)?function(e){return e}:function(e){return(0,p.default)(e)}),"object"!==(void 0===o?"undefined":(0,h.default)(o))||Array.isArray(o)?Array.isArray(o)?Array.isArray(o)&&!n?o.map(i).join(","):"multi"===n?o.map(i):o.map(i).join({csv:",",ssv:"%20",tsv:"%09",pipes:"|"}[n]):i(o):""}function s(e){var t=(0,f.default)(e).reduce(function(t,n){var r=e[n],o=!!r.skipEncoding,i=o?n:encodeURIComponent(n),a=function(e){return e&&"object"===(void 0===e?"undefined":(0,h.default)(e))}(r)&&!Array.isArray(r);return t[i]=u(a?r:{value:r},o),t},{});return m.default.stringify(t,{encode:!1,indices:!1})||""}function l(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e.url,r=void 0===t?"":t,o=e.query,i=e.form;if(i){var l=(0,f.default)(i).some(function(e){return a(i[e].value)}),p=e.headers["content-type"]||e.headers["Content-Type"];if(l||/multipart\/form-data/i.test(p)){var d=n(30);e.body=new d,(0,f.default)(i).forEach(function(t){e.body.append(t,u(i[t],!0))})}else e.body=s(i);delete e.form}if(o){var h=r.split("?"),v=(0,c.default)(h,2),g=v[0],y=v[1],b="";if(y){var _=m.default.parse(y);(0,f.default)(o).forEach(function(e){return delete _[e]}),b=m.default.stringify(_,{encode:!0})}var w=function(){for(var e=arguments.length,t=Array(e),n=0;n1&&void 0!==arguments[1]?arguments[1]:{};return d.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:if("object"===(void 0===t?"undefined":(0,h.default)(t))&&(t=(a=t).url),a.headers=a.headers||{},b.mergeInQueryOrForm(a),!a.requestInterceptor){e.next=10;break}return e.next=6,a.requestInterceptor(a);case 6:if(e.t0=e.sent,e.t0){e.next=9;break}e.t0=a;case 9:a=e.t0;case 10:return n=a.headers["content-type"]||a.headers["Content-Type"],/multipart\/form-data/i.test(n)&&(delete a.headers["content-type"],delete a.headers["Content-Type"]),r=void 0,e.prev=13,e.next=16,(a.userFetch||fetch)(a.url,a);case 16:return r=e.sent,e.next=19,b.serializeRes(r,t,a);case 19:if(r=e.sent,!a.responseInterceptor){e.next=27;break}return e.next=23,a.responseInterceptor(r);case 23:if(e.t1=e.sent,e.t1){e.next=26;break}e.t1=r;case 26:r=e.t1;case 27:e.next=37;break;case 29:if(e.prev=29,e.t2=e.catch(13),r){e.next=33;break}throw e.t2;case 33:throw(o=new Error(r.statusText)).statusCode=o.status=r.status,o.responseError=e.t2,o;case 37:if(r.ok){e.next=42;break}throw(i=new Error(r.statusText)).statusCode=i.status=r.status,i.response=r,i;case 42:return e.abrupt("return",r);case 43:case"end":return e.stop()}},e,this,[[13,29]])}));return function(t){return e.apply(this,arguments)}}();var _=t.shouldDownloadAsText=function(){return/(json|xml|yaml|text)\b/.test(arguments.length>0&&void 0!==arguments[0]?arguments[0]:"")}},function(e,t){e.exports=n(41)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){return Array.isArray(e)?e.length<1?"":"/"+e.map(function(e){return(e+"").replace(/~/g,"~0").replace(/\//g,"~1")}).join("/"):e}function i(e,t,n){return{op:"replace",path:e,value:t,meta:n}}function a(e,t,n){return f(c(e.filter(m).map(function(e){return t(e.value,n,e.path)})||[]))}function u(e,t,n){return n=n||[],Array.isArray(e)?e.map(function(e,r){return u(e,t,n.concat(r))}):p(e)?(0,w.default)(e).map(function(r){return u(e[r],t,n.concat(r))}):t(e,n[n.length-1],n)}function s(e,t,n){var r=[];if((n=n||[]).length>0){var o=t(e,n[n.length-1],n);o&&(r=r.concat(o))}if(Array.isArray(e)){var i=e.map(function(e,r){return s(e,t,n.concat(r))});i&&(r=r.concat(i))}else if(p(e)){var a=(0,w.default)(e).map(function(r){return s(e[r],t,n.concat(r))});a&&(r=r.concat(a))}return c(r)}function l(e){return Array.isArray(e)?e:[e]}function c(e){var t;return(t=[]).concat.apply(t,(0,_.default)(e.map(function(e){return Array.isArray(e)?c(e):e})))}function f(e){return e.filter(function(e){return void 0!==e})}function p(e){return e&&"object"===(void 0===e?"undefined":(0,b.default)(e))}function d(e){return e&&"function"==typeof e}function h(e){if(g(e)){var t=e.op;return"add"===t||"remove"===t||"replace"===t}return!1}function v(e){return h(e)||g(e)&&"mutation"===e.type}function m(e){return v(e)&&("add"===e.op||"replace"===e.op||"merge"===e.op||"mergeDeep"===e.op)}function g(e){return e&&"object"===(void 0===e?"undefined":(0,b.default)(e))}function y(e,t){try{return S.default.getValueByPointer(e,t)}catch(e){return console.error(e),{}}}Object.defineProperty(t,"__esModule",{value:!0});var b=r(n(1)),_=r(n(34)),w=r(n(0)),E=r(n(35)),x=r(n(2)),S=r(n(36)),C=r(n(4)),k=r(n(37)),A=r(n(38));t.default={add:function(e,t){return{op:"add",path:e,value:t}},replace:i,remove:function(e,t){return{op:"remove",path:e}},merge:function(e,t){return{type:"mutation",op:"merge",path:e,value:t}},mergeDeep:function(e,t){return{type:"mutation",op:"mergeDeep",path:e,value:t}},context:function(e,t){return{type:"context",path:e,value:t}},getIn:function(e,t){return t.reduce(function(e,t){return void 0!==t&&e?e[t]:e},e)},applyPatch:function(e,t,n){if(n=n||{},"merge"===(t=(0,x.default)({},t,{path:t.path&&o(t.path)})).op){var r=y(e,t.path);(0,x.default)(r,t.value),S.default.applyPatch(e,[i(t.path,r)])}else if("mergeDeep"===t.op){var a=y(e,t.path);for(var u in t.value){var s=t.value[u],l=Array.isArray(s);if(l){var c=a[u]||[];a[u]=c.concat(s)}else if(p(s)&&!l){var f=(0,x.default)({},a[u]);for(var d in s){if(Object.prototype.hasOwnProperty.call(f,d)){f=(0,k.default)((0,A.default)({},f),s);break}(0,x.default)(f,(0,E.default)({},d,s[d]))}a[u]=f}else a[u]=s}}else if("add"===t.op&&""===t.path&&p(t.value)){var h=(0,w.default)(t.value).reduce(function(e,n){return e.push({op:"add",path:"/"+o(n),value:t.value[n]}),e},[]);S.default.applyPatch(e,h)}else if("replace"===t.op&&""===t.path){var v=t.value;n.allowMetaPatches&&t.meta&&m(t)&&(Array.isArray(t.value)||p(t.value))&&(v=(0,x.default)({},v,t.meta)),e=v}else if(S.default.applyPatch(e,[t]),n.allowMetaPatches&&t.meta&&m(t)&&(Array.isArray(t.value)||p(t.value))){var g=y(e,t.path),b=(0,x.default)({},g,t.meta);S.default.applyPatch(e,[i(t.path,b)])}return e},parentPathMatch:function(e,t){if(!Array.isArray(t))return!1;for(var n=0,r=t.length;n1&&void 0!==arguments[1]?arguments[1]:{},n=t.requestInterceptor,r=t.responseInterceptor,o=e.withCredentials?"include":"same-origin";return function(t){return e({url:t,loadSpec:!0,requestInterceptor:n,responseInterceptor:r,headers:{Accept:"application/json"},credentials:o}).then(function(e){return e.body})}}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(4)),a=r(n(11));t.makeFetchJSON=o,t.clearCache=function(){s.plugins.refs.clearCache()},t.default=function(e){function t(e){var t=this;E&&(s.plugins.refs.docCache[E]=e),s.plugins.refs.fetchJSON=o(w,{requestInterceptor:y,responseInterceptor:b});var n=[s.plugins.refs];return"function"==typeof g&&n.push(s.plugins.parameters),"function"==typeof m&&n.push(s.plugins.properties),"strict"!==p&&n.push(s.plugins.allOf),(0,l.default)({spec:e,context:{baseDoc:E},plugins:n,allowMetaPatches:h,pathDiscriminator:v,parameterMacro:g,modelPropertyMacro:m}).then(_?function(){var e=(0,a.default)(i.default.mark(function e(n){return i.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:return e.abrupt("return",n);case 1:case"end":return e.stop()}},e,t)}));return function(t){return e.apply(this,arguments)}}():c.normalizeSwagger)}var n=e.fetch,r=e.spec,f=e.url,p=e.mode,d=e.allowMetaPatches,h=void 0===d||d,v=e.pathDiscriminator,m=e.modelPropertyMacro,g=e.parameterMacro,y=e.requestInterceptor,b=e.responseInterceptor,_=e.skipNormalization,w=e.http,E=e.baseDoc;return E=E||f,w=n||w||u.default,r?t(r):o(w,{requestInterceptor:y,responseInterceptor:b})(E).then(t)};var u=r(n(7)),s=n(31),l=r(s),c=n(5)},function(e,t){e.exports=n(204)},function(e,t){e.exports=n(91)},function(e,t){e.exports=n(2)},function(e,t){e.exports=n(3)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){function n(){Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack;for(var e=arguments.length,n=Array(e),r=0;r-1&&-1===o.indexOf(n)||i.indexOf(u)>-1||a.some(function(e){return u.indexOf(e)>-1})};var r=["properties"],o=["properties"],i=["definitions","parameters","responses","securityDefinitions","components/schemas","components/responses","components/parameters","components/securitySchemes"],a=["schema/example","items/example"]},function(e,t,n){e.exports=n(24)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=this,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};if("string"==typeof e?n.url=e:n=e,!(this instanceof o))return new o(n);(0,a.default)(this,n);var r=this.resolve().then(function(){return t.disableInterfaces||(0,a.default)(t,o.makeApisTagOperation(t)),t});return r.client=this,r}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(3)),a=r((r(n(25)),n(6))),u=r(n(14)),s=r(n(10)),l=n(7),c=r(l),f=n(16),p=r(f),d=r(n(48)),h=n(49),v=n(51),m=n(5);o.http=c.default,o.makeHttp=l.makeHttp.bind(null,o.http),o.resolve=p.default,o.resolveSubtree=d.default,o.execute=v.execute,o.serializeRes=l.serializeRes,o.serializeHeaders=l.serializeHeaders,o.clearCache=f.clearCache,o.parameterBuilders=v.PARAMETER_BUILDERS,o.makeApisTagOperation=h.makeApisTagOperation,o.buildRequest=v.buildRequest,o.helpers={opId:m.opId},o.prototype={http:c.default,execute:function(e){return this.applyDefaults(),o.execute((0,i.default)({spec:this.spec,http:this.http,securities:{authorized:this.authorizations},contextUrl:"string"==typeof this.url?this.url:void 0},e))},resolve:function(){var e=this;return o.resolve({spec:this.spec,url:this.url,allowMetaPatches:this.allowMetaPatches,requestInterceptor:this.requestInterceptor||null,responseInterceptor:this.responseInterceptor||null}).then(function(t){return e.originalSpec=e.spec,e.spec=t.spec,e.errors=t.errors,e})}},o.prototype.applyDefaults=function(){var e=this.spec,t=this.url;if(t&&(0,u.default)(t,"http")){var n=s.default.parse(t);e.host||(e.host=n.host),e.schemes||(e.schemes=[n.protocol.replace(":","")]),e.basePath||(e.basePath="/")}},t.default=o,e.exports=t.default},function(e,t){e.exports=n(905)},function(e,t){e.exports=n(17)},function(e,t){e.exports=n(906)},function(e,t){e.exports=n(907)},function(e,t){e.exports=n(342)},function(e,t){e.exports=n(910)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0}),t.plugins=t.SpecMap=void 0;var o=r(n(8)),i=r(n(1)),a=r(n(17)),u=r(n(4)),s=r(n(0)),l=r(n(18)),c=r(n(32)),f=r(n(2)),p=r(n(19)),d=r(n(20));t.default=function(e){return new w(e).dispatch()};var h=r(n(33)),v=r(n(9)),m=r(n(39)),g=r(n(43)),y=r(n(44)),b=r(n(45)),_=r(n(46)),w=function(){function e(t){(0,p.default)(this,e),(0,f.default)(this,{spec:"",debugLevel:"info",plugins:[],pluginHistory:{},errors:[],mutations:[],promisedPatches:[],state:{},patches:[],context:{},contextTree:new _.default,showDebug:!1,allPatches:[],pluginProp:"specMap",libMethods:(0,f.default)((0,c.default)(this),v.default),allowMetaPatches:!1},t),this.get=this._get.bind(this),this.getContext=this._getContext.bind(this),this.hasRun=this._hasRun.bind(this),this.wrappedPlugins=this.plugins.map(this.wrapPlugin.bind(this)).filter(v.default.isFunction),this.patches.push(v.default.add([],this.spec)),this.patches.push(v.default.context([],this.context)),this.updatePatches(this.patches)}return(0,d.default)(e,[{key:"debug",value:function(e){if(this.debugLevel===e){for(var t,n=arguments.length,r=Array(n>1?n-1:0),o=1;o1?n-1:0),o=1;o0})}},{key:"nextPromisedPatch",value:function(){if(this.promisedPatches.length>0)return a.default.race(this.promisedPatches.map(function(e){return e.value}))}},{key:"getPluginHistory",value:function(e){var t=this.getPluginName(e);return this.pluginHistory[t]||[]}},{key:"getPluginRunCount",value:function(e){return this.getPluginHistory(e).length}},{key:"getPluginHistoryTip",value:function(e){var t=this.getPluginHistory(e);return t&&t[t.length-1]||{}}},{key:"getPluginMutationIndex",value:function(e){var t=this.getPluginHistoryTip(e).mutationIndex;return"number"!=typeof t?-1:t}},{key:"getPluginName",value:function(e){return e.pluginName}},{key:"updatePluginHistory",value:function(e,t){var n=this.getPluginName(e);(this.pluginHistory[n]=this.pluginHistory[n]||[]).push(t)}},{key:"updatePatches",value:function(e,t){var n=this;v.default.normalizeArray(e).forEach(function(e){if(e instanceof Error)n.errors.push(e);else try{if(!v.default.isObject(e))return void n.debug("updatePatches","Got a non-object patch",e);if(n.showDebug&&n.allPatches.push(e),v.default.isPromise(e.value))return n.promisedPatches.push(e),void n.promisedPatchThen(e);if(v.default.isContextPatch(e))return void n.setContext(e.path,e.value);if(v.default.isMutation(e))return void n.updateMutations(e)}catch(e){console.error(e),n.errors.push(e)}})}},{key:"updateMutations",value:function(e){"object"===(0,i.default)(e.value)&&!Array.isArray(e.value)&&this.allowMetaPatches&&(e.value=(0,f.default)({},e.value));var t=v.default.applyPatch(this.state,e,{allowMetaPatches:this.allowMetaPatches});t&&(this.mutations.push(e),this.state=t)}},{key:"removePromisedPatch",value:function(e){var t=this.promisedPatches.indexOf(e);t<0?this.debug("Tried to remove a promisedPatch that isn't there!"):this.promisedPatches.splice(t,1)}},{key:"promisedPatchThen",value:function(e){var t=this;return e.value=e.value.then(function(n){var r=(0,f.default)({},e,{value:n});t.removePromisedPatch(e),t.updatePatches(r)}).catch(function(n){t.removePromisedPatch(e),t.updatePatches(n)})}},{key:"getMutations",value:function(e,t){return e=e||0,"number"!=typeof t&&(t=this.mutations.length),this.mutations.slice(e,t)}},{key:"getCurrentMutations",value:function(){return this.getMutationsForPlugin(this.getCurrentPlugin())}},{key:"getMutationsForPlugin",value:function(e){var t=this.getPluginMutationIndex(e);return this.getMutations(t+1)}},{key:"getCurrentPlugin",value:function(){return this.currentPlugin}},{key:"getPatchesOfType",value:function(e,t){return e.filter(t)}},{key:"getLib",value:function(){return this.libMethods}},{key:"_get",value:function(e){return v.default.getIn(this.state,e)}},{key:"_getContext",value:function(e){return this.contextTree.get(e)}},{key:"setContext",value:function(e,t){return this.contextTree.set(e,t)}},{key:"_hasRun",value:function(e){return this.getPluginRunCount(this.getCurrentPlugin())>(e||0)}},{key:"_clone",value:function(e){return JSON.parse((0,o.default)(e))}},{key:"dispatch",value:function(){function e(e){e&&(e=v.default.fullyNormalizeArray(e),n.updatePatches(e,r))}var t=this,n=this,r=this.nextPlugin();if(!r){var o=this.nextPromisedPatch();if(o)return o.then(function(){return t.dispatch()}).catch(function(){return t.dispatch()});var i={spec:this.state,errors:this.errors};return this.showDebug&&(i.patches=this.allPatches),a.default.resolve(i)}if(n.pluginCount=n.pluginCount||{},n.pluginCount[r]=(n.pluginCount[r]||0)+1,n.pluginCount[r]>100)return a.default.resolve({spec:n.state,errors:n.errors.concat(new Error("We've reached a hard limit of 100 plugin runs"))});if(r!==this.currentPlugin&&this.promisedPatches.length){var u=this.promisedPatches.map(function(e){return e.value});return a.default.all(u.map(function(e){return e.then(Function,Function)})).then(function(){return t.dispatch()})}return function(){n.currentPlugin=r;var t=n.getCurrentMutations(),o=n.mutations.length-1;try{if(r.isGenerator){var i=!0,a=!1,u=void 0;try{for(var s,p=(0,l.default)(r(t,n.getLib()));!(i=(s=p.next()).done);i=!0)e(s.value)}catch(e){a=!0,u=e}finally{try{!i&&p.return&&p.return()}finally{if(a)throw u}}}else e(r(t,n.getLib()))}catch(t){console.error(t),e([(0,f.default)((0,c.default)(t),{plugin:r})])}finally{n.updatePluginHistory(r,{mutationIndex:o})}return n.dispatch()}()}}]),e}(),E={refs:m.default,allOf:g.default,parameters:y.default,properties:b.default};t.SpecMap=w,t.plugins=E},function(e,t){e.exports=n(349)},function(e,t){e.exports=n(288)},function(e,t){e.exports=n(83)},function(e,t){e.exports=n(22)},function(e,t){e.exports=n(911)},function(e,t){e.exports=n(179)},function(e,t){e.exports=n(181)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e,t){if(!A.test(e)){if(!t)throw new O("Tried to resolve a relative URL, without having a basePath. path: '"+e+"' basePath: '"+t+"'");return x.default.resolve(t,e)}return e}function i(e,t){var n;return n=e&&e.response&&e.response.body?e.response.body.code+" "+e.response.body.message:e.message,new O("Could not resolve reference: "+n,t,e)}function a(e){return(e+"").split("#")}function u(e,t){var n=P[e];if(n&&!S.default.isPromise(n))try{var r=l(t,n);return(0,b.default)(g.default.resolve(r),{__value:r})}catch(e){return g.default.reject(e)}return s(e).then(function(e){return l(t,e)})}function s(e){var t=P[e];return t?S.default.isPromise(t)?t:g.default.resolve(t):(P[e]=I.fetchJSON(e).then(function(t){return P[e]=t,t}),P[e])}function l(e,t){var n=c(e);if(n.length<1)return t;var r=S.default.getIn(t,n);if(void 0===r)throw new O("Could not resolve pointer: "+e+" does not exist in document",{pointer:e});return r}function c(e){if("string"!=typeof e)throw new TypeError("Expected a string, got a "+(void 0===e?"undefined":(0,v.default)(e)));return"/"===e[0]&&(e=e.substr(1)),""===e?[]:e.split("/").map(f)}function f(e){return"string"!=typeof e?e:E.default.unescape(e.replace(/~1/g,"/").replace(/~0/g,"~"))}function p(e){return E.default.escape(e.replace(/~/g,"~0").replace(/\//g,"~1"))}function d(e,t){if(j(t))return!0;var n=e.charAt(t.length),r=t.slice(-1);return 0===e.indexOf(t)&&(!n||"/"===n||"#"===n)&&"#"!==r}function h(e,t,n,r){var o=T.get(r);o||(o={},T.set(r,o));var i=function(e){return 0===e.length?"":"/"+e.map(p).join("/")}(n),a=(t||"")+"#"+e;if(t==r.contextTree.get([]).baseDoc&&d(i,e))return!0;var u="";if(n.some(function(e){return u=u+"/"+p(e),o[u]&&o[u].some(function(e){return d(e,a)||d(a,e)})}))return!0;o[i]=(o[i]||[]).concat(a)}Object.defineProperty(t,"__esModule",{value:!0});var v=r(n(1)),m=r(n(0)),g=r(n(17)),y=r(n(40)),b=r(n(2)),_=n(41),w=r(n(15)),E=r(n(42)),x=r(n(10)),S=r(n(9)),C=r(n(21)),k=n(22),A=new RegExp("^([a-z]+://|//)","i"),O=(0,C.default)("JSONRefError",function(e,t,n){this.originalError=n,(0,b.default)(this,t||{})}),P={},T=new y.default,M={key:"$ref",plugin:function(e,t,n,r){var s=n.slice(0,-1);if(!(0,k.isFreelyNamed)(s)){var l=r.getContext(n).baseDoc;if("string"!=typeof e)return new O("$ref: must be a string (JSON-Ref)",{$ref:e,baseDoc:l,fullPath:n});var f=a(e),p=f[0],d=f[1]||"",v=void 0;try{v=l||p?o(p,l):null}catch(t){return i(t,{pointer:d,$ref:e,basePath:v,fullPath:n})}var g=void 0,y=void 0;if(!h(d,v,s,r)){if(null==v?(y=c(d),void 0===(g=r.get(y))&&(g=new O("Could not resolve reference: "+e,{pointer:d,$ref:e,baseDoc:l,fullPath:n}))):g=null!=(g=u(v,d)).__value?g.__value:g.catch(function(t){throw i(t,{pointer:d,$ref:e,baseDoc:l,fullPath:n})}),g instanceof Error)return[S.default.remove(n),g];var b=S.default.replace(s,g,{$$ref:e});if(v&&v!==l)return[b,S.default.context(s,{baseDoc:v})];try{if(!function(e,t){var n=[e];return t.path.reduce(function(e,t){return n.push(e[t]),e[t]},e),function e(t){return S.default.isObject(t)&&(n.indexOf(t)>=0||(0,m.default)(t).some(function(n){return e(t[n])}))}(t.value)}(r.state,b))return b}catch(e){return null}}}}},I=(0,b.default)(M,{docCache:P,absoluteify:o,clearCache:function(e){void 0!==e?delete P[e]:(0,m.default)(P).forEach(function(e){delete P[e]})},JSONRefError:O,wrapError:i,getDoc:s,split:a,extractFromDoc:u,fetchJSON:function(e){return(0,_.fetch)(e,{headers:{Accept:"application/json, application/yaml"},loadSpec:!0}).then(function(e){return e.text()}).then(function(e){return w.default.safeLoad(e)})},extract:l,jsonPointerToArray:c,unescapeJsonPointerToken:f});t.default=I;var j=function(e){return!e||"/"===e||"#"===e};e.exports=t.default},function(e,t){e.exports=n(914)},function(e,t){e.exports=n(925)},function(e,t){e.exports=n(926)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=function(e){return e&&e.__esModule?e:{default:e}}(n(2)),o=n(22);t.default={key:"allOf",plugin:function(e,t,n,i,a){if(!a.meta||!a.meta.$$ref){var u=n.slice(0,-1);if(!(0,o.isFreelyNamed)(u)){if(!Array.isArray(e)){var s=new TypeError("allOf must be an array");return s.fullPath=n,s}var l=!1,c=a.value;u.forEach(function(e){c&&(c=c[e])}),delete(c=(0,r.default)({},c)).allOf;var f=[i.replace(u,{})].concat(e.map(function(e,t){if(!i.isObject(e)){if(l)return null;l=!0;var r=new TypeError("Elements in allOf must be objects");return r.fullPath=n,r}return i.mergeDeep(u,e)}));return f.push(i.mergeDeep(u,c)),c.$$ref||f.push(i.remove([].concat(u,"$$ref"))),f}}}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=r(n(2)),i=r(n(9));t.default={key:"parameters",plugin:function(e,t,n,r,a){if(Array.isArray(e)&&e.length){var u=(0,o.default)([],e),s=n.slice(0,-1),l=(0,o.default)({},i.default.getIn(r.spec,s));return e.forEach(function(e,t){try{u[t].default=r.parameterMacro(l,e)}catch(e){var o=new Error(e);return o.fullPath=n,o}}),i.default.replace(n,u)}return i.default.replace(n,e)}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=r(n(2)),i=r(n(9));t.default={key:"properties",plugin:function(e,t,n,r){var a=(0,o.default)({},e);for(var u in e)try{a[u].default=r.modelPropertyMacro(a[u])}catch(e){var s=new Error(e);return s.fullPath=n,s}return i.default.replace(n,a)}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e,t){return i({children:{}},e,t)}function i(e,t,n){return e.value=t||{},e.protoValue=n?(0,u.default)({},n.protoValue,e.value):e.value,(0,a.default)(e.children).forEach(function(t){var n=e.children[t];e.children[t]=i(n,n.value,e)}),e}Object.defineProperty(t,"__esModule",{value:!0});var a=r(n(0)),u=r(n(3)),s=r(n(19)),l=r(n(20)),c=function(){function e(t){(0,s.default)(this,e),this.root=o(t||{})}return(0,l.default)(e,[{key:"set",value:function(e,t){var n=this.getParent(e,!0);if(n){var r=e[e.length-1],a=n.children;a[r]?i(a[r],t,n):a[r]=o(t,n)}else i(this.root,t,null)}},{key:"get",value:function(e){if((e=e||[]).length<1)return this.root.value;for(var t=this.root,n=void 0,r=void 0,o=0;o2&&void 0!==arguments[2]?arguments[2]:{};return o.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:return r=y.returnEntireTree,a=y.baseDoc,c=y.requestInterceptor,f=y.responseInterceptor,p=y.parameterMacro,d=y.modelPropertyMacro,h={pathDiscriminator:n,baseDoc:a,requestInterceptor:c,responseInterceptor:f,parameterMacro:p,modelPropertyMacro:d},v=(0,l.normalizeSwagger)({spec:t}),m=v.spec,e.next=5,(0,s.default)((0,i.default)({},h,{spec:m,allowMetaPatches:!0,skipNormalization:!0}));case 5:return g=e.sent,!r&&Array.isArray(n)&&n.length&&(g.spec=(0,u.default)(g.spec,n)||null),e.abrupt("return",g);case 8:case"end":return e.stop()}},e,this)}));return function(t,n){return e.apply(this,arguments)}}(),e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return function(t){var n=t.pathName,r=t.method,o=t.operationId;return function(t){var i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return e.execute((0,a.default)({spec:e.spec},(0,u.default)(e,"requestInterceptor","responseInterceptor","userFetch"),{pathName:n,method:r,parameters:t,operationId:o},i))}}}function i(e){var t=e.spec,n=e.cb,r=void 0===n?l:n,o=e.defaultTag,i=void 0===o?"default":o,a=e.v2OperationIdCompatibilityMode,u={},f={};return(0,s.eachOperation)(t,function(e){var n=e.pathName,o=e.method,l=e.operation;(l.tags?c(l.tags):[i]).forEach(function(e){if("string"==typeof e){var i=f[e]=f[e]||{},c=(0,s.opId)(l,n,o,{v2OperationIdCompatibilityMode:a}),p=r({spec:t,pathName:n,method:o,operation:l,operationId:c});if(u[c])u[c]++,i[""+c+u[c]]=p;else if(void 0!==i[c]){var d=u[c]||1;u[c]=d+1,i[""+c+u[c]]=p;var h=i[c];delete i[c],i[""+c+d]=h}else i[c]=p}})}),f}Object.defineProperty(t,"__esModule",{value:!0}),t.self=void 0;var a=r(n(3));t.makeExecute=o,t.makeApisTagOperationsOperationExecute=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=f.makeExecute(e),n=f.mapTagOperations({v2OperationIdCompatibilityMode:e.v2OperationIdCompatibilityMode,spec:e.spec,cb:t}),r={};for(var o in n)for(var i in r[o]={operations:{}},n[o])r[o].operations[i]={execute:n[o][i]};return{apis:r}},t.makeApisTagOperation=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=f.makeExecute(e);return{apis:f.mapTagOperations({v2OperationIdCompatibilityMode:e.v2OperationIdCompatibilityMode,spec:e.spec,cb:t})}},t.mapTagOperations=i;var u=r(n(50)),s=n(5),l=function(){return null},c=function(e){return Array.isArray(e)?e:[e]},f=t.self={mapTagOperations:i,makeExecute:o}},function(e,t){e.exports=n(927)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=e.spec,n=e.operationId,r=(e.securities,e.requestContentType,e.responseContentType),o=e.scheme,a=e.requestInterceptor,s=e.responseInterceptor,c=e.contextUrl,f=e.userFetch,p=(e.requestBody,e.server),d=e.serverVariables,h=e.http,g=e.parameters,y=e.parameterBuilders,O=(0,x.isOAS3)(t);y||(y=O?_.default:b.default);var P={url:"",credentials:h&&h.withCredentials?"include":"same-origin",headers:{},cookies:{}};a&&(P.requestInterceptor=a),s&&(P.responseInterceptor=s),f&&(P.userFetch=f);var T=(0,x.getOperationRaw)(t,n);if(!T)throw new C("Operation "+n+" not found");var M=T.operation,I=void 0===M?{}:M,j=T.method,N=T.pathName;if(P.url+=i({spec:t,scheme:o,contextUrl:c,server:p,serverVariables:d,pathName:N,method:j}),!n)return delete P.cookies,P;P.url+=N,P.method=(""+j).toUpperCase(),g=g||{};var R=t.paths[N]||{};r&&(P.headers.accept=r);var D=A([].concat(S(I.parameters)).concat(S(R.parameters)));D.forEach(function(e){var n=y[e.in],r=void 0;if("body"===e.in&&e.schema&&e.schema.properties&&(r=g),void 0===(r=e&&e.name&&g[e.name])?r=e&&e.name&&g[e.in+"."+e.name]:k(e.name,D).length>1&&console.warn("Parameter '"+e.name+"' is ambiguous because the defined spec has more than one parameter with the name: '"+e.name+"' and the passed-in parameter values did not define an 'in' value."),null!==r){if(void 0!==e.default&&void 0===r&&(r=e.default),void 0===r&&e.required&&!e.allowEmptyValue)throw new Error("Required parameter "+e.name+" is not provided");if(O&&e.schema&&"object"===e.schema.type&&"string"==typeof r)try{r=JSON.parse(r)}catch(e){throw new Error("Could not parse object parameter value string as JSON")}n&&n({req:P,parameter:e,value:r,operation:I,spec:t})}});var L=(0,u.default)({},e,{operation:I});if((P=O?(0,w.default)(L,P):(0,E.default)(L,P)).cookies&&(0,l.default)(P.cookies).length){var U=(0,l.default)(P.cookies).reduce(function(e,t){var n=P.cookies[t];return e+(e?"&":"")+v.default.serialize(t,n)},"");P.headers.Cookie=U}return P.cookies&&delete P.cookies,(0,m.mergeInQueryOrForm)(P),P}function i(e){return(0,x.isOAS3)(e.spec)?function(e){var t=e.spec,n=e.pathName,r=e.method,o=e.server,i=e.contextUrl,a=e.serverVariables,u=void 0===a?{}:a,s=(0,f.default)(t,["paths",n,(r||"").toLowerCase(),"servers"])||(0,f.default)(t,["paths",n,"servers"])||(0,f.default)(t,["servers"]),l="",c=null;if(o&&s&&s.length){var p=s.map(function(e){return e.url});p.indexOf(o)>-1&&(l=o,c=s[p.indexOf(o)])}!l&&s&&s.length&&(l=s[0].url,c=s[0]),l.indexOf("{")>-1&&function(e){for(var t=[],n=/{([^}]+)}/g,r=void 0;r=n.exec(e);)t.push(r[1]);return t}(l).forEach(function(e){if(c.variables&&c.variables[e]){var t=c.variables[e],n=u[e]||t.default,r=new RegExp("{"+e+"}","g");l=l.replace(r,n)}});return function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"",n=h.default.parse(e),r=h.default.parse(t),o=P(n.protocol)||P(r.protocol)||"",i=n.host||r.host,a=n.pathname||"",u=void 0;return"/"===(u=o&&i?o+"://"+(i+a):a)[u.length-1]?u.slice(0,-1):u}(l,i)}(e):function(e){var t=e.spec,n=e.scheme,r=e.contextUrl,o=void 0===r?"":r,i=h.default.parse(o),a=Array.isArray(t.schemes)?t.schemes[0]:null,u=n||a||P(i.protocol)||"http",s=t.host||i.host||"",l=t.basePath||"",c=void 0;return"/"===(c=u&&s?u+"://"+(s+l):l)[c.length-1]?c.slice(0,-1):c}(e)}Object.defineProperty(t,"__esModule",{value:!0}),t.self=void 0;var a=r(n(8)),u=r(n(3)),s=r(n(52)),l=r(n(0)),c=r(n(2));t.execute=function(e){var t=e.http,n=e.fetch,r=e.spec,o=e.operationId,i=e.pathName,l=e.method,c=e.parameters,f=e.securities,h=(0,s.default)(e,["http","fetch","spec","operationId","pathName","method","parameters","securities"]),v=t||n||g.default;i&&l&&!o&&(o=(0,x.legacyIdFromPathMethod)(i,l));var m=O.buildRequest((0,u.default)({spec:r,operationId:o,parameters:c,securities:f,http:v},h));return m.body&&((0,p.default)(m.body)||(0,d.default)(m.body))&&(m.body=(0,a.default)(m.body)),v(m)},t.buildRequest=o,t.baseUrl=i;var f=r((r(n(6)),n(12))),p=r(n(53)),d=r(n(54)),h=r((r(n(13)),n(10))),v=r(n(55)),m=n(7),g=r(m),y=r(n(21)),b=r(n(56)),_=r(n(57)),w=r(n(62)),E=r(n(64)),x=n(5),S=function(e){return Array.isArray(e)?e:[]},C=(0,y.default)("OperationNotFoundError",function(e,t,n){this.originalError=n,(0,c.default)(this,t||{})}),k=function(e,t){return t.filter(function(t){return t.name===e})},A=function(e){var t={};e.forEach(function(e){t[e.in]||(t[e.in]={}),t[e.in][e.name]=e});var n=[];return(0,l.default)(t).forEach(function(e){(0,l.default)(t[e]).forEach(function(r){n.push(t[e][r])})}),n},O=t.self={buildRequest:o},P=function(e){return e?e.replace(/\W/g,""):null}},function(e,t){e.exports=n(84)},function(e,t){e.exports=n(228)},function(e,t){e.exports=n(24)},function(e,t){e.exports=n(930)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default={body:function(e){var t=e.req,n=e.value;t.body=n},header:function(e){var t=e.req,n=e.parameter,r=e.value;t.headers=t.headers||{},void 0!==r&&(t.headers[n.name]=r)},query:function(e){var t=e.req,n=e.value,r=e.parameter;if(t.query=t.query||{},!1===n&&"boolean"===r.type&&(n="false"),0===n&&["number","integer"].indexOf(r.type)>-1&&(n="0"),n)t.query[r.name]={collectionFormat:r.collectionFormat,value:n};else if(r.allowEmptyValue&&void 0!==n){var o=r.name;t.query[o]=t.query[o]||{},t.query[o].allowEmptyValue=!0}},path:function(e){var t=e.req,n=e.value,r=e.parameter;t.url=t.url.split("{"+r.name+"}").join(encodeURIComponent(n))},formData:function(e){var t=e.req,n=e.value,r=e.parameter;(n||r.allowEmptyValue)&&(t.form=t.form||{},t.form[r.name]={value:n,allowEmptyValue:r.allowEmptyValue,collectionFormat:r.collectionFormat})}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=r(n(0)),i=r(n(1)),a=r(n(58));t.default={path:function(e){var t=e.req,n=e.value,r=e.parameter,o=r.name,i=r.style,u=r.explode,s=(0,a.default)({key:r.name,value:n,style:i||"simple",explode:u||!1,escape:!0});t.url=t.url.split("{"+o+"}").join(s)},query:function(e){var t=e.req,n=e.value,r=e.parameter;if(t.query=t.query||{},!1===n&&(n="false"),0===n&&(n="0"),n){var u=void 0===n?"undefined":(0,i.default)(n);"deepObject"===r.style?(0,o.default)(n).forEach(function(e){var o=n[e];t.query[r.name+"["+e+"]"]={value:(0,a.default)({key:e,value:o,style:"deepObject",escape:r.allowReserved?"unsafe":"reserved"}),skipEncoding:!0}}):"object"!==u||Array.isArray(n)||"form"!==r.style&&r.style||!r.explode&&void 0!==r.explode?t.query[r.name]={value:(0,a.default)({key:r.name,value:n,style:r.style||"form",explode:void 0===r.explode||r.explode,escape:r.allowReserved?"unsafe":"reserved"}),skipEncoding:!0}:(0,o.default)(n).forEach(function(e){var o=n[e];t.query[e]={value:(0,a.default)({key:e,value:o,style:r.style||"form",escape:r.allowReserved?"unsafe":"reserved"}),skipEncoding:!0}})}else if(r.allowEmptyValue&&void 0!==n){var s=r.name;t.query[s]=t.query[s]||{},t.query[s].allowEmptyValue=!0}},header:function(e){var t=e.req,n=e.parameter,r=e.value;t.headers=t.headers||{},u.indexOf(n.name.toLowerCase())>-1||void 0!==r&&(t.headers[n.name]=(0,a.default)({key:n.name,value:r,style:n.style||"simple",explode:void 0!==n.explode&&n.explode,escape:!1}))},cookie:function(e){var t=e.req,n=e.parameter,r=e.value;t.headers=t.headers||{};var o=void 0===r?"undefined":(0,i.default)(r);if("undefined"!==o){var u="object"===o&&!Array.isArray(r)&&n.explode?"":n.name+"=";t.headers.Cookie=u+(0,a.default)({key:n.name,value:r,escape:!1,style:n.style||"form",explode:void 0!==n.explode&&n.explode})}}};var u=["accept","authorization","content-type"];e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=(arguments.length>1&&void 0!==arguments[1]?arguments[1]:{}).escape,n=arguments[2];return"number"==typeof e&&(e=e.toString()),"string"==typeof e&&e.length&&t?n?JSON.parse(e):(0,s.stringToCharArray)(e).map(function(e){return c(e)?e:l(e)&&"unsafe"===t?e:((0,u.default)(e)||[]).map(function(e){return("0"+e.toString(16).toUpperCase()).slice(-2)}).map(function(e){return"%"+e}).join("")}).join(""):e}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(0)),a=r(n(1));t.encodeDisallowedCharacters=o,t.default=function(e){var t=e.value;return Array.isArray(t)?function(e){var t=e.key,n=e.value,r=e.style,i=e.explode,a=e.escape,u=function(e){return o(e,{escape:a})};if("simple"===r)return n.map(function(e){return u(e)}).join(",");if("label"===r)return"."+n.map(function(e){return u(e)}).join(".");if("matrix"===r)return n.map(function(e){return u(e)}).reduce(function(e,n){return!e||i?(e||"")+";"+t+"="+n:e+","+n},"");if("form"===r){var s=i?"&"+t+"=":",";return n.map(function(e){return u(e)}).join(s)}if("spaceDelimited"===r){var l=i?t+"=":"";return n.map(function(e){return u(e)}).join(" "+l)}if("pipeDelimited"===r){var c=i?t+"=":"";return n.map(function(e){return u(e)}).join("|"+c)}}(e):"object"===(void 0===t?"undefined":(0,a.default)(t))?function(e){var t=e.key,n=e.value,r=e.style,a=e.explode,u=e.escape,s=function(e){return o(e,{escape:u})},l=(0,i.default)(n);return"simple"===r?l.reduce(function(e,t){var r=s(n[t]);return(e?e+",":"")+t+(a?"=":",")+r},""):"label"===r?l.reduce(function(e,t){var r=s(n[t]);return(e?e+".":".")+t+(a?"=":".")+r},""):"matrix"===r&&a?l.reduce(function(e,t){var r=s(n[t]);return(e?e+";":";")+t+"="+r},""):"matrix"===r?l.reduce(function(e,r){var o=s(n[r]);return(e?e+",":";"+t+"=")+r+","+o},""):"form"===r?l.reduce(function(e,t){var r=s(n[t]);return(e?e+(a?"&":","):"")+t+(a?"=":",")+r},""):void 0}(e):function(e){var t=e.key,n=e.value,r=e.style,i=e.escape,a=function(e){return o(e,{escape:i})};return"simple"===r?a(n):"label"===r?"."+a(n):"matrix"===r?";"+t+"="+a(n):"form"===r?a(n):"deepObject"===r?a(n):void 0}(e)};var u=r((r(n(59)),n(60))),s=n(61),l=function(e){return":/?#[]@!$&'()*+,;=".indexOf(e)>-1},c=function(e){return/^[a-z0-9\-._~]+$/i.test(e)}},function(e,t){e.exports=n(931)},function(e,t){e.exports=n(932)},function(e,t){e.exports=n(933)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=e.request,n=e.securities,r=void 0===n?{}:n,o=e.operation,i=void 0===o?{}:o,a=e.spec,f=(0,s.default)({},t),p=r.authorized,d=void 0===p?{}:p,h=i.security||a.security||[],v=d&&!!(0,u.default)(d).length,m=(0,l.default)(a,["components","securitySchemes"])||{};return f.headers=f.headers||{},f.query=f.query||{},(0,u.default)(r).length&&v&&h&&(!Array.isArray(i.security)||i.security.length)?(h.forEach(function(e,t){for(var n in e){var r=d[n],o=m[n];if(r){var i=r.value||r,a=o.type;if(r)if("apiKey"===a)"query"===o.in&&(f.query[o.name]=i),"header"===o.in&&(f.headers[o.name]=i),"cookie"===o.in&&(f.cookies[o.name]=i);else if("http"===a){if("basic"===o.scheme){var u=i.username,s=i.password,l=(0,c.default)(u+":"+s);f.headers.Authorization="Basic "+l}"bearer"===o.scheme&&(f.headers.Authorization="Bearer "+i)}else if("oauth2"===a){var p=r.token||{},h=p.access_token,v=p.token_type;v&&"bearer"!==v.toLowerCase()||(v="Bearer"),f.headers.Authorization=v+" "+h}}}}),f):t}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(8)),a=r(n(1)),u=r(n(0));t.default=function(e,t){var n=e.operation,r=e.requestBody,s=e.securities,l=e.spec,c=e.attachContentTypeForEmptyPayload,p=e.requestContentType;t=o({request:t,securities:s,operation:n,spec:l});var d=n.requestBody||{},h=(0,u.default)(d.content||{}),v=p&&h.indexOf(p)>-1;if(r||c){if(p&&v)t.headers["Content-Type"]=p;else if(!p){var m=h[0];m&&(t.headers["Content-Type"]=m,p=m)}}else p&&v&&(t.headers["Content-Type"]=p);return r&&(p?h.indexOf(p)>-1&&("application/x-www-form-urlencoded"===p||0===p.indexOf("multipart/")?"object"===(void 0===r?"undefined":(0,a.default)(r))?(t.form={},(0,u.default)(r).forEach(function(e){var n,o=r[e],u=void 0;"undefined"!=typeof File&&(u=o instanceof File),"undefined"!=typeof Blob&&(u=u||o instanceof Blob),void 0!==f.Buffer&&(u=u||f.Buffer.isBuffer(o)),n="object"!==(void 0===o?"undefined":(0,a.default)(o))||u?o:Array.isArray(o)?o.toString():(0,i.default)(o),t.form[e]={value:n}})):t.form=r:t.body=r):t.body=r),t},t.applySecurities=o;var s=r(n(6)),l=r(n(12)),c=r(n(13)),f=n(63)},function(e,t){e.exports=n(54)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=e.request,n=e.securities,r=void 0===n?{}:n,o=e.operation,s=void 0===o?{}:o,l=e.spec,c=(0,u.default)({},t),f=r.authorized,p=void 0===f?{}:f,d=r.specSecurity,h=void 0===d?[]:d,v=s.security||h,m=p&&!!(0,i.default)(p).length,g=l.securityDefinitions;return c.headers=c.headers||{},c.query=c.query||{},(0,i.default)(r).length&&m&&v&&(!Array.isArray(s.security)||s.security.length)?(v.forEach(function(e,t){for(var n in e){var r=p[n];if(r){var o=r.token,i=r.value||r,u=g[n],s=u.type,l=u["x-tokenName"]||"access_token",f=o&&o[l],d=o&&o.token_type;if(r)if("apiKey"===s){var h="query"===u.in?"query":"headers";c[h]=c[h]||{},c[h][u.name]=i}else"basic"===s?i.header?c.headers.authorization=i.header:(i.base64=(0,a.default)(i.username+":"+i.password),c.headers.authorization="Basic "+i.base64):"oauth2"===s&&f&&(d=d&&"bearer"!==d.toLowerCase()?d:"Bearer",c.headers.authorization=d+" "+f)}}}),c):t}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(0));t.default=function(e,t){var n=e.spec,r=e.operation,i=e.securities,a=e.requestContentType,u=e.attachContentTypeForEmptyPayload;if((t=o({request:t,securities:i,operation:r,spec:n})).body||t.form||u)a?t.headers["Content-Type"]=a:Array.isArray(r.consumes)?t.headers["Content-Type"]=r.consumes[0]:Array.isArray(n.consumes)?t.headers["Content-Type"]=n.consumes[0]:r.parameters&&r.parameters.filter(function(e){return"file"===e.type}).length?t.headers["Content-Type"]="multipart/form-data":r.parameters&&r.parameters.filter(function(e){return"formData"===e.in}).length&&(t.headers["Content-Type"]="application/x-www-form-urlencoded");else if(a){var s=r.parameters&&r.parameters.filter(function(e){return"body"===e.in}).length>0,l=r.parameters&&r.parameters.filter(function(e){return"formData"===e.in}).length>0;(s||l)&&(t.headers["Content-Type"]=a)}return t},t.applySecurities=o;var a=r(n(13)),u=r(n(6));r(n(7))}])},function(e,t,n){"use strict";var r=Object.prototype.hasOwnProperty,o=function(){for(var e=[],t=0;t<256;++t)e.push("%"+((t<16?"0":"")+t.toString(16)).toUpperCase());return e}();t.arrayToObject=function(e,t){for(var n=t&&t.plainObjects?Object.create(null):{},r=0;r=48&&i<=57||i>=65&&i<=90||i>=97&&i<=122?n+=t.charAt(r):i<128?n+=o[i]:i<2048?n+=o[192|i>>6]+o[128|63&i]:i<55296||i>=57344?n+=o[224|i>>12]+o[128|i>>6&63]+o[128|63&i]:(r+=1,i=65536+((1023&i)<<10|1023&t.charCodeAt(r)),n+=o[240|i>>18]+o[128|i>>12&63]+o[128|i>>6&63]+o[128|63&i])}return n},t.compact=function(e){for(var t=[{obj:{o:e},prop:"o"}],n=[],r=0;r=0;l--)if(f[l]!=p[l])return!1;for(l=f.length-1;l>=0;l--)if(c=f[l],!a(e[c],t[c],n))return!1;return typeof e==typeof t}(e,t,n))};function u(e){return null===e||void 0===e}function s(e){return!(!e||"object"!=typeof e||"number"!=typeof e.length)&&("function"==typeof e.copy&&"function"==typeof e.slice&&!(e.length>0&&"number"!=typeof e[0]))}},function(e,t,n){var r={strict:!0},o=n(390),i=function(e,t){return o(e,t,r)},a=n(231);t.JsonPatchError=a.PatchError,t.deepClone=a._deepClone;var u={add:function(e,t,n){return e[t]=this.value,{newDocument:n}},remove:function(e,t,n){var r=e[t];return delete e[t],{newDocument:n,removed:r}},replace:function(e,t,n){var r=e[t];return e[t]=this.value,{newDocument:n,removed:r}},move:function(e,t,n){var r=l(n,this.path);r&&(r=a._deepClone(r));var o=c(n,{op:"remove",path:this.from}).removed;return c(n,{op:"add",path:this.path,value:o}),{newDocument:n,removed:r}},copy:function(e,t,n){var r=l(n,this.from);return c(n,{op:"add",path:this.path,value:a._deepClone(r)}),{newDocument:n}},test:function(e,t,n){return{newDocument:n,test:i(e[t],this.value)}},_get:function(e,t,n){return this.value=e[t],{newDocument:n}}},s={add:function(e,t,n){return a.isInteger(t)?e.splice(t,0,this.value):e[t]=this.value,{newDocument:n,index:t}},remove:function(e,t,n){return{newDocument:n,removed:e.splice(t,1)[0]}},replace:function(e,t,n){var r=e[t];return e[t]=this.value,{newDocument:n,removed:r}},move:u.move,copy:u.copy,test:u.test,_get:u._get};function l(e,t){if(""==t)return e;var n={op:"_get",path:t};return c(e,n),n.value}function c(e,n,r,o){if(void 0===r&&(r=!1),void 0===o&&(o=!0),r&&("function"==typeof r?r(n,0,e,n.path):p(n,0)),""===n.path){var c={newDocument:e};if("add"===n.op)return c.newDocument=n.value,c;if("replace"===n.op)return c.newDocument=n.value,c.removed=e,c;if("move"===n.op||"copy"===n.op)return c.newDocument=l(e,n.from),"move"===n.op&&(c.removed=e),c;if("test"===n.op){if(c.test=i(e,n.value),!1===c.test)throw new t.JsonPatchError("Test operation failed","TEST_OPERATION_FAILED",0,n,e);return c.newDocument=e,c}if("remove"===n.op)return c.removed=e,c.newDocument=null,c;if("_get"===n.op)return n.value=e,c;if(r)throw new t.JsonPatchError("Operation `op` property is not one of operations defined in RFC-6902","OPERATION_OP_INVALID",0,n,e);return c}o||(e=a._deepClone(e));var f=(n.path||"").split("/"),d=e,h=1,v=f.length,m=void 0,g=void 0,y=void 0;for(y="function"==typeof r?r:p;;){if(g=f[h],r&&void 0===m&&(void 0===d[g]?m=f.slice(0,h).join("/"):h==v-1&&(m=n.path),void 0!==m&&y(n,0,e,m)),h++,Array.isArray(d)){if("-"===g)g=d.length;else{if(r&&!a.isInteger(g))throw new t.JsonPatchError("Expected an unsigned base-10 integer value, making the new referenced value the array element with the zero-based index","OPERATION_PATH_ILLEGAL_ARRAY_INDEX",0,n.path,n);a.isInteger(g)&&(g=~~g)}if(h>=v){if(r&&"add"===n.op&&g>d.length)throw new t.JsonPatchError("The specified index MUST NOT be greater than the number of elements in the array","OPERATION_VALUE_OUT_OF_BOUNDS",0,n.path,n);if(!1===(c=s[n.op].call(n,d,g,e)).test)throw new t.JsonPatchError("Test operation failed","TEST_OPERATION_FAILED",0,n,e);return c}}else if(g&&-1!=g.indexOf("~")&&(g=a.unescapePathComponent(g)),h>=v){if(!1===(c=u[n.op].call(n,d,g,e)).test)throw new t.JsonPatchError("Test operation failed","TEST_OPERATION_FAILED",0,n,e);return c}d=d[g]}}function f(e,n,r,o){if(void 0===o&&(o=!0),r&&!Array.isArray(n))throw new t.JsonPatchError("Patch sequence must be an array","SEQUENCE_NOT_AN_ARRAY");o||(e=a._deepClone(e));for(var i=new Array(n.length),u=0,s=n.length;u0)throw new t.JsonPatchError('Operation `path` property must start with "/"',"OPERATION_PATH_INVALID",n,e,r);if(("move"===e.op||"copy"===e.op)&&"string"!=typeof e.from)throw new t.JsonPatchError("Operation `from` property is not present (applicable in `move` and `copy` operations)","OPERATION_FROM_REQUIRED",n,e,r);if(("add"===e.op||"replace"===e.op||"test"===e.op)&&void 0===e.value)throw new t.JsonPatchError("Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)","OPERATION_VALUE_REQUIRED",n,e,r);if(("add"===e.op||"replace"===e.op||"test"===e.op)&&a.hasUndefined(e.value))throw new t.JsonPatchError("Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)","OPERATION_VALUE_CANNOT_CONTAIN_UNDEFINED",n,e,r);if(r)if("add"==e.op){var i=e.path.split("/").length,s=o.split("/").length;if(i!==s+1&&i!==s)throw new t.JsonPatchError("Cannot perform an `add` operation at the desired path","OPERATION_PATH_CANNOT_ADD",n,e,r)}else if("replace"===e.op||"remove"===e.op||"_get"===e.op){if(e.path!==o)throw new t.JsonPatchError("Cannot perform the operation at a path that does not exist","OPERATION_PATH_UNRESOLVABLE",n,e,r)}else if("move"===e.op||"copy"===e.op){var l=d([{op:"_get",path:e.from,value:void 0}],r);if(l&&"OPERATION_PATH_UNRESOLVABLE"===l.name)throw new t.JsonPatchError("Cannot perform the operation from a path that does not exist","OPERATION_FROM_UNRESOLVABLE",n,e,r)}}function d(e,n,r){try{if(!Array.isArray(e))throw new t.JsonPatchError("Patch sequence must be an array","SEQUENCE_NOT_AN_ARRAY");if(n)f(a._deepClone(n),a._deepClone(e),r||!0);else{r=r||p;for(var o=0;o1&&void 0!==arguments[1]?arguments[1]:(0,a.List)();return function(e){return(e.authSelectors.definitionsToAuthorize()||(0,a.List)()).filter(function(e){return t.some(function(t){return t.get(e.keySeq().first())})})}},t.authorized=(0,i.createSelector)(s,function(e){return e.get("authorized")||(0,a.Map)()}),t.isAuthorized=function(e,t){return function(e){var n=e.authSelectors.authorized();return a.List.isList(t)?!!t.toJS().filter(function(e){return-1===(0,r.default)(e).map(function(e){return!!n.get(e)}).indexOf(!1)}).length:null}},t.getConfigs=(0,i.createSelector)(s,function(e){return e.get("configs")})},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.execute=void 0;var r,o=n(25),i=(r=o)&&r.__esModule?r:{default:r};t.execute=function(e,t){var n=t.authSelectors,r=t.specSelectors;return function(t){var o=t.path,a=t.method,u=t.operation,s=t.extras,l={authorized:n.authorized()&&n.authorized().toJS(),definitions:r.securityDefinitions()&&r.securityDefinitions().toJS(),specSecurity:r.security()&&r.security().toJS()};return e((0,i.default)({path:o,method:a,operation:u,securities:l},s))}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{fn:{shallowEqualKeys:r.shallowEqualKeys}}};var r=n(9)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=s(n(41)),o=s(n(23));t.default=function(e){var t=e.fn,n={download:function(e){return function(n){var r=n.errActions,i=n.specSelectors,a=n.specActions,s=n.getConfigs,l=t.fetch,c=s();function f(t){if(t instanceof Error||t.status>=400)return a.updateLoadingStatus("failed"),r.newThrownErr((0,o.default)(new Error((t.message||t.statusText)+" "+e),{source:"fetch"})),void(!t.status&&t instanceof Error&&function(){try{var t=void 0;if("URL"in u.default?t=new URL(e):(t=document.createElement("a")).href=e,"https:"!==t.protocol&&"https:"===u.default.location.protocol){var n=(0,o.default)(new Error("Possible mixed-content issue? The page was loaded over https:// but a "+t.protocol+"// URL was specified. Check that you are not attempting to load mixed content."),{source:"fetch"});return void r.newThrownErr(n)}if(t.origin!==u.default.location.origin){var i=(0,o.default)(new Error("Possible cross-origin (CORS) issue? The URL origin ("+t.origin+") does not match the page ("+u.default.location.origin+"). Check the server returns the correct 'Access-Control-Allow-*' headers."),{source:"fetch"});r.newThrownErr(i)}}catch(e){return}}());a.updateLoadingStatus("success"),a.updateSpec(t.text),i.url()!==e&&a.updateUrl(e)}e=e||i.url(),a.updateLoadingStatus("loading"),r.clear({source:"fetch"}),l({url:e,loadSpec:!0,requestInterceptor:c.requestInterceptor||function(e){return e},responseInterceptor:c.responseInterceptor||function(e){return e},credentials:"same-origin",headers:{Accept:"application/json,*/*"}}).then(f,f)}},updateLoadingStatus:function(e){var t=[null,"loading","failed","success","failedConfig"];return-1===t.indexOf(e)&&console.error("Error: "+e+" is not one of "+(0,r.default)(t)),{type:"spec_update_loading_status",payload:e}}},s={loadingStatus:(0,i.createSelector)(function(e){return e||(0,a.Map)()},function(e){return e.get("loadingStatus")||null})};return{statePlugins:{spec:{actions:n,reducers:{spec_update_loading_status:function(e,t){return"string"==typeof t.payload?e.set("loadingStatus",t.payload):e}},selectors:s}}}};var i=n(58),a=n(7),u=s(n(32));function s(e){return e&&e.__esModule?e:{default:e}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{statePlugins:{spec:{actions:a,selectors:f},configs:{reducers:s.default,actions:i,selectors:u}}}};var r=c(n(934)),o=n(234),i=l(n(235)),a=l(n(401)),u=l(n(402)),s=c(n(403));function l(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}function c(e){return e&&e.__esModule?e:{default:e}}var f={getLocalConfig:function(){return(0,o.parseYamlConfig)(r.default)}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getConfigByUrl=t.downloadConfig=void 0;var r=n(234);t.downloadConfig=function(e){return function(t){return(0,t.fn.fetch)(e)}},t.getConfigByUrl=function(e,t){return function(n){var o=n.specActions;if(e)return o.downloadConfig(e).then(i,i);function i(n){n instanceof Error||n.status>=400?(o.updateLoadingStatus("failedConfig"),o.updateLoadingStatus("failedConfig"),o.updateUrl(""),console.error(n.statusText+" "+e.url),t(null)):t((0,r.parseYamlConfig)(n.text))}}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});t.get=function(e,t){return e.getIn(Array.isArray(t)?t:[t])}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r,o,i=n(22),a=(r=i)&&r.__esModule?r:{default:r},u=n(7),s=n(235);t.default=(o={},(0,a.default)(o,s.UPDATE_CONFIGS,function(e,t){return e.merge((0,u.fromJS)(t.payload))}),(0,a.default)(o,s.TOGGLE_CONFIGS,function(e,t){var n=t.payload,r=e.get(n);return e.set(n,!r)}),o)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return[r.default,{statePlugins:{configs:{wrapActions:{loaded:function(e,t){return function(){e.apply(void 0,arguments);var n=decodeURIComponent(window.location.hash);t.layoutActions.parseDeepLinkHash(n)}}}}},wrapComponents:{operation:o.default,OperationTag:i.default}}]};var r=a(n(405)),o=a(n(407)),i=a(n(408));function a(e){return e&&e.__esModule?e:{default:e}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.clearScrollTo=t.scrollToElement=t.readyToScroll=t.parseDeepLinkHash=t.scrollTo=t.show=void 0;var r,o=f(n(22)),i=f(n(17)),a=n(406),u=f(n(935)),s=n(9),l=n(7),c=f(l);function f(e){return e&&e.__esModule?e:{default:e}}var p=t.show=function(e,t){var n=t.getConfigs,r=t.layoutSelectors;return function(){for(var t=arguments.length,o=Array(t),u=0;u-1&&(console.warn("Warning: escaping deep link whitespace with `_` will be unsupported in v4.0, use `%20` instead."),n.show(h.map(function(e){return e.replace(/_/g," ")}),!0)),n.show(h,!0)}(f.indexOf("_")>-1||d.indexOf("_")>-1)&&(console.warn("Warning: escaping deep link whitespace with `_` will be unsupported in v4.0, use `%20` instead."),n.show(u.map(function(e){return e.replace(/_/g," ")}),!0)),n.show(u,!0),n.scrollTo(u)}}},v=t.readyToScroll=function(e,t){return function(n){var r=n.layoutSelectors.getScrollToKey();c.default.is(r,(0,l.fromJS)(e))&&(n.layoutActions.scrollToElement(t),n.layoutActions.clearScrollTo())}},m=t.scrollToElement=function(e,t){return function(n){try{t=t||n.fn.getScrollParent(e),u.default.createScroller(t).to(e)}catch(e){console.error(e)}}},g=t.clearScrollTo=function(){return{type:"layout_clear_scroll"}};t.default={fn:{getScrollParent:function(e,t){var n=document.documentElement,r=getComputedStyle(e),o="absolute"===r.position,i=t?/(auto|scroll|hidden)/:/(auto|scroll)/;if("fixed"===r.position)return n;for(var a=e;a=a.parentElement;)if(r=getComputedStyle(a),(!o||"static"!==r.position)&&i.test(r.overflow+r.overflowY+r.overflowX))return a;return n}},statePlugins:{layout:{actions:{scrollToElement:m,scrollTo:d,clearScrollTo:g,readyToScroll:v,parseDeepLinkHash:h},selectors:{getScrollToKey:function(e){return e.get("scrollToKey")},isShownKeyFromUrlHashArray:function(e,t){var n=(0,i.default)(t,2),r=n[0],o=n[1];return o?["operations",r,o]:r?["operations-tag",r]:[]},urlHashArrayFromIsShownKey:function(e,t){var n=(0,i.default)(t,3),r=n[0],o=n[1],a=n[2];return"operations"==r?[o,a]:"operations-tag"==r?[o]:[]}},reducers:(r={},(0,o.default)(r,"layout_scroll_to",function(e,t){return e.set("scrollToKey",c.default.fromJS(t.payload))}),(0,o.default)(r,"layout_clear_scroll",function(e){return e.delete("scrollToKey")}),r),wrapActions:{show:p}}}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});t.setHash=function(e){return e?history.pushState(null,null,"#"+e):window.location.hash=""}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=l(n(4)),o=l(n(2)),i=l(n(3)),a=l(n(5)),u=l(n(6)),s=l(n(0));l(n(12));function l(e){return e&&e.__esModule?e:{default:e}}t.default=function(e,t){return function(n){function l(){var e,n,i,u;(0,o.default)(this,l);for(var s=arguments.length,c=Array(s),f=0;f",Gt:"≫",gt:">",gtcc:"⪧",gtcir:"⩺",gtdot:"⋗",gtlPar:"⦕",gtquest:"⩼",gtrapprox:"⪆",gtrarr:"⥸",gtrdot:"⋗",gtreqless:"⋛",gtreqqless:"⪌",gtrless:"≷",gtrsim:"≳",gvertneqq:"≩︀",gvnE:"≩︀",Hacek:"ˇ",hairsp:" ",half:"½",hamilt:"ℋ",HARDcy:"Ъ",hardcy:"ъ",hArr:"⇔",harr:"↔",harrcir:"⥈",harrw:"↭",Hat:"^",hbar:"ℏ",Hcirc:"Ĥ",hcirc:"ĥ",hearts:"♥",heartsuit:"♥",hellip:"…",hercon:"⊹",Hfr:"ℌ",hfr:"𝔥",HilbertSpace:"ℋ",hksearow:"⤥",hkswarow:"⤦",hoarr:"⇿",homtht:"∻",hookleftarrow:"↩",hookrightarrow:"↪",Hopf:"ℍ",hopf:"𝕙",horbar:"―",HorizontalLine:"─",Hscr:"ℋ",hscr:"𝒽",hslash:"ℏ",Hstrok:"Ħ",hstrok:"ħ",HumpDownHump:"≎",HumpEqual:"≏",hybull:"⁃",hyphen:"‐",Iacute:"Í",iacute:"í",ic:"⁣",Icirc:"Î",icirc:"î",Icy:"И",icy:"и",Idot:"İ",IEcy:"Е",iecy:"е",iexcl:"¡",iff:"⇔",Ifr:"ℑ",ifr:"𝔦",Igrave:"Ì",igrave:"ì",ii:"ⅈ",iiiint:"⨌",iiint:"∭",iinfin:"⧜",iiota:"℩",IJlig:"IJ",ijlig:"ij",Im:"ℑ",Imacr:"Ī",imacr:"ī",image:"ℑ",ImaginaryI:"ⅈ",imagline:"ℐ",imagpart:"ℑ",imath:"ı",imof:"⊷",imped:"Ƶ",Implies:"⇒",in:"∈",incare:"℅",infin:"∞",infintie:"⧝",inodot:"ı",Int:"∬",int:"∫",intcal:"⊺",integers:"ℤ",Integral:"∫",intercal:"⊺",Intersection:"⋂",intlarhk:"⨗",intprod:"⨼",InvisibleComma:"⁣",InvisibleTimes:"⁢",IOcy:"Ё",iocy:"ё",Iogon:"Į",iogon:"į",Iopf:"𝕀",iopf:"𝕚",Iota:"Ι",iota:"ι",iprod:"⨼",iquest:"¿",Iscr:"ℐ",iscr:"𝒾",isin:"∈",isindot:"⋵",isinE:"⋹",isins:"⋴",isinsv:"⋳",isinv:"∈",it:"⁢",Itilde:"Ĩ",itilde:"ĩ",Iukcy:"І",iukcy:"і",Iuml:"Ï",iuml:"ï",Jcirc:"Ĵ",jcirc:"ĵ",Jcy:"Й",jcy:"й",Jfr:"𝔍",jfr:"𝔧",jmath:"ȷ",Jopf:"𝕁",jopf:"𝕛",Jscr:"𝒥",jscr:"𝒿",Jsercy:"Ј",jsercy:"ј",Jukcy:"Є",jukcy:"є",Kappa:"Κ",kappa:"κ",kappav:"ϰ",Kcedil:"Ķ",kcedil:"ķ",Kcy:"К",kcy:"к",Kfr:"𝔎",kfr:"𝔨",kgreen:"ĸ",KHcy:"Х",khcy:"х",KJcy:"Ќ",kjcy:"ќ",Kopf:"𝕂",kopf:"𝕜",Kscr:"𝒦",kscr:"𝓀",lAarr:"⇚",Lacute:"Ĺ",lacute:"ĺ",laemptyv:"⦴",lagran:"ℒ",Lambda:"Λ",lambda:"λ",Lang:"⟪",lang:"⟨",langd:"⦑",langle:"⟨",lap:"⪅",Laplacetrf:"ℒ",laquo:"«",Larr:"↞",lArr:"⇐",larr:"←",larrb:"⇤",larrbfs:"⤟",larrfs:"⤝",larrhk:"↩",larrlp:"↫",larrpl:"⤹",larrsim:"⥳",larrtl:"↢",lat:"⪫",lAtail:"⤛",latail:"⤙",late:"⪭",lates:"⪭︀",lBarr:"⤎",lbarr:"⤌",lbbrk:"❲",lbrace:"{",lbrack:"[",lbrke:"⦋",lbrksld:"⦏",lbrkslu:"⦍",Lcaron:"Ľ",lcaron:"ľ",Lcedil:"Ļ",lcedil:"ļ",lceil:"⌈",lcub:"{",Lcy:"Л",lcy:"л",ldca:"⤶",ldquo:"“",ldquor:"„",ldrdhar:"⥧",ldrushar:"⥋",ldsh:"↲",lE:"≦",le:"≤",LeftAngleBracket:"⟨",LeftArrow:"←",Leftarrow:"⇐",leftarrow:"←",LeftArrowBar:"⇤",LeftArrowRightArrow:"⇆",leftarrowtail:"↢",LeftCeiling:"⌈",LeftDoubleBracket:"⟦",LeftDownTeeVector:"⥡",LeftDownVector:"⇃",LeftDownVectorBar:"⥙",LeftFloor:"⌊",leftharpoondown:"↽",leftharpoonup:"↼",leftleftarrows:"⇇",LeftRightArrow:"↔",Leftrightarrow:"⇔",leftrightarrow:"↔",leftrightarrows:"⇆",leftrightharpoons:"⇋",leftrightsquigarrow:"↭",LeftRightVector:"⥎",LeftTee:"⊣",LeftTeeArrow:"↤",LeftTeeVector:"⥚",leftthreetimes:"⋋",LeftTriangle:"⊲",LeftTriangleBar:"⧏",LeftTriangleEqual:"⊴",LeftUpDownVector:"⥑",LeftUpTeeVector:"⥠",LeftUpVector:"↿",LeftUpVectorBar:"⥘",LeftVector:"↼",LeftVectorBar:"⥒",lEg:"⪋",leg:"⋚",leq:"≤",leqq:"≦",leqslant:"⩽",les:"⩽",lescc:"⪨",lesdot:"⩿",lesdoto:"⪁",lesdotor:"⪃",lesg:"⋚︀",lesges:"⪓",lessapprox:"⪅",lessdot:"⋖",lesseqgtr:"⋚",lesseqqgtr:"⪋",LessEqualGreater:"⋚",LessFullEqual:"≦",LessGreater:"≶",lessgtr:"≶",LessLess:"⪡",lesssim:"≲",LessSlantEqual:"⩽",LessTilde:"≲",lfisht:"⥼",lfloor:"⌊",Lfr:"𝔏",lfr:"𝔩",lg:"≶",lgE:"⪑",lHar:"⥢",lhard:"↽",lharu:"↼",lharul:"⥪",lhblk:"▄",LJcy:"Љ",ljcy:"љ",Ll:"⋘",ll:"≪",llarr:"⇇",llcorner:"⌞",Lleftarrow:"⇚",llhard:"⥫",lltri:"◺",Lmidot:"Ŀ",lmidot:"ŀ",lmoust:"⎰",lmoustache:"⎰",lnap:"⪉",lnapprox:"⪉",lnE:"≨",lne:"⪇",lneq:"⪇",lneqq:"≨",lnsim:"⋦",loang:"⟬",loarr:"⇽",lobrk:"⟦",LongLeftArrow:"⟵",Longleftarrow:"⟸",longleftarrow:"⟵",LongLeftRightArrow:"⟷",Longleftrightarrow:"⟺",longleftrightarrow:"⟷",longmapsto:"⟼",LongRightArrow:"⟶",Longrightarrow:"⟹",longrightarrow:"⟶",looparrowleft:"↫",looparrowright:"↬",lopar:"⦅",Lopf:"𝕃",lopf:"𝕝",loplus:"⨭",lotimes:"⨴",lowast:"∗",lowbar:"_",LowerLeftArrow:"↙",LowerRightArrow:"↘",loz:"◊",lozenge:"◊",lozf:"⧫",lpar:"(",lparlt:"⦓",lrarr:"⇆",lrcorner:"⌟",lrhar:"⇋",lrhard:"⥭",lrm:"‎",lrtri:"⊿",lsaquo:"‹",Lscr:"ℒ",lscr:"𝓁",Lsh:"↰",lsh:"↰",lsim:"≲",lsime:"⪍",lsimg:"⪏",lsqb:"[",lsquo:"‘",lsquor:"‚",Lstrok:"Ł",lstrok:"ł",LT:"<",Lt:"≪",lt:"<",ltcc:"⪦",ltcir:"⩹",ltdot:"⋖",lthree:"⋋",ltimes:"⋉",ltlarr:"⥶",ltquest:"⩻",ltri:"◃",ltrie:"⊴",ltrif:"◂",ltrPar:"⦖",lurdshar:"⥊",luruhar:"⥦",lvertneqq:"≨︀",lvnE:"≨︀",macr:"¯",male:"♂",malt:"✠",maltese:"✠",Map:"⤅",map:"↦",mapsto:"↦",mapstodown:"↧",mapstoleft:"↤",mapstoup:"↥",marker:"▮",mcomma:"⨩",Mcy:"М",mcy:"м",mdash:"—",mDDot:"∺",measuredangle:"∡",MediumSpace:" ",Mellintrf:"ℳ",Mfr:"𝔐",mfr:"𝔪",mho:"℧",micro:"µ",mid:"∣",midast:"*",midcir:"⫰",middot:"·",minus:"−",minusb:"⊟",minusd:"∸",minusdu:"⨪",MinusPlus:"∓",mlcp:"⫛",mldr:"…",mnplus:"∓",models:"⊧",Mopf:"𝕄",mopf:"𝕞",mp:"∓",Mscr:"ℳ",mscr:"𝓂",mstpos:"∾",Mu:"Μ",mu:"μ",multimap:"⊸",mumap:"⊸",nabla:"∇",Nacute:"Ń",nacute:"ń",nang:"∠⃒",nap:"≉",napE:"⩰̸",napid:"≋̸",napos:"ʼn",napprox:"≉",natur:"♮",natural:"♮",naturals:"ℕ",nbsp:" ",nbump:"≎̸",nbumpe:"≏̸",ncap:"⩃",Ncaron:"Ň",ncaron:"ň",Ncedil:"Ņ",ncedil:"ņ",ncong:"≇",ncongdot:"⩭̸",ncup:"⩂",Ncy:"Н",ncy:"н",ndash:"–",ne:"≠",nearhk:"⤤",neArr:"⇗",nearr:"↗",nearrow:"↗",nedot:"≐̸",NegativeMediumSpace:"​",NegativeThickSpace:"​",NegativeThinSpace:"​",NegativeVeryThinSpace:"​",nequiv:"≢",nesear:"⤨",nesim:"≂̸",NestedGreaterGreater:"≫",NestedLessLess:"≪",NewLine:"\n",nexist:"∄",nexists:"∄",Nfr:"𝔑",nfr:"𝔫",ngE:"≧̸",nge:"≱",ngeq:"≱",ngeqq:"≧̸",ngeqslant:"⩾̸",nges:"⩾̸",nGg:"⋙̸",ngsim:"≵",nGt:"≫⃒",ngt:"≯",ngtr:"≯",nGtv:"≫̸",nhArr:"⇎",nharr:"↮",nhpar:"⫲",ni:"∋",nis:"⋼",nisd:"⋺",niv:"∋",NJcy:"Њ",njcy:"њ",nlArr:"⇍",nlarr:"↚",nldr:"‥",nlE:"≦̸",nle:"≰",nLeftarrow:"⇍",nleftarrow:"↚",nLeftrightarrow:"⇎",nleftrightarrow:"↮",nleq:"≰",nleqq:"≦̸",nleqslant:"⩽̸",nles:"⩽̸",nless:"≮",nLl:"⋘̸",nlsim:"≴",nLt:"≪⃒",nlt:"≮",nltri:"⋪",nltrie:"⋬",nLtv:"≪̸",nmid:"∤",NoBreak:"⁠",NonBreakingSpace:" ",Nopf:"ℕ",nopf:"𝕟",Not:"⫬",not:"¬",NotCongruent:"≢",NotCupCap:"≭",NotDoubleVerticalBar:"∦",NotElement:"∉",NotEqual:"≠",NotEqualTilde:"≂̸",NotExists:"∄",NotGreater:"≯",NotGreaterEqual:"≱",NotGreaterFullEqual:"≧̸",NotGreaterGreater:"≫̸",NotGreaterLess:"≹",NotGreaterSlantEqual:"⩾̸",NotGreaterTilde:"≵",NotHumpDownHump:"≎̸",NotHumpEqual:"≏̸",notin:"∉",notindot:"⋵̸",notinE:"⋹̸",notinva:"∉",notinvb:"⋷",notinvc:"⋶",NotLeftTriangle:"⋪",NotLeftTriangleBar:"⧏̸",NotLeftTriangleEqual:"⋬",NotLess:"≮",NotLessEqual:"≰",NotLessGreater:"≸",NotLessLess:"≪̸",NotLessSlantEqual:"⩽̸",NotLessTilde:"≴",NotNestedGreaterGreater:"⪢̸",NotNestedLessLess:"⪡̸",notni:"∌",notniva:"∌",notnivb:"⋾",notnivc:"⋽",NotPrecedes:"⊀",NotPrecedesEqual:"⪯̸",NotPrecedesSlantEqual:"⋠",NotReverseElement:"∌",NotRightTriangle:"⋫",NotRightTriangleBar:"⧐̸",NotRightTriangleEqual:"⋭",NotSquareSubset:"⊏̸",NotSquareSubsetEqual:"⋢",NotSquareSuperset:"⊐̸",NotSquareSupersetEqual:"⋣",NotSubset:"⊂⃒",NotSubsetEqual:"⊈",NotSucceeds:"⊁",NotSucceedsEqual:"⪰̸",NotSucceedsSlantEqual:"⋡",NotSucceedsTilde:"≿̸",NotSuperset:"⊃⃒",NotSupersetEqual:"⊉",NotTilde:"≁",NotTildeEqual:"≄",NotTildeFullEqual:"≇",NotTildeTilde:"≉",NotVerticalBar:"∤",npar:"∦",nparallel:"∦",nparsl:"⫽⃥",npart:"∂̸",npolint:"⨔",npr:"⊀",nprcue:"⋠",npre:"⪯̸",nprec:"⊀",npreceq:"⪯̸",nrArr:"⇏",nrarr:"↛",nrarrc:"⤳̸",nrarrw:"↝̸",nRightarrow:"⇏",nrightarrow:"↛",nrtri:"⋫",nrtrie:"⋭",nsc:"⊁",nsccue:"⋡",nsce:"⪰̸",Nscr:"𝒩",nscr:"𝓃",nshortmid:"∤",nshortparallel:"∦",nsim:"≁",nsime:"≄",nsimeq:"≄",nsmid:"∤",nspar:"∦",nsqsube:"⋢",nsqsupe:"⋣",nsub:"⊄",nsubE:"⫅̸",nsube:"⊈",nsubset:"⊂⃒",nsubseteq:"⊈",nsubseteqq:"⫅̸",nsucc:"⊁",nsucceq:"⪰̸",nsup:"⊅",nsupE:"⫆̸",nsupe:"⊉",nsupset:"⊃⃒",nsupseteq:"⊉",nsupseteqq:"⫆̸",ntgl:"≹",Ntilde:"Ñ",ntilde:"ñ",ntlg:"≸",ntriangleleft:"⋪",ntrianglelefteq:"⋬",ntriangleright:"⋫",ntrianglerighteq:"⋭",Nu:"Ν",nu:"ν",num:"#",numero:"№",numsp:" ",nvap:"≍⃒",nVDash:"⊯",nVdash:"⊮",nvDash:"⊭",nvdash:"⊬",nvge:"≥⃒",nvgt:">⃒",nvHarr:"⤄",nvinfin:"⧞",nvlArr:"⤂",nvle:"≤⃒",nvlt:"<⃒",nvltrie:"⊴⃒",nvrArr:"⤃",nvrtrie:"⊵⃒",nvsim:"∼⃒",nwarhk:"⤣",nwArr:"⇖",nwarr:"↖",nwarrow:"↖",nwnear:"⤧",Oacute:"Ó",oacute:"ó",oast:"⊛",ocir:"⊚",Ocirc:"Ô",ocirc:"ô",Ocy:"О",ocy:"о",odash:"⊝",Odblac:"Ő",odblac:"ő",odiv:"⨸",odot:"⊙",odsold:"⦼",OElig:"Œ",oelig:"œ",ofcir:"⦿",Ofr:"𝔒",ofr:"𝔬",ogon:"˛",Ograve:"Ò",ograve:"ò",ogt:"⧁",ohbar:"⦵",ohm:"Ω",oint:"∮",olarr:"↺",olcir:"⦾",olcross:"⦻",oline:"‾",olt:"⧀",Omacr:"Ō",omacr:"ō",Omega:"Ω",omega:"ω",Omicron:"Ο",omicron:"ο",omid:"⦶",ominus:"⊖",Oopf:"𝕆",oopf:"𝕠",opar:"⦷",OpenCurlyDoubleQuote:"“",OpenCurlyQuote:"‘",operp:"⦹",oplus:"⊕",Or:"⩔",or:"∨",orarr:"↻",ord:"⩝",order:"ℴ",orderof:"ℴ",ordf:"ª",ordm:"º",origof:"⊶",oror:"⩖",orslope:"⩗",orv:"⩛",oS:"Ⓢ",Oscr:"𝒪",oscr:"ℴ",Oslash:"Ø",oslash:"ø",osol:"⊘",Otilde:"Õ",otilde:"õ",Otimes:"⨷",otimes:"⊗",otimesas:"⨶",Ouml:"Ö",ouml:"ö",ovbar:"⌽",OverBar:"‾",OverBrace:"⏞",OverBracket:"⎴",OverParenthesis:"⏜",par:"∥",para:"¶",parallel:"∥",parsim:"⫳",parsl:"⫽",part:"∂",PartialD:"∂",Pcy:"П",pcy:"п",percnt:"%",period:".",permil:"‰",perp:"⊥",pertenk:"‱",Pfr:"𝔓",pfr:"𝔭",Phi:"Φ",phi:"φ",phiv:"ϕ",phmmat:"ℳ",phone:"☎",Pi:"Π",pi:"π",pitchfork:"⋔",piv:"ϖ",planck:"ℏ",planckh:"ℎ",plankv:"ℏ",plus:"+",plusacir:"⨣",plusb:"⊞",pluscir:"⨢",plusdo:"∔",plusdu:"⨥",pluse:"⩲",PlusMinus:"±",plusmn:"±",plussim:"⨦",plustwo:"⨧",pm:"±",Poincareplane:"ℌ",pointint:"⨕",Popf:"ℙ",popf:"𝕡",pound:"£",Pr:"⪻",pr:"≺",prap:"⪷",prcue:"≼",prE:"⪳",pre:"⪯",prec:"≺",precapprox:"⪷",preccurlyeq:"≼",Precedes:"≺",PrecedesEqual:"⪯",PrecedesSlantEqual:"≼",PrecedesTilde:"≾",preceq:"⪯",precnapprox:"⪹",precneqq:"⪵",precnsim:"⋨",precsim:"≾",Prime:"″",prime:"′",primes:"ℙ",prnap:"⪹",prnE:"⪵",prnsim:"⋨",prod:"∏",Product:"∏",profalar:"⌮",profline:"⌒",profsurf:"⌓",prop:"∝",Proportion:"∷",Proportional:"∝",propto:"∝",prsim:"≾",prurel:"⊰",Pscr:"𝒫",pscr:"𝓅",Psi:"Ψ",psi:"ψ",puncsp:" ",Qfr:"𝔔",qfr:"𝔮",qint:"⨌",Qopf:"ℚ",qopf:"𝕢",qprime:"⁗",Qscr:"𝒬",qscr:"𝓆",quaternions:"ℍ",quatint:"⨖",quest:"?",questeq:"≟",QUOT:'"',quot:'"',rAarr:"⇛",race:"∽̱",Racute:"Ŕ",racute:"ŕ",radic:"√",raemptyv:"⦳",Rang:"⟫",rang:"⟩",rangd:"⦒",range:"⦥",rangle:"⟩",raquo:"»",Rarr:"↠",rArr:"⇒",rarr:"→",rarrap:"⥵",rarrb:"⇥",rarrbfs:"⤠",rarrc:"⤳",rarrfs:"⤞",rarrhk:"↪",rarrlp:"↬",rarrpl:"⥅",rarrsim:"⥴",Rarrtl:"⤖",rarrtl:"↣",rarrw:"↝",rAtail:"⤜",ratail:"⤚",ratio:"∶",rationals:"ℚ",RBarr:"⤐",rBarr:"⤏",rbarr:"⤍",rbbrk:"❳",rbrace:"}",rbrack:"]",rbrke:"⦌",rbrksld:"⦎",rbrkslu:"⦐",Rcaron:"Ř",rcaron:"ř",Rcedil:"Ŗ",rcedil:"ŗ",rceil:"⌉",rcub:"}",Rcy:"Р",rcy:"р",rdca:"⤷",rdldhar:"⥩",rdquo:"”",rdquor:"”",rdsh:"↳",Re:"ℜ",real:"ℜ",realine:"ℛ",realpart:"ℜ",reals:"ℝ",rect:"▭",REG:"®",reg:"®",ReverseElement:"∋",ReverseEquilibrium:"⇋",ReverseUpEquilibrium:"⥯",rfisht:"⥽",rfloor:"⌋",Rfr:"ℜ",rfr:"𝔯",rHar:"⥤",rhard:"⇁",rharu:"⇀",rharul:"⥬",Rho:"Ρ",rho:"ρ",rhov:"ϱ",RightAngleBracket:"⟩",RightArrow:"→",Rightarrow:"⇒",rightarrow:"→",RightArrowBar:"⇥",RightArrowLeftArrow:"⇄",rightarrowtail:"↣",RightCeiling:"⌉",RightDoubleBracket:"⟧",RightDownTeeVector:"⥝",RightDownVector:"⇂",RightDownVectorBar:"⥕",RightFloor:"⌋",rightharpoondown:"⇁",rightharpoonup:"⇀",rightleftarrows:"⇄",rightleftharpoons:"⇌",rightrightarrows:"⇉",rightsquigarrow:"↝",RightTee:"⊢",RightTeeArrow:"↦",RightTeeVector:"⥛",rightthreetimes:"⋌",RightTriangle:"⊳",RightTriangleBar:"⧐",RightTriangleEqual:"⊵",RightUpDownVector:"⥏",RightUpTeeVector:"⥜",RightUpVector:"↾",RightUpVectorBar:"⥔",RightVector:"⇀",RightVectorBar:"⥓",ring:"˚",risingdotseq:"≓",rlarr:"⇄",rlhar:"⇌",rlm:"‏",rmoust:"⎱",rmoustache:"⎱",rnmid:"⫮",roang:"⟭",roarr:"⇾",robrk:"⟧",ropar:"⦆",Ropf:"ℝ",ropf:"𝕣",roplus:"⨮",rotimes:"⨵",RoundImplies:"⥰",rpar:")",rpargt:"⦔",rppolint:"⨒",rrarr:"⇉",Rrightarrow:"⇛",rsaquo:"›",Rscr:"ℛ",rscr:"𝓇",Rsh:"↱",rsh:"↱",rsqb:"]",rsquo:"’",rsquor:"’",rthree:"⋌",rtimes:"⋊",rtri:"▹",rtrie:"⊵",rtrif:"▸",rtriltri:"⧎",RuleDelayed:"⧴",ruluhar:"⥨",rx:"℞",Sacute:"Ś",sacute:"ś",sbquo:"‚",Sc:"⪼",sc:"≻",scap:"⪸",Scaron:"Š",scaron:"š",sccue:"≽",scE:"⪴",sce:"⪰",Scedil:"Ş",scedil:"ş",Scirc:"Ŝ",scirc:"ŝ",scnap:"⪺",scnE:"⪶",scnsim:"⋩",scpolint:"⨓",scsim:"≿",Scy:"С",scy:"с",sdot:"⋅",sdotb:"⊡",sdote:"⩦",searhk:"⤥",seArr:"⇘",searr:"↘",searrow:"↘",sect:"§",semi:";",seswar:"⤩",setminus:"∖",setmn:"∖",sext:"✶",Sfr:"𝔖",sfr:"𝔰",sfrown:"⌢",sharp:"♯",SHCHcy:"Щ",shchcy:"щ",SHcy:"Ш",shcy:"ш",ShortDownArrow:"↓",ShortLeftArrow:"←",shortmid:"∣",shortparallel:"∥",ShortRightArrow:"→",ShortUpArrow:"↑",shy:"­",Sigma:"Σ",sigma:"σ",sigmaf:"ς",sigmav:"ς",sim:"∼",simdot:"⩪",sime:"≃",simeq:"≃",simg:"⪞",simgE:"⪠",siml:"⪝",simlE:"⪟",simne:"≆",simplus:"⨤",simrarr:"⥲",slarr:"←",SmallCircle:"∘",smallsetminus:"∖",smashp:"⨳",smeparsl:"⧤",smid:"∣",smile:"⌣",smt:"⪪",smte:"⪬",smtes:"⪬︀",SOFTcy:"Ь",softcy:"ь",sol:"/",solb:"⧄",solbar:"⌿",Sopf:"𝕊",sopf:"𝕤",spades:"♠",spadesuit:"♠",spar:"∥",sqcap:"⊓",sqcaps:"⊓︀",sqcup:"⊔",sqcups:"⊔︀",Sqrt:"√",sqsub:"⊏",sqsube:"⊑",sqsubset:"⊏",sqsubseteq:"⊑",sqsup:"⊐",sqsupe:"⊒",sqsupset:"⊐",sqsupseteq:"⊒",squ:"□",Square:"□",square:"□",SquareIntersection:"⊓",SquareSubset:"⊏",SquareSubsetEqual:"⊑",SquareSuperset:"⊐",SquareSupersetEqual:"⊒",SquareUnion:"⊔",squarf:"▪",squf:"▪",srarr:"→",Sscr:"𝒮",sscr:"𝓈",ssetmn:"∖",ssmile:"⌣",sstarf:"⋆",Star:"⋆",star:"☆",starf:"★",straightepsilon:"ϵ",straightphi:"ϕ",strns:"¯",Sub:"⋐",sub:"⊂",subdot:"⪽",subE:"⫅",sube:"⊆",subedot:"⫃",submult:"⫁",subnE:"⫋",subne:"⊊",subplus:"⪿",subrarr:"⥹",Subset:"⋐",subset:"⊂",subseteq:"⊆",subseteqq:"⫅",SubsetEqual:"⊆",subsetneq:"⊊",subsetneqq:"⫋",subsim:"⫇",subsub:"⫕",subsup:"⫓",succ:"≻",succapprox:"⪸",succcurlyeq:"≽",Succeeds:"≻",SucceedsEqual:"⪰",SucceedsSlantEqual:"≽",SucceedsTilde:"≿",succeq:"⪰",succnapprox:"⪺",succneqq:"⪶",succnsim:"⋩",succsim:"≿",SuchThat:"∋",Sum:"∑",sum:"∑",sung:"♪",Sup:"⋑",sup:"⊃",sup1:"¹",sup2:"²",sup3:"³",supdot:"⪾",supdsub:"⫘",supE:"⫆",supe:"⊇",supedot:"⫄",Superset:"⊃",SupersetEqual:"⊇",suphsol:"⟉",suphsub:"⫗",suplarr:"⥻",supmult:"⫂",supnE:"⫌",supne:"⊋",supplus:"⫀",Supset:"⋑",supset:"⊃",supseteq:"⊇",supseteqq:"⫆",supsetneq:"⊋",supsetneqq:"⫌",supsim:"⫈",supsub:"⫔",supsup:"⫖",swarhk:"⤦",swArr:"⇙",swarr:"↙",swarrow:"↙",swnwar:"⤪",szlig:"ß",Tab:"\t",target:"⌖",Tau:"Τ",tau:"τ",tbrk:"⎴",Tcaron:"Ť",tcaron:"ť",Tcedil:"Ţ",tcedil:"ţ",Tcy:"Т",tcy:"т",tdot:"⃛",telrec:"⌕",Tfr:"𝔗",tfr:"𝔱",there4:"∴",Therefore:"∴",therefore:"∴",Theta:"Θ",theta:"θ",thetasym:"ϑ",thetav:"ϑ",thickapprox:"≈",thicksim:"∼",ThickSpace:"  ",thinsp:" ",ThinSpace:" ",thkap:"≈",thksim:"∼",THORN:"Þ",thorn:"þ",Tilde:"∼",tilde:"˜",TildeEqual:"≃",TildeFullEqual:"≅",TildeTilde:"≈",times:"×",timesb:"⊠",timesbar:"⨱",timesd:"⨰",tint:"∭",toea:"⤨",top:"⊤",topbot:"⌶",topcir:"⫱",Topf:"𝕋",topf:"𝕥",topfork:"⫚",tosa:"⤩",tprime:"‴",TRADE:"™",trade:"™",triangle:"▵",triangledown:"▿",triangleleft:"◃",trianglelefteq:"⊴",triangleq:"≜",triangleright:"▹",trianglerighteq:"⊵",tridot:"◬",trie:"≜",triminus:"⨺",TripleDot:"⃛",triplus:"⨹",trisb:"⧍",tritime:"⨻",trpezium:"⏢",Tscr:"𝒯",tscr:"𝓉",TScy:"Ц",tscy:"ц",TSHcy:"Ћ",tshcy:"ћ",Tstrok:"Ŧ",tstrok:"ŧ",twixt:"≬",twoheadleftarrow:"↞",twoheadrightarrow:"↠",Uacute:"Ú",uacute:"ú",Uarr:"↟",uArr:"⇑",uarr:"↑",Uarrocir:"⥉",Ubrcy:"Ў",ubrcy:"ў",Ubreve:"Ŭ",ubreve:"ŭ",Ucirc:"Û",ucirc:"û",Ucy:"У",ucy:"у",udarr:"⇅",Udblac:"Ű",udblac:"ű",udhar:"⥮",ufisht:"⥾",Ufr:"𝔘",ufr:"𝔲",Ugrave:"Ù",ugrave:"ù",uHar:"⥣",uharl:"↿",uharr:"↾",uhblk:"▀",ulcorn:"⌜",ulcorner:"⌜",ulcrop:"⌏",ultri:"◸",Umacr:"Ū",umacr:"ū",uml:"¨",UnderBar:"_",UnderBrace:"⏟",UnderBracket:"⎵",UnderParenthesis:"⏝",Union:"⋃",UnionPlus:"⊎",Uogon:"Ų",uogon:"ų",Uopf:"𝕌",uopf:"𝕦",UpArrow:"↑",Uparrow:"⇑",uparrow:"↑",UpArrowBar:"⤒",UpArrowDownArrow:"⇅",UpDownArrow:"↕",Updownarrow:"⇕",updownarrow:"↕",UpEquilibrium:"⥮",upharpoonleft:"↿",upharpoonright:"↾",uplus:"⊎",UpperLeftArrow:"↖",UpperRightArrow:"↗",Upsi:"ϒ",upsi:"υ",upsih:"ϒ",Upsilon:"Υ",upsilon:"υ",UpTee:"⊥",UpTeeArrow:"↥",upuparrows:"⇈",urcorn:"⌝",urcorner:"⌝",urcrop:"⌎",Uring:"Ů",uring:"ů",urtri:"◹",Uscr:"𝒰",uscr:"𝓊",utdot:"⋰",Utilde:"Ũ",utilde:"ũ",utri:"▵",utrif:"▴",uuarr:"⇈",Uuml:"Ü",uuml:"ü",uwangle:"⦧",vangrt:"⦜",varepsilon:"ϵ",varkappa:"ϰ",varnothing:"∅",varphi:"ϕ",varpi:"ϖ",varpropto:"∝",vArr:"⇕",varr:"↕",varrho:"ϱ",varsigma:"ς",varsubsetneq:"⊊︀",varsubsetneqq:"⫋︀",varsupsetneq:"⊋︀",varsupsetneqq:"⫌︀",vartheta:"ϑ",vartriangleleft:"⊲",vartriangleright:"⊳",Vbar:"⫫",vBar:"⫨",vBarv:"⫩",Vcy:"В",vcy:"в",VDash:"⊫",Vdash:"⊩",vDash:"⊨",vdash:"⊢",Vdashl:"⫦",Vee:"⋁",vee:"∨",veebar:"⊻",veeeq:"≚",vellip:"⋮",Verbar:"‖",verbar:"|",Vert:"‖",vert:"|",VerticalBar:"∣",VerticalLine:"|",VerticalSeparator:"❘",VerticalTilde:"≀",VeryThinSpace:" ",Vfr:"𝔙",vfr:"𝔳",vltri:"⊲",vnsub:"⊂⃒",vnsup:"⊃⃒",Vopf:"𝕍",vopf:"𝕧",vprop:"∝",vrtri:"⊳",Vscr:"𝒱",vscr:"𝓋",vsubnE:"⫋︀",vsubne:"⊊︀",vsupnE:"⫌︀",vsupne:"⊋︀",Vvdash:"⊪",vzigzag:"⦚",Wcirc:"Ŵ",wcirc:"ŵ",wedbar:"⩟",Wedge:"⋀",wedge:"∧",wedgeq:"≙",weierp:"℘",Wfr:"𝔚",wfr:"𝔴",Wopf:"𝕎",wopf:"𝕨",wp:"℘",wr:"≀",wreath:"≀",Wscr:"𝒲",wscr:"𝓌",xcap:"⋂",xcirc:"◯",xcup:"⋃",xdtri:"▽",Xfr:"𝔛",xfr:"𝔵",xhArr:"⟺",xharr:"⟷",Xi:"Ξ",xi:"ξ",xlArr:"⟸",xlarr:"⟵",xmap:"⟼",xnis:"⋻",xodot:"⨀",Xopf:"𝕏",xopf:"𝕩",xoplus:"⨁",xotime:"⨂",xrArr:"⟹",xrarr:"⟶",Xscr:"𝒳",xscr:"𝓍",xsqcup:"⨆",xuplus:"⨄",xutri:"△",xvee:"⋁",xwedge:"⋀",Yacute:"Ý",yacute:"ý",YAcy:"Я",yacy:"я",Ycirc:"Ŷ",ycirc:"ŷ",Ycy:"Ы",ycy:"ы",yen:"¥",Yfr:"𝔜",yfr:"𝔶",YIcy:"Ї",yicy:"ї",Yopf:"𝕐",yopf:"𝕪",Yscr:"𝒴",yscr:"𝓎",YUcy:"Ю",yucy:"ю",Yuml:"Ÿ",yuml:"ÿ",Zacute:"Ź",zacute:"ź",Zcaron:"Ž",zcaron:"ž",Zcy:"З",zcy:"з",Zdot:"Ż",zdot:"ż",zeetrf:"ℨ",ZeroWidthSpace:"​",Zeta:"Ζ",zeta:"ζ",Zfr:"ℨ",zfr:"𝔷",ZHcy:"Ж",zhcy:"ж",zigrarr:"⇝",Zopf:"ℤ",zopf:"𝕫",Zscr:"𝒵",zscr:"𝓏",zwj:"‍",zwnj:"‌"}},function(e,t,n){"use strict";var r=n(419),o=n(27).unescapeMd;e.exports=function(e,t){var n,i,a,u=t,s=e.posMax;if(60===e.src.charCodeAt(t)){for(t++;t8&&n<14);)if(92===n&&t+11)break;if(41===n&&--i<0)break;t++}return u!==t&&(a=o(e.src.slice(u,t)),!!e.parser.validateLink(a)&&(e.linkContent=a,e.pos=t,!0))}},function(e,t,n){"use strict";var r=n(27).replaceEntities;e.exports=function(e){var t=r(e);try{t=decodeURI(t)}catch(e){}return encodeURI(t)}},function(e,t,n){"use strict";var r=n(27).unescapeMd;e.exports=function(e,t){var n,o=t,i=e.posMax,a=e.src.charCodeAt(t);if(34!==a&&39!==a&&40!==a)return!1;for(t++,40===a&&(a=41);t1?r-1:0),i=1;i1?t-1:0),r=1;r0){var S=a("JsonSchemaForm"),C=a("ParameterExt"),k=w.get("properties",(0,o.OrderedMap)());return n=o.Map.isMap(n)?n:(0,o.OrderedMap)(),r.default.createElement("div",{className:"table-container"},y&&r.default.createElement(h,{source:y}),r.default.createElement("table",null,r.default.createElement("tbody",null,k.map(function(e,t){var u=g?(0,i.getCommonExtensions)(e):null,s=w.get("required",(0,o.List)()).includes(t),c=e.get("type"),p=e.get("format"),v=e.get("description"),m=n.get(t),y=e.get("default")||e.get("example")||"";""===y&&"object"===c&&(y=(0,i.getSampleSchema)(e,!1,{includeWriteOnly:!0})),"string"!=typeof y&&"object"===c&&(y=(0,i.stringify)(y));var b="string"===c&&("binary"===p||"base64"===p);return r.default.createElement("tr",{key:t,className:"parameters","data-property-name":t},r.default.createElement("td",{className:"col parameters-col_name"},r.default.createElement("div",{className:s?"parameter__name required":"parameter__name"},t,s?r.default.createElement("span",{style:{color:"red"}}," *"):null),r.default.createElement("div",{className:"parameter__type"},c,p&&r.default.createElement("span",{className:"prop-format"},"($",p,")"),g&&u.size?u.map(function(e,t){return r.default.createElement(C,{key:t+"-"+e,xKey:t,xVal:e})}):null),r.default.createElement("div",{className:"parameter__deprecated"},e.get("deprecated")?"deprecated":null)),r.default.createElement("td",{className:"col parameters-col_description"},r.default.createElement(h,{source:v}),f?r.default.createElement("div",null,r.default.createElement(S,{fn:l,dispatchInitialValue:!b,schema:e,description:t,getComponent:a,value:void 0===m?y:m,onChange:function(e){d(e,[t])}})):null))}))))}return r.default.createElement("div",null,y&&r.default.createElement(h,{source:y}),r.default.createElement(v,{getComponent:a,getConfigs:u,specSelectors:s,expandDepth:1,isExecute:f,schema:_.get("schema"),specPath:p.push("content",c),example:r.default.createElement(m,{requestBody:t,onChange:d,mediaType:c,getComponent:a,isExecute:f,specSelectors:s})}))}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=f(n(41)),o=f(n(4)),i=f(n(2)),a=f(n(3)),u=f(n(5)),s=f(n(6)),l=n(0),c=f(l);f(n(1)),f(n(12));function f(e){return e&&e.__esModule?e:{default:e}}var p=function(e){function t(){return(0,i.default)(this,t),(0,u.default)(this,(t.__proto__||(0,o.default)(t)).apply(this,arguments))}return(0,s.default)(t,e),(0,a.default)(t,[{key:"render",value:function(){var e=this.props,t=e.link,n=e.name,o=(0,e.getComponent)("Markdown"),i=t.get("operationId")||t.get("operationRef"),a=t.get("parameters")&&t.get("parameters").toJS(),u=t.get("description");return c.default.createElement("div",{style:{marginBottom:"1.5em"}},c.default.createElement("div",{style:{marginBottom:".5em"}},c.default.createElement("b",null,c.default.createElement("code",null,n)),u?c.default.createElement(o,{source:u}):null),c.default.createElement("pre",null,"Operation `",i,"`",c.default.createElement("br",null),c.default.createElement("br",null),"Parameters ",function(e,t){if("string"!=typeof t)return"";return t.split("\n").map(function(t,n){return n>0?Array(e+1).join(" ")+t:t}).join("\n")}(0,(0,r.default)(a,null,2))||"{}",c.default.createElement("br",null)))}}]),t}(l.Component);t.default=p},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=c(n(4)),o=c(n(2)),i=c(n(3)),a=c(n(5)),u=c(n(6)),s=c(n(0)),l=n(7);c(n(1)),c(n(12));function c(e){return e&&e.__esModule?e:{default:e}}var f=function(e){function t(){var e,n,i,u;(0,o.default)(this,t);for(var s=arguments.length,l=Array(s),c=0;c=e.length?(this._t=void 0,o(1)):o(0,"keys"==t?n:"values"==t?e[n]:[n,e[n]])},"values"),i.Arguments=i.Array,r("keys"),r("values"),r("entries")},function(e,t){e.exports=function(){}},function(e,t){e.exports=function(e,t){return{value:t,done:!!e}}},function(e,t,n){"use strict";var r=n(160),o=n(95),i=n(97),a={};n(50)(a,n(19)("iterator"),function(){return this}),e.exports=function(e,t,n){e.prototype=r(a,{next:o(1,n)}),i(e,t+" Iterator")}},function(e,t,n){var r=n(40),o=n(36),i=n(96);e.exports=n(44)?Object.defineProperties:function(e,t){o(e);for(var n,a=i(t),u=a.length,s=0;u>s;)r.f(e,n=a[s++],t[n]);return e}},function(e,t,n){var r=n(71),o=n(115),i=n(455);e.exports=function(e){return function(t,n,a){var u,s=r(t),l=o(s.length),c=i(a,l);if(e&&n!=n){for(;l>c;)if((u=s[c++])!=u)return!0}else for(;l>c;c++)if((e||c in s)&&s[c]===n)return e||c||0;return!e&&-1}}},function(e,t,n){var r=n(161),o=Math.max,i=Math.min;e.exports=function(e,t){return(e=r(e))<0?o(e+t,0):i(e,t)}},function(e,t,n){var r=n(161),o=n(156);e.exports=function(e){return function(t,n){var i,a,u=String(o(t)),s=r(n),l=u.length;return s<0||s>=l?e?"":void 0:(i=u.charCodeAt(s))<55296||i>56319||s+1===l||(a=u.charCodeAt(s+1))<56320||a>57343?e?u.charAt(s):i:e?u.slice(s,s+2):a-56320+(i-55296<<10)+65536}}},function(e,t,n){var r=n(36),o=n(165);e.exports=n(15).getIterator=function(e){var t=o(e);if("function"!=typeof t)throw TypeError(e+" is not iterable!");return r(t.call(e))}},function(e,t,n){n(459),n(245),n(470),n(474),n(485),n(486),e.exports=n(61).Promise},function(e,t,n){"use strict";var r=n(167),o={};o[n(18)("toStringTag")]="z",o+""!="[object z]"&&n(73)(Object.prototype,"toString",function(){return"[object "+r(this)+"]"},!0)},function(e,t,n){e.exports=!n(100)&&!n(101)(function(){return 7!=Object.defineProperty(n(169)("div"),"a",{get:function(){return 7}}).a})},function(e,t,n){var r=n(74);e.exports=function(e,t){if(!r(e))return e;var n,o;if(t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;if("function"==typeof(n=e.valueOf)&&!r(o=n.call(e)))return o;if(!t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;throw TypeError("Can't convert object to primitive value")}},function(e,t,n){"use strict";var r=n(463),o=n(244),i=n(171),a={};n(59)(a,n(18)("iterator"),function(){return this}),e.exports=function(e,t,n){e.prototype=r(a,{next:o(1,n)}),i(e,t+" Iterator")}},function(e,t,n){var r=n(60),o=n(464),i=n(251),a=n(170)("IE_PROTO"),u=function(){},s=function(){var e,t=n(169)("iframe"),r=i.length;for(t.style.display="none",n(252).appendChild(t),t.src="javascript:",(e=t.contentWindow.document).open(),e.write(" + + + + diff --git a/swagger/oauth2-redirect.html b/swagger/oauth2-redirect.html new file mode 100644 index 00000000..fb68399d --- /dev/null +++ b/swagger/oauth2-redirect.html @@ -0,0 +1,67 @@ + + + + + + From d3543e31c058b3d96e9742e8e3173bd451cb9205 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 13:32:30 -0700 Subject: [PATCH 176/323] -> Delete .pycs because they make problems --- shell/install-deps.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/shell/install-deps.sh b/shell/install-deps.sh index 1691e936..7edd14b5 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -3,16 +3,16 @@ if [ -n "$CI" ]; then export HOMEROOT=/home/circleci/${CIRCLE_PROJECT_REPONAME} # Clone dependencies git clone -b master https://github.com/isb-cgc/ISB-CGC-Common.git - - # Remove .pyc files; these can sometimes stick around and if a - # model has changed names it will cause various load failures - find . -type f -name '*.pyc' -delete else export $(cat /home/vagrant/API/.env | grep -v ^# | xargs) 2> /dev/null export HOME=/home/vagrant export HOMEROOT=/home/vagrant/API fi +# Remove .pyc files; these can sometimes stick around and if a +# model has changed names it will cause various load failures +find . -type f -name '*.pyc' -delete + export DEBIAN_FRONTEND=noninteractive # Install and update apt-get info From ad29c13361e9b23068ff6b6cfce2f12b0aaeca67 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 14:17:53 -0700 Subject: [PATCH 177/323] -> Deal with static URL issues (of which there are many) --- apiv4/__init__.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index c444e95b..73c8193b 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -19,17 +19,17 @@ import logging import os import sys -from flask import Flask, jsonify, request +from flask import Flask, jsonify, request, render_template, url_for as flask_url_for from flask_cors import cross_origin from flask_talisman import Talisman +app = Flask(__name__, static_folder='api_static') +Talisman(app, strict_transport_security_max_age=300) + import django django.setup() from django.conf import settings -app = Flask(__name__, static_url_path=settings.STATIC_URL, static_folder='api_static') -Talisman(app, strict_transport_security_max_age=300) - from auth import auth_info from main_routes import * from cohorts_routes import * @@ -41,6 +41,19 @@ logger = logging.getLogger(settings.LOGGER_NAME) +def url_for(path, **kwargs): + url = flask_url_for(path, **kwargs) + if url.startswith("/static"): + url = url.replace("/static", "", 1) + return settings.STATIC_URL + url + return url + + +# Swagger UI +@app.route('/apiv4/swagger/', methods=['GET'], strict_slashes=False) +def swagger(): + render_template() + # Error handlers @app.errorhandler(500) def unexpected_error(e): From 5271a9b13261376365f3a4edf513e8e840af1116 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 14:20:18 -0700 Subject: [PATCH 178/323] -> Proper structure --- apiv4/__init__.py | 2 +- {swagger => apiv4/templates/swagger}/index.html | 0 {swagger => apiv4/templates/swagger}/oauth2-redirect.html | 0 3 files changed, 1 insertion(+), 1 deletion(-) rename {swagger => apiv4/templates/swagger}/index.html (100%) rename {swagger => apiv4/templates/swagger}/oauth2-redirect.html (100%) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 73c8193b..049b09e8 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -52,7 +52,7 @@ def url_for(path, **kwargs): # Swagger UI @app.route('/apiv4/swagger/', methods=['GET'], strict_slashes=False) def swagger(): - render_template() + render_template('swagger/index.html') # Error handlers @app.errorhandler(500) diff --git a/swagger/index.html b/apiv4/templates/swagger/index.html similarity index 100% rename from swagger/index.html rename to apiv4/templates/swagger/index.html diff --git a/swagger/oauth2-redirect.html b/apiv4/templates/swagger/oauth2-redirect.html similarity index 100% rename from swagger/oauth2-redirect.html rename to apiv4/templates/swagger/oauth2-redirect.html From 1814f818f7fdbfc7add99c7c538c24700215df1e Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 14:52:01 -0700 Subject: [PATCH 179/323] -> Proper structure --- apiv4/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 049b09e8..8cce5400 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -52,7 +52,8 @@ def url_for(path, **kwargs): # Swagger UI @app.route('/apiv4/swagger/', methods=['GET'], strict_slashes=False) def swagger(): - render_template('swagger/index.html') + return render_template('swagger/index.html') + # Error handlers @app.errorhandler(500) From 31f62a7e74b6f60c8990eb474323aadd8e6e55e7 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 15:42:46 -0700 Subject: [PATCH 180/323] -> No need to support POST on a fundamentally GET operation --- apiv4/__init__.py | 6 ++++-- apiv4/main_routes.py | 4 +--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 8cce5400..131a8326 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -59,10 +59,12 @@ def swagger(): @app.errorhandler(500) def unexpected_error(e): """Handle exceptions by returning swagger-compliant json.""" - logging.exception('An error occured while processing the request.') + logging.error('[ERROR] An error occurred while processing the request:') + logger.exeception(e) response = jsonify({ 'code': 500, - 'message': 'Exception: {}'.format(e)}) + 'message': 'Exception: {}'.format(e) + }) response.status_code = 500 return response diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py index c9c95abf..8005fd0a 100644 --- a/apiv4/main_routes.py +++ b/apiv4/main_routes.py @@ -25,7 +25,7 @@ logger = logging.getLogger(settings.LOGGER_NAME) -@app.route('/apiv4/', methods=['GET', 'POST'], strict_slashes=False) +@app.route('/apiv4/', methods=['GET'], strict_slashes=False) def apiv4(): """Base response""" response = jsonify({ @@ -35,5 +35,3 @@ def apiv4(): }) response.status_code = 200 return response - - From 95e6d333539a07d7a13c7bd73c978ea080d9d783 Mon Sep 17 00:00:00 2001 From: s-paquett Date: Tue, 7 May 2019 15:48:47 -0700 Subject: [PATCH 181/323] -> Another static fix... --- apiv4/__init__.py | 11 ++++------- apiv4/templates/swagger/index.html | 10 +++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/apiv4/__init__.py b/apiv4/__init__.py index 131a8326..5bec8174 100644 --- a/apiv4/__init__.py +++ b/apiv4/__init__.py @@ -19,7 +19,7 @@ import logging import os import sys -from flask import Flask, jsonify, request, render_template, url_for as flask_url_for +from flask import Flask, jsonify, request, render_template from flask_cors import cross_origin from flask_talisman import Talisman @@ -41,12 +41,9 @@ logger = logging.getLogger(settings.LOGGER_NAME) -def url_for(path, **kwargs): - url = flask_url_for(path, **kwargs) - if url.startswith("/static"): - url = url.replace("/static", "", 1) - return settings.STATIC_URL + url - return url +@app.context_processor +def static_uri(): + return dict(static_uri=settings.STATIC_URL) # Swagger UI diff --git a/apiv4/templates/swagger/index.html b/apiv4/templates/swagger/index.html index 9ec7ac7b..53fe8bc3 100644 --- a/apiv4/templates/swagger/index.html +++ b/apiv4/templates/swagger/index.html @@ -4,9 +4,9 @@ Swagger UI - - - + + +