diff --git a/.circleci/config.yml b/.circleci/config.yml index 49e673c2..4a099676 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,12 +1,84 @@ -version: 2 +version: 2.1 +commands: + deployment_setup: + steps: + - run: + name: "Set the Tier" + command: | + TIER=DEV + if [[ ${CIRCLE_BRANCH} =~ (prod|uat|test).* ]]; then + TIER=$(awk -F[\-_] '{print toupper($1)}' \<<< ${CIRCLE_BRANCH}) + fi + echo "export TIER=${TIER}" >> $BASH_ENV + echo "Tier was identified as ${TIER} for branch ${CIRCLE_BRANCH}" + - run: + name: "Assign Project-level vars" + command: | + if [ ! -f deployment.key.json ]; then + echo "Deployment JSON keyfile not found - loading from CircleCI." + KEY=${DEPLOYMENT_KEY_ISB_CGC} + else + echo "Deployment JSON keyfile found." + KEY="NA" + fi + CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_ISB_CGC} + PROJECT_ID=${DEPLOYMENT_PROJECT_ID_ISB_CGC} + BUCKET=${DEPLOYMENT_BUCKET_ISB_CGC} + if [[ ${TIER} == "PROD" ]] || [[ ${TIER} == "DEV" ]]; then + BUCKET="${BUCKET}/${TIER,,}" + echo "Using default project ${PROJECT_ID} and related deployment SA and bucket." + elif [[ ${TIER} == "UAT" ]]; then + KEY=${DEPLOYMENT_KEY_ISB_CGC_UAT} + CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_ISB_CGC_UAT} + PROJECT_ID=${DEPLOYMENT_PROJECT_ID_ISB_CGC_UAT} + BUCKET=${DEPLOYMENT_BUCKET_ISB_CGC_UAT} + elif [[ ${TIER} == "TEST" ]]; then + KEY=${DEPLOYMENT_KEY_ISB_CGC_TEST} + CLIENT_EMAIL=${DEPLOYMENT_CLIENT_EMAIL_ISB_CGC_TEST} + PROJECT_ID=${DEPLOYMENT_PROJECT_ID_ISB_CGC_TEST} + BUCKET=${DEPLOYMENT_BUCKET_ISB_CGC_TEST} + else + echo "[ERROR] - Unrecognized tier: ${TIER} - exitng." + exit 1 + fi + + echo "export DEPLOYMENT_KEY=\"${KEY}\"" >> $BASH_ENV + echo "export DEPLOYMENT_CLIENT_EMAIL=${CLIENT_EMAIL}" >> $BASH_ENV + echo "export DEPLOYMENT_PROJECT_ID=${PROJECT_ID}" >> $BASH_ENV + echo "export DEPLOYMENT_BUCKET=${BUCKET}" >> $BASH_ENV + + deployment_config: + steps: + - run: + name: "Set tier-specific configuration file" + command: | + sudo -E /bin/bash ./shell/pull_config.sh + xargs -a deployment_config.txt -I{} echo "export {}" >> $BASH_ENV + + auth: + steps: + - run: + name: "Service Account Auth and Project Settings" + command: | + sudo -E /bin/bash ./shell/gcloud_authenticate.sh + + install_cloud_sdk: + steps: + - run: + name: "Install CloudSDK" + command: | + echo "export CLOUDSDK_CORE_DISABLE_PROMPTS=1" >> $BASH_ENV + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + sudo apt-get update && sudo apt-get -y --allow-downgrades install google-cloud-sdk=251.0.0-0 + jobs: build_job: environment: TZ: "/usr/share/zoneinfo/America/Los_Angeles" - name: Building Deployment Container working_directory: ~/ISB-CGC-API docker: - - image: circleci/python:2.7.14 + - image: circleci/python:3.7.0-stretch - image: circleci/mysql:5.7 environment: MYSQL_ROOT_HOST: "%" @@ -33,30 +105,22 @@ jobs: key: isb-cgc-api-lib-{{ checksum "requirements.txt" }} paths: - ./lib - - save_cache: - key: isb-cgc-api-google-deps-{{ checksum "./shell/install-deps.sh" }} - paths: - - ./google-cloud-sdk - - ./google_appengine + - deployment_setup + - auth + - deployment_config - run: - name: Auth and Staging + name: Staging command: | - sudo -E /bin/sh ./shell/gcloud_authenticate.sh sudo -E /bin/bash ./shell/gcloud-pull-staging-files.sh - save_cache: key: isb-cgc-api-{{ epoch }} paths: - - ./api - - ./api_3 - - ./bq_data_access + - ./apiv4 - ./shell - ./ISB-CGC-Common - - ./saml - - ./lib - - ./appengine_config.py - - ./cgc_api.py + - ./Dockerfile - ./app.yaml - - ./manage.py + - ./openapi-appengine.yaml - ./settings.py - ./txt - ./json @@ -65,9 +129,8 @@ jobs: deploy_job: environment: TZ: "/usr/share/zoneinfo/America/Los_Angeles" - name: Deploy to AppEngine Standard docker: - - image: circleci/python:2.7.14 + - image: circleci/python:3.7.0-stretch working_directory: ~/ISB-CGC-API steps: - restore_cache: @@ -76,11 +139,16 @@ jobs: - restore_cache: keys: - isb-cgc-api-google-deps- + - install_cloud_sdk + - deployment_setup + - auth + - deployment_config - deploy: + nane: Deployment command: | - sudo -E /bin/bash ./shell/gcloud_authenticate.sh sudo -E /bin/bash ./shell/unpack_for_deployment.sh - sudo -E ./google-cloud-sdk/bin/gcloud app deploy --verbosity=debug ./app.yaml --quiet + sudo -E gcloud endpoints services deploy ./openapi-appengine.yaml + sudo -E gcloud app deploy --verbosity=debug ./app.yaml --quiet workflows: version: 2 build_and_deploy: @@ -91,4 +159,8 @@ workflows: - build_job filters: branches: - only: prod + only: + - master + - test + - uat + - prod diff --git a/.gitignore b/.gitignore index cb8ba822..c590b3f3 100644 --- a/.gitignore +++ b/.gitignore @@ -73,19 +73,12 @@ target/ .sass-cache #* *.json -!isb_cgc_apiv3_openapiv2.json -!isb_cgc_ccle_apiv3_openapiv2.json -!isb_cgc_target_apiv3_openapiv2.json -!isb_cgc_tcga_apiv3_openapiv2.json -!isb_cgc_apiv3_openapiv3.json -!isb_cgc_ccle_apiv3_openapiv3.json -!isb_cgc_target_apiv3_openapiv3.json -!isb_cgc_tcga_apiv3_openapiv3.json *.patch .env *.env -scripts/metadata_featdef_tables.sql +openapi-appengine.yaml app.yaml +scripts/metadata_featdef_tables.sql # dummy datasets *.tsv diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e2332ce5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,81 @@ +### +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +### + +# Dockerfile extending the generic Python image with application files for a +# single application. +FROM gcr.io/google_appengine/python + +# Create a virtualenv for dependencies. This isolates these packages from +# system-level packages. +# Use -p python3 or -p python3.7 to select python version. Default is version 2. +RUN virtualenv /env -p python3 + +# Setting these environment variables are the same as running +# source /env/bin/activate. +ENV VIRTUAL_ENV /env +ENV PATH /env/bin:$PATH + +RUN apt-get update +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get install -y wget +RUN wget "http://repo.mysql.com/mysql-apt-config_0.8.9-1_all.deb" -P /tmp + +# install lsb-release (a dependency of mysql-apt-config), since dpkg doesn't +# do dependency resolution +RUN apt-get install -y lsb-release +# add a debconf entry to select mysql-5.7 as the server version when we install +# the mysql config package +RUN echo "mysql-apt-config mysql-apt-config/select-server select mysql-5.7" | debconf-set-selections +# having 'selected' mysql-5.7 for 'server', install the mysql config package +RUN echo 'download mysql public build key' +RUN apt-key del 1550412832 +RUN wget -O - -q 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x8C718D3B5072E1F5' | grep -v '>' | grep -v '<' | grep -v '{' > mysql_pubkey.asc +RUN apt-key add mysql_pubkey.asc || exit 1 +RUN dpkg --install /tmp/mysql-apt-config_0.8.9-1_all.deb + +# fetch the updated package metadata (in particular, mysql-server-5.7) +RUN apt-get update + +# aaaand now let's install mysql-server +RUN apt-get install -y mysql-server + +# Get pip3 installed +RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3 + +RUN apt-get -y install build-essential +RUN apt-get -y install --reinstall python-m2crypto python3-crypto +RUN apt-get -y install libxml2-dev libxmlsec1-dev swig +RUN pip3 install pexpect + +RUN apt-get -y install unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl +RUN easy_install -U distribute + +ADD . /app + +# We need to recompile some of the items because of differences in compiler versions +RUN pip3 install -r /app/requirements.txt -t /app/lib/ --upgrade +RUN pip3 install gunicorn==19.9.0 + +# Install Cloud SDK +RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - && apt-get update -y && apt-get install google-cloud-sdk -y +# Install the Python Libraries +RUN apt-get -y install google-cloud-sdk-app-engine-python + +ENV PYTHONPATH=/app:/app/apiv4:/app/lib:/app/ISB-CGC-Common:${PYTHONPATH} + +CMD gunicorn -b :$PORT apiv4:app -w 3 -t 130 diff --git a/Vagrantfile b/Vagrantfile new file mode 100644 index 00000000..3f8edac6 --- /dev/null +++ b/Vagrantfile @@ -0,0 +1,20 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : +Vagrant.configure(2) do |config| + config.vm.box_url = "https://app.vagrantup.com/ubuntu/boxes/xenial64" + config.vm.box = "ubuntu/xenial64" + + # API ports + config.vm.network "forwarded_port", guest: 8090, host: 8090 + config.vm.network "forwarded_port", guest: 9000, host: 9000 + + config.vm.synced_folder ".", "/home/vagrant/API" + config.vm.synced_folder "../", "/home/vagrant/parentDir" + + # Map Common and lib for API + config.vm.synced_folder "../ISB-CGC-Common", "/home/vagrant/API/ISB-CGC-Common" + + config.vm.provision "shell", path: 'shell/install-deps.sh' + config.vm.provision "shell", path: 'shell/vagrant-start-server.sh' + config.vm.provision "shell", path: 'shell/vagrant-set-env.sh' +end \ No newline at end of file diff --git a/api/Cohort.py b/api/Cohort.py deleted file mode 100644 index 3add69bc..00000000 --- a/api/Cohort.py +++ /dev/null @@ -1,1859 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - - - -import logging -import re -import collections -from datetime import datetime -import endpoints -from protorpc import messages, message_types -from protorpc import remote -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -import django -import MySQLdb -import json -from metadata import MetadataItem -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms, Samples, Filters -from bq_data_access.cohort_bigquery import BigQueryCohortSupport -from api_helpers import * - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID -BASE_URL = settings.BASE_URL - -DEFAULT_COHORT_NAME = 'Untitled Cohort' - -IMPORTANT_FEATURES = [ - 'tumor_tissue_site', - 'gender', - 'vital_status', - 'country', - 'Study', - 'age_at_initial_pathologic_diagnosis', - 'TP53', - 'RB1', - 'NF1', - 'APC', - 'CTNNB1', - 'PIK3CA', - 'PTEN', - 'FBXW7', - 'NRAS', - 'ARID1A', - 'CDKN2A', - 'SMAD4', - 'BRAF', - 'NFE2L2', - 'IDH1', - 'PIK3R1', - 'HRAS', - 'EGFR', - 'BAP1', - 'KRAS', - 'DNAseq_data', - 'mirnPlatform', - 'cnvrPlatform', - 'methPlatform', - 'gexpPlatform', - 'rppaPlatform' -] - -BUILTIN_ENDPOINTS_PARAMETERS = [ - 'alt', - 'fields', - 'enum', - 'enumDescriptions', - 'key', - 'oauth_token', - 'prettyPrint', - 'quotaUser', - 'userIp' -] - -MAX_FILTER_VALUE_LENGTH = 496 # max_len of Filter.value is 512, but 507 is too long. 495 is ok. - - -class ReturnJSON(messages.Message): - msg = messages.StringField(1) - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -class Cohort(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - perm = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.IntegerField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - num_patients = messages.StringField(11) - num_samples = messages.StringField(12) - - -class CohortsList(messages.Message): - items = messages.MessageField(Cohort, 1, repeated=True) - count = messages.IntegerField(2) - - -class CohortPatientsSamplesList(messages.Message): - patients = messages.StringField(1, repeated=True) - patient_count = messages.IntegerField(2) - samples = messages.StringField(3, repeated=True) - sample_count = messages.IntegerField(4) - cohort_id = messages.IntegerField(5) - - -class PatientDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - - -class DataDetails(messages.Message): - sample_barcode = messages.StringField(1) - DataCenterName = messages.StringField(2) - DataCenterType = messages.StringField(3) - DataFileName = messages.StringField(4) - DataFileNameKey = messages.StringField(5) - DatafileUploaded = messages.StringField(6) - DataLevel = messages.StringField(7) - Datatype = messages.StringField(8) - GenomeReference = messages.StringField(9) - GG_dataset_id = messages.StringField(10) - GG_readgroupset_id = messages.StringField(11) - Pipeline = messages.StringField(12) - Platform = messages.StringField(13) - platform_full_name = messages.StringField(14) - Project = messages.StringField(15) - Repository = messages.StringField(16) - SDRFFileName = messages.StringField(17) - SecurityProtocol = messages.StringField(18) - CloudStoragePath = messages.StringField(19) - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(MetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - patient = messages.StringField(3) - data_details = messages.MessageField(DataDetails, 4, repeated=True) - data_details_count = messages.IntegerField(5) - error = messages.StringField(6) - - -class DataFileNameKeyList(messages.Message): - datafilenamekeys = messages.StringField(1, repeated=True) - count = messages.IntegerField(2) - - -class GoogleGenomicsItem(messages.Message): - sample_barcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomicsItem, 1, repeated=True) - count = messages.IntegerField(2) - - -class MetadataRangesItem(messages.Message): - age_at_initial_pathologic_diagnosis = messages.IntegerField(1, repeated=True) - age_at_initial_pathologic_diagnosis_lte = messages.IntegerField(2) - age_at_initial_pathologic_diagnosis_gte = messages.IntegerField(3) - - anatomic_neoplasm_subdivision = messages.StringField(4, repeated=True) - - avg_percent_lymphocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_lymphocyte_infiltration_lte = messages.FloatField(6) - avg_percent_lymphocyte_infiltration_gte = messages.FloatField(7) - - avg_percent_monocyte_infiltration = messages.FloatField(8, repeated=True) - avg_percent_monocyte_infiltration_lte = messages.FloatField(9) - avg_percent_monocyte_infiltration_gte = messages.FloatField(10) - - avg_percent_necrosis = messages.FloatField(11, repeated=True) - avg_percent_necrosis_lte = messages.FloatField(12) - avg_percent_necrosis_gte = messages.FloatField(13) - - avg_percent_neutrophil_infiltration = messages.FloatField(14, repeated=True) - avg_percent_neutrophil_infiltration_lte = messages.FloatField(15) - avg_percent_neutrophil_infiltration_gte = messages.FloatField(16) - - avg_percent_normal_cells = messages.FloatField(17, repeated=True) - avg_percent_normal_cells_lte = messages.FloatField(18) - avg_percent_normal_cells_gte = messages.FloatField(19) - - avg_percent_stromal_cells = messages.FloatField(20, repeated=True) - avg_percent_stromal_cells_lte = messages.FloatField(21) - avg_percent_stromal_cells_gte = messages.FloatField(22) - - avg_percent_tumor_cells = messages.FloatField(23, repeated=True) - avg_percent_tumor_cells_lte = messages.FloatField(24) - avg_percent_tumor_cells_gte = messages.FloatField(25) - - avg_percent_tumor_nuclei = messages.FloatField(26, repeated=True) - avg_percent_tumor_nuclei_lte = messages.FloatField(27) - avg_percent_tumor_nuclei_gte = messages.FloatField(28) - - batch_number = messages.IntegerField(29, repeated=True) - bcr = messages.StringField(30, repeated=True) - clinical_M = messages.StringField(31, repeated=True) - clinical_N = messages.StringField(32, repeated=True) - clinical_stage = messages.StringField(33, repeated=True) - clinical_T = messages.StringField(34, repeated=True) - colorectal_cancer = messages.StringField(35, repeated=True) - country = messages.StringField(36, repeated=True) - - days_to_birth = messages.IntegerField(37, repeated=True) - days_to_birth_lte = messages.IntegerField(38) - days_to_birth_gte = messages.IntegerField(39) - - days_to_collection = messages.IntegerField(40, repeated=True) - days_to_collection_lte = messages.IntegerField(41) - days_to_collection_gte = messages.IntegerField(42) - - days_to_death = messages.IntegerField(43, repeated=True) - days_to_death_lte = messages.IntegerField(44) - days_to_death_gte = messages.IntegerField(45) - - days_to_initial_pathologic_diagnosis = messages.IntegerField(46, repeated=True) - days_to_initial_pathologic_diagnosis_lte = messages.IntegerField(47) - days_to_initial_pathologic_diagnosis_gte = messages.IntegerField(48) - - days_to_last_followup = messages.IntegerField(49, repeated=True) - days_to_last_followup_lte = messages.IntegerField(50) - days_to_last_followup_gte = messages.IntegerField(51) - - days_to_submitted_specimen_dx = messages.IntegerField(52, repeated=True) - days_to_submitted_specimen_dx_lte = messages.IntegerField(53) - days_to_submitted_specimen_dx_gte = messages.IntegerField(54) - - ethnicity = messages.StringField(55, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(56, repeated=True) - gender = messages.StringField(57, repeated=True) - - has_Illumina_DNASeq = messages.StringField(58, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(59, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(60, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(61, repeated=True) - has_UNC_GA_RNASeq = messages.StringField(62, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(63, repeated=True) - has_GA_miRNASeq = messages.StringField(64, repeated=True) - has_RPPA = messages.StringField(65, repeated=True) - has_SNP6 = messages.StringField(66, repeated=True) - has_27k = messages.StringField(67, repeated=True) - has_450k = messages.StringField(68, repeated=True) - - height = messages.IntegerField(69, repeated=True) - height_lte = messages.IntegerField(70) - height_gte = messages.IntegerField(71) - - histological_type = messages.StringField(72, repeated=True) - history_of_colon_polyps = messages.StringField(73, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(74, repeated=True) - history_of_prior_malignancy = messages.StringField(75, repeated=True) - hpv_calls = messages.StringField(76, repeated=True) - hpv_status = messages.StringField(77, repeated=True) - icd_10 = messages.StringField(78, repeated=True) - icd_o_3_histology = messages.StringField(79, repeated=True) - icd_o_3_site = messages.StringField(80, repeated=True) - lymphatic_invasion = messages.StringField(81, repeated=True) - lymphnodes_examined = messages.StringField(82, repeated=True) - lymphovascular_invasion_present = messages.StringField(83, repeated=True) - - max_percent_lymphocyte_infiltration = messages.IntegerField(84, repeated=True) - max_percent_lymphocyte_infiltration_lte = messages.IntegerField(85) - max_percent_lymphocyte_infiltration_gte = messages.IntegerField(86) - - max_percent_monocyte_infiltration = messages.IntegerField(87, repeated=True) - max_percent_monocyte_infiltration_lte = messages.IntegerField(88) - max_percent_monocyte_infiltration_gte = messages.IntegerField(89) - - max_percent_necrosis = messages.IntegerField(90, repeated=True) - max_percent_necrosis_lte = messages.IntegerField(91) - max_percent_necrosis_gte = messages.IntegerField(92) - - max_percent_neutrophil_infiltration = messages.IntegerField(93, repeated=True) - max_percent_neutrophil_infiltration_lte = messages.IntegerField(94) - max_percent_neutrophil_infiltration_gte = messages.IntegerField(95) - - max_percent_normal_cells = messages.IntegerField(96, repeated=True) - max_percent_normal_cells_lte = messages.IntegerField(97) - max_percent_normal_cells_gte = messages.IntegerField(98) - - max_percent_stromal_cells = messages.IntegerField(99, repeated=True) - max_percent_stromal_cells_lte = messages.IntegerField(100) - max_percent_stromal_cells_gte = messages.IntegerField(101) - - max_percent_tumor_cells = messages.IntegerField(102, repeated=True) - max_percent_tumor_cells_lte = messages.IntegerField(103) - max_percent_tumor_cells_gte = messages.IntegerField(104) - - max_percent_tumor_nuclei = messages.IntegerField(105, repeated=True) - max_percent_tumor_nuclei_lte = messages.IntegerField(106) - max_percent_tumor_nuclei_gte = messages.IntegerField(107) - - menopause_status = messages.StringField(108, repeated=True) - - min_percent_lymphocyte_infiltration = messages.IntegerField(109, repeated=True) - min_percent_lymphocyte_infiltration_lte = messages.IntegerField(110) - min_percent_lymphocyte_infiltration_gte = messages.IntegerField(111) - - min_percent_monocyte_infiltration = messages.IntegerField(112, repeated=True) - min_percent_monocyte_infiltration_lte = messages.IntegerField(113) - min_percent_monocyte_infiltration_gte = messages.IntegerField(114) - - min_percent_necrosis = messages.IntegerField(115, repeated=True) - min_percent_necrosis_lte = messages.IntegerField(116) - min_percent_necrosis_gte = messages.IntegerField(117) - - min_percent_neutrophil_infiltration = messages.IntegerField(118, repeated=True) - min_percent_neutrophil_infiltration_lte = messages.IntegerField(119) - min_percent_neutrophil_infiltration_gte = messages.IntegerField(120) - - min_percent_normal_cells = messages.IntegerField(121, repeated=True) - min_percent_normal_cells_lte = messages.IntegerField(122) - min_percent_normal_cells_gte = messages.IntegerField(123) - - min_percent_stromal_cells = messages.IntegerField(124, repeated=True) - min_percent_stromal_cells_lte = messages.IntegerField(125) - min_percent_stromal_cells_gte = messages.IntegerField(126) - - min_percent_tumor_cells = messages.IntegerField(127, repeated=True) - min_percent_tumor_cells_lte = messages.IntegerField(128) - min_percent_tumor_cells_gte = messages.IntegerField(129) - - min_percent_tumor_nuclei = messages.IntegerField(130, repeated=True) - min_percent_tumor_nuclei_lte = messages.IntegerField(131) - min_percent_tumor_nuclei_gte = messages.IntegerField(132) - - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(133, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(134, repeated=True) - neoplasm_histologic_grade = messages.StringField(135, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(136, repeated=True) - - number_of_lymphnodes_examined = messages.IntegerField(137, repeated=True) - number_of_lymphnodes_examined_lte = messages.IntegerField(138) - number_of_lymphnodes_examined_gte = messages.IntegerField(139) - - number_of_lymphnodes_positive_by_he = messages.IntegerField(140, repeated=True) - number_of_lymphnodes_positive_by_he_lte = messages.IntegerField(141) - number_of_lymphnodes_positive_by_he_gte = messages.IntegerField(142) - - case_barcode = messages.StringField(143, repeated=True) - pathologic_M = messages.StringField(144, repeated=True) - pathologic_N = messages.StringField(145, repeated=True) - pathologic_stage = messages.StringField(146, repeated=True) - pathologic_T = messages.StringField(147, repeated=True) - person_neoplasm_cancer_status = messages.StringField(148, repeated=True) - pregnancies = messages.StringField(149, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(150, repeated=True) - primary_therapy_outcome_success = messages.StringField(151, repeated=True) - prior_dx = messages.StringField(152, repeated=True) - Project = messages.StringField(153, repeated=True) - - psa_value = messages.FloatField(154, repeated=True) - psa_value_lte = messages.FloatField(155) - psa_value_gte = messages.FloatField(156) - - race = messages.StringField(157, repeated=True) - residual_tumor = messages.StringField(158, repeated=True) - sample_barcode = messages.StringField(159, repeated=True) - SampleTypeCode = messages.StringField(160, repeated=True) - Study = messages.StringField(161, repeated=True) - tobacco_smoking_history = messages.StringField(162, repeated=True) - tumor_tissue_site = messages.StringField(163, repeated=True) - tumor_type = messages.StringField(164, repeated=True) - weiss_venous_invasion = messages.StringField(165, repeated=True) - vital_status = messages.StringField(166, repeated=True) - - weight = messages.IntegerField(167, repeated=True) - weight_lte = messages.IntegerField(168) - weight_gte = messages.IntegerField(169) - - year_of_initial_pathologic_diagnosis = messages.IntegerField(170, repeated=True) - year_of_initial_pathologic_diagnosis_lte = messages.IntegerField(171) - year_of_initial_pathologic_diagnosis_gte = messages.IntegerField(172) - - -def are_there_bad_keys(request): - ''' - Checks for unrecognized fields in an endpoint request - :param request: the request object from the endpoint - :return: boolean indicating True if bad (unrecognized) fields are present in the request - ''' - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - return unrecognized_param_dict != {} - - -def are_there_no_acceptable_keys(request): - """ - Checks for a lack of recognized fields in an endpoints request. Used in save_cohort and preview_cohort endpoints. - :param request: the request object from the endpoint - :return: boolean indicating True if there are no recognized fields in the request. - """ - param_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - } - return param_dict == {} - - -def construct_parameter_error_message(request, filter_required): - err_msg = '' - sorted_acceptable_keys = sorted([k.name for k in request.all_fields()], key=lambda s: s.lower()) - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - if unrecognized_param_dict: - bad_key_str = "'" + "', '".join(unrecognized_param_dict.keys()) + "'" - err_msg += "The following filters were not recognized: {}. ".format(bad_key_str) - if filter_required: - err_msg += "You must specify at least one of the following " \ - "case-sensitive filters: {}".format(sorted_acceptable_keys) - else: - err_msg += "Acceptable filters are: {}".format(sorted_acceptable_keys) - - return err_msg - - -def get_list_of_split_values_for_filter_model(large_value_list): - ''' - :rtype: list - :param large_value_list: protorpc.messages.FieldList - :return: list of smaller protorpc.messages.FieldLists - ''' - - return_list = [] - - # if length_of_list is larger than 512 characters, - # the Filter model will not be able to be saved - # with this as the value field - length_of_list = len('"' + '", "'.join(large_value_list) + '"') - while length_of_list > MAX_FILTER_VALUE_LENGTH: - new_smaller_list = [] - length_of_smaller_list = len('"' + '", "'.join(new_smaller_list) + '"') - while length_of_smaller_list < MAX_FILTER_VALUE_LENGTH: - try: - new_smaller_list.append(large_value_list.pop()) - except IndexError: - break - length_of_smaller_list = len('"' + '", "'.join(new_smaller_list) + '"') - large_value_list.append(new_smaller_list.pop()) - return_list.append(new_smaller_list) - length_of_list = len('"' + '", "'.join(large_value_list) + '"') - - if len(large_value_list): - return_list.append(large_value_list) - - return return_list - - -Cohort_Endpoints = endpoints.api(name='cohort_api', version='v1', - description="Get information about cohorts, patients, and samples. Create and delete cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - - -@Cohort_Endpoints.api_class(resource_name='cohort_endpoints') -class Cohort_Endpoints_API(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(token=messages.StringField(1), cohort_id=messages.IntegerField(2)) - - @endpoints.method(GET_RESOURCE, CohortsList, - path='cohorts_list', http_method='GET', name='cohorts.list') - def cohorts_list(self, request): - ''' - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. Optionally takes a cohort id as a parameter to - only list information about one cohort. - ''' - user_email = None - cursor = None - filter_cursor = None - parent_cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - query_dict = {'cohorts_cohort_perms.user_id': user_id, 'cohorts_cohort.active': unicode('1')} - - if cohort_id: - query_dict['cohorts_cohort.id'] = cohort_id - - query_str = 'select cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'cohorts_cohort_comments.content as comments, ' \ - 'cohorts_source.type as source_type, ' \ - 'cohorts_source.notes as source_notes ' \ - 'from cohorts_cohort_perms ' \ - 'join cohorts_cohort ' \ - 'on cohorts_cohort.id=cohorts_cohort_perms.cohort_id ' \ - 'join auth_user ' \ - 'on auth_user.id=cohorts_cohort_perms.user_id ' \ - 'left join cohorts_cohort_comments ' \ - 'on cohorts_cohort_comments.user_id=cohorts_cohort_perms.user_id ' \ - 'and cohorts_cohort_comments.cohort_id=cohorts_cohort.id ' \ - 'left join cohorts_source ' \ - 'on cohorts_source.cohort_id=cohorts_cohort_perms.cohort_id ' - - - - query_tuple = () - if query_dict: - query_str += ' where ' + '=%s and '.join(key for key in query_dict.keys()) + '=%s ' - query_tuple = tuple(value for value in query_dict.values()) - - query_str += 'group by ' \ - 'cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'comments, ' \ - 'source_type, ' \ - 'source_notes ' - - print query_str - print query_tuple - - filter_query_str = '' - parent_id_query_str = '' - row = None - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - data = [] - - for row in cursor.fetchall(): - filter_query_str = 'SELECT name, value ' \ - 'FROM cohorts_filters ' \ - 'WHERE cohorts_filters.resulting_cohort_id=%s' - - filter_cursor = db.cursor(MySQLdb.cursors.DictCursor) - filter_cursor.execute(filter_query_str, (str(row['id']),)) - filter_data = [] - for filter_row in filter_cursor.fetchall(): - filter_data.append(FilterDetails( - name=str(filter_row['name']), - value=str(filter_row['value']) - )) - - # getting the parent_id is a separate query since a single cohort - # may have multiple parent cohorts - parent_id_query_str = 'SELECT parent_id ' \ - 'FROM cohorts_source ' \ - 'WHERE cohort_id=%s' - - parent_cursor = db.cursor(MySQLdb.cursors.DictCursor) - parent_cursor.execute(parent_id_query_str, (str(row['id']),)) - parent_id_data = [] - for parent_row in parent_cursor.fetchall(): - if row.get('parent_id') is not None: - parent_id_data.append(int(parent_row['parent_id'])) - - data.append(Cohort( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - perm=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=None if row['source_type'] is None else str(row['source_type']), - source_notes=None if row['source_notes'] is None else str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data - )) - - if len(data) == 0: - optional_message = " matching cohort id " + str(cohort_id) if cohort_id is not None else "" - raise endpoints.NotFoundException("{} has no active cohorts{}." - .format(user_email, optional_message)) - return CohortsList(items=data, count=len(data)) - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "User {}'s cohorts not found. {}: {}".format(user_email, type(e), e)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tcohort query: {} {}\n\tfilter query: {} {}\n\tparent id query: {} {}' \ - .format(e, query_str, query_tuple, filter_query_str, str(row['id']), parent_id_query_str, - str(row['id'])) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(msg)) - finally: - if cursor: cursor.close() - if filter_cursor: filter_cursor.close() - if parent_cursor: parent_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - token=messages.StringField(2)) - - @endpoints.method(GET_RESOURCE, CohortPatientsSamplesList, - path='cohort_patients_samples_list', http_method='GET', - name='cohorts.cohort_patients_samples_list') - def cohort_patients_samples_list(self, request): - """ - Takes a cohort id as a required parameter and returns information about the participants - and samples in a particular cohort. Authentication is required. - User must have either READER or OWNER permissions on the cohort. - """ - - db = None - cursor = None - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - cohort_perms_query = "select count(*) from cohorts_cohort_perms where user_id=%s and cohort_id=%s" - cohort_perms_tuple = (user_id, cohort_id) - cohort_query = "select count(*) from cohorts_cohort where id=%s and active=%s" - cohort_tuple = (cohort_id, unicode('0')) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(cohort_perms_query, cohort_perms_tuple) - result = cursor.fetchone() - if int(result['count(*)']) == 0: - error_message = "{} does not have owner or reader permissions on cohort {}.".format(user_email, - cohort_id) - request_finished.send(self) - raise endpoints.ForbiddenException(error_message) - - cursor.execute(cohort_query, cohort_tuple) - result = cursor.fetchone() - if int(result['count(*)']) > 0: - error_message = "Cohort {} was deleted.".format(cohort_id) - request_finished.send(self) - raise endpoints.NotFoundException(error_message) - - except (IndexError, TypeError) as e: - logger.warn(e) - raise endpoints.NotFoundException("Cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tcohort permissions query: {} {}\n\tcohort query: {} {}' \ - .format(e, cohort_perms_query, cohort_perms_tuple, cohort_query, cohort_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving cohorts or cohort permissions. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - patient_query_str = 'select cohorts_samples.case_barcode ' \ - 'from cohorts_samples cs ' \ - 'inner join cohorts_cohort_perms ccp' \ - 'on ccp.cohort_id=cs.cohort_id ' \ - 'inner join cohorts_cohort cc' \ - 'on cs.cohort_id=cc.id ' \ - 'where cs.cohort_id=%s ' \ - 'and ccp.user_id=%s ' \ - 'and cc.active=%s ' \ - 'group by cs.case_barcode ' - - patient_query_tuple = (cohort_id, user_id, unicode('1')) - - sample_query_str = 'select cohorts_samples.sample_barcode ' \ - 'from cohorts_samples ' \ - 'inner join cohorts_cohort_perms ' \ - 'on cohorts_cohort_perms.cohort_id=cohorts_samples.cohort_id ' \ - 'inner join cohorts_cohort ' \ - 'on cohorts_samples.cohort_id=cohorts_cohort.id ' \ - 'where cohorts_samples.cohort_id=%s ' \ - 'and cohorts_cohort_perms.user_id=%s ' \ - 'and cohorts_cohort.active=%s ' \ - 'group by cohorts_samples.sample_barcode ' - - sample_query_tuple = (cohort_id, user_id, unicode('1')) - - try: - db = sql_connection() - - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(patient_query_str, patient_query_tuple) - patient_data = [] - for row in cursor.fetchall(): - patient_data.append(row['case_barcode']) - - cursor.execute(sample_query_str, sample_query_tuple) - sample_data = [] - for row in cursor.fetchall(): - sample_data.append(row['sample_barcode']) - - return CohortPatientsSamplesList(patients=patient_data, - patient_count=len(patient_data), - samples=sample_data, - sample_count=len(sample_data), - cohort_id=int(cohort_id)) - except (IndexError, TypeError) as e: - logger.warn(e) - raise endpoints.NotFoundException("Cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tpatient query: {} {}\n\tsample query: {} {}' \ - .format(e, patient_query_str, patient_query_tuple, sample_query_str, sample_query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving patients or samples. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(patient_barcode=messages.StringField(1, required=True)) - - @endpoints.method(GET_RESOURCE, PatientDetails, - path='patient_details', http_method='GET', name='cohorts.patient_details') - def patient_details(self, request): - """ - Returns information about a specific participant, - including a list of samples and aliquots derived from this patient. - Takes a participant barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - - clinical_cursor = None - sample_cursor = None - aliquot_cursor = None - db = None - - patient_barcode = request.get_assigned_value('patient_barcode') - - clinical_query_str = 'select * ' \ - 'from metadata_clinical ' \ - 'where case_barcode=%s' - - query_tuple = (str(patient_barcode),) - - sample_query_str = 'select sample_barcode ' \ - 'from metadata_biospecimen ' \ - 'where case_barcode=%s' - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where case_barcode=%s ' \ - 'group by AliquotBarcode' - try: - db = sql_connection() - clinical_cursor = db.cursor(MySQLdb.cursors.DictCursor) - clinical_cursor.execute(clinical_query_str, query_tuple) - row = clinical_cursor.fetchone() - - item = MetadataItem( - age_at_initial_pathologic_diagnosis=None if "age_at_initial_pathologic_diagnosis" not in row or row[ - "age_at_initial_pathologic_diagnosis"] is None else int( - row["age_at_initial_pathologic_diagnosis"]), - anatomic_neoplasm_subdivision=str(row["anatomic_neoplasm_subdivision"]), - batch_number=None if "batch_number" not in row or row["batch_number"] is None else int( - row["batch_number"]), - bcr=str(row["bcr"]), - clinical_M=str(row["clinical_M"]), - clinical_N=str(row["clinical_N"]), - clinical_stage=str(row["clinical_stage"]), - clinical_T=str(row["clinical_T"]), - colorectal_cancer=str(row["colorectal_cancer"]), - country=str(row["country"]), - days_to_birth=None if "days_to_birth" not in row or row['days_to_birth'] is None else int( - row["days_to_birth"]), - days_to_death=None if "days_to_death" not in row or row['days_to_death'] is None else int( - row["days_to_death"]), - days_to_initial_pathologic_diagnosis=None if "days_to_initial_pathologic_diagnosis" not in row or row[ - 'days_to_initial_pathologic_diagnosis'] is None else int( - row["days_to_initial_pathologic_diagnosis"]), - days_to_last_followup=None if "days_to_last_followup" not in row or row[ - 'days_to_last_followup'] is None else int( - row["days_to_last_followup"]), - days_to_submitted_specimen_dx=None if "days_to_submitted_specimen_dx" not in row or row[ - 'days_to_submitted_specimen_dx'] is None else int( - row["days_to_submitted_specimen_dx"]), - Study=str(row["Study"]), - ethnicity=str(row["ethnicity"]), - frozen_specimen_anatomic_site=str(row["frozen_specimen_anatomic_site"]), - gender=str(row["gender"]), - height=None if "height" not in row or row['height'] is None else int(row["height"]), - histological_type=str(row["histological_type"]), - history_of_colon_polyps=str(row["history_of_colon_polyps"]), - history_of_neoadjuvant_treatment=str(row["history_of_neoadjuvant_treatment"]), - history_of_prior_malignancy=str(row["history_of_prior_malignancy"]), - hpv_calls=str(row["hpv_calls"]), - hpv_status=str(row["hpv_status"]), - icd_10=str(row["icd_10"]), - icd_o_3_histology=str(row["icd_o_3_histology"]), - icd_o_3_site=str(row["icd_o_3_site"]), - lymphatic_invasion=str(row["lymphatic_invasion"]), - lymphnodes_examined=str(row["lymphnodes_examined"]), - lymphovascular_invasion_present=str(row["lymphovascular_invasion_present"]), - menopause_status=str(row["menopause_status"]), - mononucleotide_and_dinucleotide_marker_panel_analysis_status=str( - row["mononucleotide_and_dinucleotide_marker_panel_analysis_status"]), - mononucleotide_marker_panel_analysis_status=str(row["mononucleotide_marker_panel_analysis_status"]), - neoplasm_histologic_grade=str(row["neoplasm_histologic_grade"]), - new_tumor_event_after_initial_treatment=str(row["new_tumor_event_after_initial_treatment"]), - number_of_lymphnodes_examined=None if "number_of_lymphnodes_examined" not in row or row[ - 'number_of_lymphnodes_examined'] is None else int( - row["number_of_lymphnodes_examined"]), - number_of_lymphnodes_positive_by_he=None if "number_of_lymphnodes_positive_by_he" not in row or row[ - 'number_of_lymphnodes_positive_by_he'] is None else int( - row["number_of_lymphnodes_positive_by_he"]), - case_barcode=str(row["case_barcode"]), - pathologic_M=str(row["pathologic_M"]), - pathologic_N=str(row["pathologic_N"]), - pathologic_stage=str(row["pathologic_stage"]), - pathologic_T=str(row["pathologic_T"]), - person_neoplasm_cancer_status=str(row["person_neoplasm_cancer_status"]), - pregnancies=str(row["pregnancies"]), - primary_neoplasm_melanoma_dx=str(row["primary_neoplasm_melanoma_dx"]), - primary_therapy_outcome_success=str(row["primary_therapy_outcome_success"]), - prior_dx=str(row["prior_dx"]), - Project=str(row["Project"]), - psa_value=None if "psa_value" not in row or row["psa_value"] is None else float(row["psa_value"]), - race=str(row["race"]), - residual_tumor=str(row["residual_tumor"]), - tobacco_smoking_history=str(row["tobacco_smoking_history"]), - tumor_tissue_site=str(row["tumor_tissue_site"]), - tumor_type=str(row["tumor_type"]), - weiss_venous_invasion=str(row["weiss_venous_invasion"]), - vital_status=str(row["vital_status"]), - weight=None if "weight" not in row or row["weight"] is None else int(float(row["weight"])), - year_of_initial_pathologic_diagnosis=str(row["year_of_initial_pathologic_diagnosis"]) - ) - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, query_tuple) - sample_data = [] - for row in sample_cursor.fetchall(): - sample_data.append(row['sample_barcode']) - - aliquot_cursor = db.cursor(MySQLdb.cursors.DictCursor) - aliquot_cursor.execute(aliquot_query_str, query_tuple) - aliquot_data = [] - for row in aliquot_cursor.fetchall(): - aliquot_data.append(row['AliquotBarcode']) - - return PatientDetails(clinical_data=item, samples=sample_data, aliquots=aliquot_data) - except (IndexError, TypeError), e: - logger.info("Patient {} not found. Error: {}".format(patient_barcode, e)) - raise endpoints.NotFoundException("Patient {} not found.".format(patient_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tpatient query: {} {}\n\tsample query: {} {}\n\taliquot query: {} {}' \ - .format(e, clinical_query_str, query_tuple, sample_query_str, query_tuple, - aliquot_query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving patient, sample, or aliquot data. {}".format(msg)) - finally: - if clinical_cursor: clinical_cursor.close() - if sample_cursor: sample_cursor.close() - if aliquot_cursor: aliquot_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, SampleDetails, - path='sample_details', http_method='GET', name='cohorts.sample_details') - def sample_details(self, request): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated patient barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - - biospecimen_cursor = None - aliquot_cursor = None - patient_cursor = None - data_cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - biospecimen_query_str = 'select * ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s' - - query_tuple = (str(sample_barcode),) - extra_query_tuple = query_tuple - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' - - patient_query_str = 'select case_barcode ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s ' - - data_query_str = 'select ' \ - 'sample_barcode, ' \ - 'DataCenterName, ' \ - 'DataCenterType, ' \ - 'DataFileName, ' \ - 'DataFileNameKey, ' \ - 'DatafileUploaded, ' \ - 'DataLevel,' \ - 'Datatype,' \ - 'GenomeReference,' \ - 'GG_dataset_id, ' \ - 'GG_readgroupset_id, ' \ - 'Pipeline,' \ - 'Platform,' \ - 'platform_full_name,' \ - 'Project,' \ - 'Repository,' \ - 'SDRFFileName,' \ - 'SecurityProtocol ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' - - if request.get_assigned_value('platform') is not None: - platform = request.get_assigned_value('platform') - aliquot_query_str += ' and platform=%s ' - data_query_str += ' and platform=%s ' - extra_query_tuple += (str(platform),) - - if request.get_assigned_value('pipeline') is not None: - pipeline = request.get_assigned_value('pipeline') - aliquot_query_str += ' and pipeline=%s ' - data_query_str += ' and pipeline=%s ' - extra_query_tuple += (str(pipeline),) - - aliquot_query_str += ' group by AliquotBarcode' - patient_query_str += ' group by case_barcode' - - try: - db = sql_connection() - biospecimen_cursor = db.cursor(MySQLdb.cursors.DictCursor) - biospecimen_cursor.execute(biospecimen_query_str, query_tuple) - row = biospecimen_cursor.fetchone() - - item = MetadataItem( - avg_percent_lymphocyte_infiltration=None if "avg_percent_lymphocyte_infiltration" not in row or row[ - "avg_percent_lymphocyte_infiltration"] is None else float( - row["avg_percent_lymphocyte_infiltration"]), - avg_percent_monocyte_infiltration=None if "avg_percent_monocyte_infiltration" not in row or row[ - "avg_percent_monocyte_infiltration"] is None else float( - row["avg_percent_monocyte_infiltration"]), - avg_percent_necrosis=None if "avg_percent_necrosis" not in row or row[ - "avg_percent_necrosis"] is None else float( - row["avg_percent_necrosis"]), - avg_percent_neutrophil_infiltration=None if "avg_percent_neutrophil_infiltration" not in row or row[ - "avg_percent_neutrophil_infiltration"] is None else float( - row["avg_percent_neutrophil_infiltration"]), - avg_percent_normal_cells=None if "avg_percent_normal_cells" not in row or row[ - "avg_percent_normal_cells"] is None else float( - row["avg_percent_normal_cells"]), - avg_percent_stromal_cells=None if "avg_percent_stromal_cells" not in row or row[ - "avg_percent_stromal_cells"] is None else float( - row["avg_percent_stromal_cells"]), - avg_percent_tumor_cells=None if "avg_percent_tumor_cells" not in row or row[ - "avg_percent_tumor_cells"] is None else float( - row["avg_percent_tumor_cells"]), - avg_percent_tumor_nuclei=None if "avg_percent_tumor_nuclei" not in row or row[ - "avg_percent_tumor_nuclei"] is None else float( - row["avg_percent_tumor_nuclei"]), - batch_number=None if "batch_number" not in row or row["batch_number"] is None else int( - row["batch_number"]), - bcr=str(row["bcr"]), - days_to_collection=None if "days_to_collection" not in row or row[ - 'days_to_collection'] is None else int( - row["days_to_collection"]), - max_percent_lymphocyte_infiltration=None if "max_percent_lymphocyte_infiltration" not in row or row[ - "max_percent_lymphocyte_infiltration"] is None else int( - row["max_percent_lymphocyte_infiltration"]), # 46) - max_percent_monocyte_infiltration=None if "max_percent_monocyte_infiltration" not in row or row[ - "max_percent_monocyte_infiltration"] is None else int( - row["max_percent_monocyte_infiltration"]), # 47) - max_percent_necrosis=None if "max_percent_necrosis" not in row or row[ - "max_percent_necrosis"] is None else int( - row["max_percent_necrosis"]), # 48) - max_percent_neutrophil_infiltration=None if "max_percent_neutrophil_infiltration" not in row or row[ - "max_percent_neutrophil_infiltration"] is None else int( - row["max_percent_neutrophil_infiltration"]), # 49) - max_percent_normal_cells=None if "max_percent_normal_cells" not in row or row[ - "max_percent_normal_cells"] is None else int( - row["max_percent_normal_cells"]), # 50) - max_percent_stromal_cells=None if "max_percent_stromal_cells" not in row or row[ - "max_percent_stromal_cells"] is None else int( - row["max_percent_stromal_cells"]), # 51) - max_percent_tumor_cells=None if "max_percent_tumor_cells" not in row or row[ - "max_percent_tumor_cells"] is None else int( - row["max_percent_tumor_cells"]), # 52) - max_percent_tumor_nuclei=None if "max_percent_tumor_nuclei" not in row or row[ - "max_percent_tumor_nuclei"] is None else int( - row["max_percent_tumor_nuclei"]), # 53) - min_percent_lymphocyte_infiltration=None if "min_percent_lymphocyte_infiltration" not in row or row[ - "min_percent_lymphocyte_infiltration"] is None else int( - row["min_percent_lymphocyte_infiltration"]), # 55) - min_percent_monocyte_infiltration=None if "min_percent_monocyte_infiltration" not in row or row[ - "min_percent_monocyte_infiltration"] is None else int( - row["min_percent_monocyte_infiltration"]), # 56) - min_percent_necrosis=None if "min_percent_necrosis" not in row or row[ - "min_percent_necrosis"] is None else int( - row["min_percent_necrosis"]), # 57) - min_percent_neutrophil_infiltration=None if "min_percent_neutrophil_infiltration" not in row or row[ - "min_percent_neutrophil_infiltration"] is None else int( - row["min_percent_neutrophil_infiltration"]), # 58) - min_percent_normal_cells=None if "min_percent_normal_cells" not in row or row[ - "min_percent_normal_cells"] is None else int( - row["min_percent_normal_cells"]), # 59) - min_percent_stromal_cells=None if "min_percent_stromal_cells" not in row or row[ - "min_percent_stromal_cells"] is None else int( - row["min_percent_stromal_cells"]), # 60) - min_percent_tumor_cells=None if "min_percent_tumor_cells" not in row or row[ - "min_percent_tumor_cells"] is None else int( - row["min_percent_tumor_cells"]), # 61) - min_percent_tumor_nuclei=None if "min_percent_tumor_nuclei" not in row or row[ - "min_percent_tumor_nuclei"] is None else int( - row["min_percent_tumor_nuclei"]), # 62) - case_barcode=str(row["case_barcode"]), - Project=str(row["Project"]), - sample_barcode=str(row["sample_barcode"]), - Study=str(row["Study"]) - ) - aliquot_cursor = db.cursor(MySQLdb.cursors.DictCursor) - aliquot_cursor.execute(aliquot_query_str, extra_query_tuple) - aliquot_data = [] - for row in aliquot_cursor.fetchall(): - aliquot_data.append(row['AliquotBarcode']) - - patient_cursor = db.cursor(MySQLdb.cursors.DictCursor) - patient_cursor.execute(patient_query_str, query_tuple) - row = patient_cursor.fetchone() - if row is None: - aliquot_cursor.close() - patient_cursor.close() - biospecimen_cursor.close() - db.close() - error_message = "Sample barcode {} not found in metadata_biospecimen table.".format(sample_barcode) - return SampleDetails(biospecimen_data=None, aliquots=[], patient=None, data_details=[], - data_details_count=None, error=error_message) - patient_barcode = str(row["case_barcode"]) - - data_cursor = db.cursor(MySQLdb.cursors.DictCursor) - data_cursor.execute(data_query_str, extra_query_tuple) - data_data = [] - bad_repo_count = 0 - bad_repo_set = set() - for row in data_cursor.fetchall(): - if not row.get('DataFileNameKey'): - continue - if 'controlled' not in str(row['SecurityProtocol']).lower(): - cloud_storage_path = "gs://{}{}".format(settings.OPEN_DATA_BUCKET, row.get('DataFileNameKey')) - else: # not filtering on dbGaP_authorized: - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: # shouldn't ever happen - bad_repo_count += 1 - bad_repo_set.add(row['Repository']) - continue - cloud_storage_path = "gs://{}{}".format(bucket_name, row.get('DataFileNameKey')) - - data_item = DataDetails( - sample_barcode=str(row['sample_barcode']), - DataCenterName=str(row['DataCenterName']), - DataCenterType=str(row['DataCenterType']), - DataFileName=str(row['DataFileName']), - DataFileNameKey=str(row.get('DataFileNameKey')), - DatafileUploaded=str(row['DatafileUploaded']), - DataLevel=str(row['DataLevel']), - Datatype=str(row['Datatype']), - GenomeReference=str(row['GenomeReference']), - GG_dataset_id=str(row['GG_dataset_id']), - GG_readgroupset_id=str(row['GG_readgroupset_id']), - Pipeline=str(row['Pipeline']), - Platform=str(row['Platform']), - platform_full_name=str(row['platform_full_name']), - Project=str(row['Project']), - Repository=str(row['Repository']), - SDRFFileName=str(row['SDRFFileName']), - SecurityProtocol=str(row['SecurityProtocol']), - CloudStoragePath=cloud_storage_path - ) - data_data.append(data_item) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return SampleDetails(biospecimen_data=item, aliquots=aliquot_data, - patient=patient_barcode, data_details=data_data, - data_details_count=len(data_data)) - - except (IndexError, TypeError) as e: - logger.info("Sample details for barcode {} not found. Error: {}".format(sample_barcode, e)) - raise endpoints.NotFoundException( - "Sample details for barcode {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tbiospecimen query: {} {}\n\tpatient query: {} {}\n\tdata query: {} {}' \ - .format(e, biospecimen_query_str, query_tuple, patient_query_str, query_tuple, - data_query_str, extra_query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving biospecimen, patient, or other data. {}".format(msg)) - finally: - if biospecimen_cursor: biospecimen_cursor.close() - if aliquot_cursor: aliquot_cursor.close() - if patient_cursor: patient_cursor.close() - if data_cursor: data_cursor.close() - if db and db.open: db.close() - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - limit=messages.IntegerField(2), - platform=messages.StringField(3), - pipeline=messages.StringField(4), - token=messages.StringField(5)) - - @endpoints.method(GET_RESOURCE, DataFileNameKeyList, - path='datafilenamekey_list_from_cohort', http_method='GET', - name='cohorts.datafilenamekey_list_from_cohort') - def datafilenamekey_list_from_cohort(self, request): - """ - Takes a cohort id as a required parameter and returns cloud storage paths to files - associated with all the samples in that cohort, up to a default limit of 10,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - user_email = None - cursor = None - db = None - - limit = request.get_assigned_value('limit') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - cohort_id = request.get_assigned_value('cohort_id') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email: - django.setup() - - query_str = 'SELECT DataFileNameKey, SecurityProtocol, Repository ' \ - 'FROM metadata_data ' - - try: - user_id = Django_User.objects.get(email=user_email).id - django_cohort = Django_Cohort.objects.get(id=cohort_id) - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. Error: {}" \ - .format(user_email, cohort_id, e) - request_finished.send(self) - raise endpoints.UnauthorizedException(err_msg) - - query_str += 'JOIN cohorts_samples ON metadata_data.sample_barcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' \ - 'AND DataFileNameKey != "" AND DataFileNameKey is not null ' - query_tuple = (cohort_id,) - - if platform: - query_str += ' and metadata_data.Platform=%s ' - query_tuple += (platform,) - - if pipeline: - query_str += ' and metadata_data.Pipeline=%s ' - query_tuple += (pipeline,) - - query_str += ' GROUP BY DataFileNameKey, SecurityProtocol, Repository ' - if limit is None: - query_str += ' LIMIT 10000' - else: - query_str += ' LIMIT %s' - query_tuple += (limit,) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - datafilenamekeys = [] - bad_repo_count = 0 - bad_repo_set = set() - for row in cursor.fetchall(): - if not row.get('DataFileNameKey'): - continue - if 'controlled' not in str(row['SecurityProtocol']).lower(): - datafilenamekeys.append( - "gs://{}{}".format(settings.OPEN_DATA_BUCKET, row.get('DataFileNameKey'))) - else: # not filtering on dbGaP_authorized - bucket_name = '' - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: # shouldn't ever happen - bad_repo_count += 1 - bad_repo_set.add(row['Repository']) - continue - datafilenamekeys.append("gs://{}{}".format(bucket_name, row.get('DataFileNameKey'))) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return DataFileNameKeyList(datafilenamekeys=datafilenamekeys, count=len(datafilenamekeys)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\t query: {} {}'.format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, DataFileNameKeyList, - path='datafilenamekey_list_from_sample', http_method='GET', - name='cohorts.datafilenamekey_list_from_sample') - def datafilenamekey_list_from_sample(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - query_str = 'SELECT DataFileNameKey, SecurityProtocol, Repository ' \ - 'FROM metadata_data WHERE sample_barcode=%s ' - - query_tuple = (sample_barcode,) - - if platform: - query_str += ' and Platform=%s ' - query_tuple += (platform,) - - if pipeline: - query_str += ' and Pipeline=%s ' - query_tuple += (pipeline,) - - query_str += ' GROUP BY DataFileNameKey, SecurityProtocol, Repository' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - datafilenamekeys = [] - bad_repo_count = 0 - bad_repo_set = set() - for row in cursor.fetchall(): - if not row.get('DataFileNameKey'): - continue - if 'controlled' not in str(row['SecurityProtocol']).lower(): - datafilenamekeys.append("gs://{}{}".format(settings.OPEN_DATA_BUCKET, row.get('DataFileNameKey'))) - else: # not filtering on dbGaP_authorized - bucket_name = '' - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: # shouldn't ever happen - bad_repo_count += 0 - bad_repo_set.add(row['Repository']) - continue - datafilenamekeys.append("gs://{}{}".format(bucket_name, row.get('DataFileNameKey'))) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return DataFileNameKeyList(datafilenamekeys=datafilenamekeys, count=len(datafilenamekeys)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for sample {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\t query: {} {}'.format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, - name=messages.StringField(2, required=True), - token=messages.StringField(3)) - - @endpoints.method(POST_RESOURCE, Cohort, - path='save_cohort', http_method='POST', name='cohorts.save_cohort') - def save_cohort(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of patients and samples in that cohort. - """ - user_email = None - patient_cursor = None - sample_cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email: - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - request_finished.send(self) - raise endpoints.BadRequestException(err_msg) - - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name is not 'name' and k.name is not 'token' - and not k.name.endswith('_gte') - and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - sample_query_str = 'SELECT sample_barcode, IF(case_barcode="", LEFT(sample_barcode,12), case_barcode) AS case_barcode' \ - 'FROM metadata_samples ' \ - 'WHERE ' - value_tuple = () - - for key, value_list in query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - if isinstance(value_list, collections.Iterable) and "None" in value_list: - value_list.remove("None") - sample_query_str += ' ( {key} is null '.format(key=key) - if len(value_list) > 0: - sample_query_str += ' OR {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ') ' - else: - sample_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} >=%s '.format(key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} <=%s '.format(key) - value_tuple += (value,) - - sample_query_str += ' GROUP BY sample_barcode' - - patient_barcodes = [] - sample_barcodes = [] - try: - db = sql_connection() - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, value_tuple) - for row in sample_cursor.fetchall(): - sample_barcodes.append((row['sample_barcode'],row['case_barcode'],)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("Error retrieving samples or patients") - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tsample query: {} {}' \ - .format(e, sample_query_str, value_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error saving cohort. {}".format(msg)) - finally: - if patient_cursor: patient_cursor.close() - if sample_cursor: sample_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - cohort_name = request.get_assigned_value('name') - - # Validate the cohort name against a whitelist - whitelist = re.compile(WHITELIST_RE, re.UNICODE) - match = whitelist.search(unicode(cohort_name)) - if match: - # XSS risk, log and fail this cohort save - match = whitelist.findall(unicode(cohort_name)) - logger.error('[ERROR] While saving a cohort, saw a malformed name: ' + cohort_name + ', characters: ' + match.__str__()) - raise endpoints.BadRequestException("Your cohort's name contains invalid characters ("+match.__str__()+"); please choose another name.") - - # 1. create new cohorts_cohort with name, active=True, last_date_saved=now - created_cohort = Django_Cohort.objects.create(name=cohort_name, active=True, - last_date_saved=datetime.utcnow()) - created_cohort.save() - - # 2. insert samples into cohort_samples - sample_barcodes = list(set(sample_barcodes)) - sample_list = [Samples(cohort=created_cohort, sample_barcode=sample[0], case_barcode=sample[1]) for sample in sample_barcodes] - Samples.objects.bulk_create(sample_list) - - # 3. Set permission for user to be owner - perm = Cohort_Perms(cohort=created_cohort, user=django_user, perm=Cohort_Perms.OWNER) - perm.save() - - # 4. Create filters applied - filter_data = [] - for key, value_list in query_dict.items(): - for val in value_list: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - for key, val in [(k + '_lte', v) for k, v in lte_query_dict.items()] + [(k + '_gte', v) for k, v in gte_query_dict.items()]: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - # 5. Store cohort to BigQuery - project_id = settings.BQ_PROJECT_ID - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - bcs = BigQueryCohortSupport(project_id, cohort_settings.dataset_id, cohort_settings.table_id) - bcs.add_cohort_with_sample_barcodes(created_cohort.id, sample_barcodes) - - request_finished.send(self) - return Cohort(id=str(created_cohort.id), - name=cohort_name, - last_date_saved=str(datetime.utcnow()), - num_patients=str(len(patient_barcodes)), - num_samples=str(len(sample_barcodes)) - ) - - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - DELETE_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - token=messages.StringField(2)) - - @endpoints.method(DELETE_RESOURCE, ReturnJSON, - path='delete_cohort', http_method='POST', name='cohorts.delete') - def delete_cohort(self, request): - """ - Deletes a cohort. User must have owner permissions on the cohort. - """ - user_email = None - return_message = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email: - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_to_deactivate = Django_Cohort.objects.get(id=cohort_id) - if cohort_to_deactivate.active is True: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - if cohort_perm.perm == 'OWNER': - cohort_to_deactivate.active = False - cohort_to_deactivate.save() - return_message = 'Cohort %d successfully deactivated.' % cohort_id - else: - return_message = 'You do not have owner permission on cohort %d.' % cohort_id - else: - return_message = "Cohort %d was already deactivated." % cohort_id - - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Either cohort %d does not have an entry in the database " - "or you do not have owner or reader permissions on this cohort." % cohort_id) - finally: - request_finished.send(self) - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - return ReturnJSON(msg=return_message) - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem) - - @endpoints.method(POST_RESOURCE, CohortPatientsSamplesList, - path='preview_cohort', http_method='POST', name='cohorts.preview_cohort') - def preview_cohort(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of participant (aka patient) barcodes, and the list of sample barcodes. - Authentication is not required. - """ - # print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - patient_cursor = None - sample_cursor = None - db = None - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - raise endpoints.BadRequestException(err_msg) - - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and not k.name.endswith('_gte') and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - sample_query_str = 'SELECT sample_barcode,IF(case_barcode="", LEFT(sample_barcode,12), case_barcode) AS case_barcode ' \ - 'FROM metadata_samples ' \ - 'WHERE ' - - value_tuple = () - - for key, value_list in query_dict.iteritems(): - - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - if "None" in value_list: - value_list.remove("None") - sample_query_str += ' ( {key} is null '.format(key=key) - if len(value_list) > 0: - sample_query_str += ' OR {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ') ' - else: - sample_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} >=%s '.format(key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} <=%s '.format(key) - value_tuple += (value,) - - sample_query_str += ' GROUP BY sample_barcode' - - patient_barcodes = [] - sample_barcodes = [] - - try: - db = sql_connection() - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, value_tuple) - for row in sample_cursor.fetchall(): - sample_barcodes.append(row['sample_barcode']) - patient_barcodes.append(row['case_barcode']) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("Error retrieving samples or patients: {}".format(e)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tsample query: {} {}' \ - .format(e, sample_query_str, value_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error previewing cohort. {}".format(msg)) - finally: - if sample_cursor: sample_cursor.close() - if db and db.open: db.close() - - return CohortPatientsSamplesList(patients=patient_barcodes, - patient_count=len(patient_barcodes), - samples=sample_barcodes, - sample_count=len(sample_barcodes)) - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - token=messages.StringField(2)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, - # path='google_genomics_from_cohort', http_method='GET', name='cohorts.google_genomics_from_cohort') - def google_genomics_from_cohort(self, request): - """ - Returns a list of Google Genomics dataset and readgroupset ids associated with - all the samples in a specified cohort. - Authentication is required. User must have either READER or OWNER permissions on the cohort. - """ - cursor = None - db = None - user_email = None - cohort_id = request.get_assigned_value('cohort_id') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.get_assigned_value('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - django_cohort = Django_Cohort.objects.get(id=cohort_id) - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. Error: {}" \ - .format(user_email, cohort_id, e) - request_finished.send(self) - raise endpoints.UnauthorizedException(err_msg) - - query_str = 'SELECT sample_barcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'JOIN cohorts_samples ON metadata_data.sample_barcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY sample_barcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (cohort_id,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [] - for row in cursor.fetchall(): - google_genomics_items.append( - GoogleGenomicsItem( - sample_barcode=row['sample_barcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - ) - - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for cohort {} not found." - .format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for cohort. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - else: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, - # path='google_genomics_from_sample', http_method='GET', - # name='cohorts.google_genomics_from_sample') - def google_genomics_from_sample(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - # print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - query_str = 'SELECT sample_barcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE sample_barcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY sample_barcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [] - for row in cursor.fetchall(): - google_genomics_items.append( - GoogleGenomicsItem( - sample_barcode=row['sample_barcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - ) - - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() diff --git a/api/__init__.py b/api/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api/api_helpers.py b/api/api_helpers.py deleted file mode 100755 index 4a2af19d..00000000 --- a/api/api_helpers.py +++ /dev/null @@ -1,508 +0,0 @@ -""" - -Copyright 2018, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import os -import MySQLdb -import httplib2 -from oauth2client.client import GoogleCredentials, AccessTokenCredentials -from django.conf import settings -from googleapiclient.discovery import build - -debug = settings.DEBUG - -WHITELIST_RE = ur'([^\\\_\|\"\+~@:#\$%\^&\*=\-\.,\(\)0-9a-zA-Z\s\xc7\xfc\xe9\xe2\xe4\xe0\xe5\xe7\xea\xeb\xe8\xef\xee\xed\xec\xc4\xc5\xc9\xe6\xc6\xf4\xf6\xf2\xfb\xf9\xd6\xdc\xe1\xf3\xfa\xf1\xd1\xc0\xc1\xc2\xc3\xc8\xca\xcb\xcc\xcd\xce\xcf\xd0\xd2\xd3\xd4\xd5\xd8\xd9\xda\xdb\xdd\xdf\xe3\xf0\xf5\xf8\xfd\xfe\xff])' - -MOLECULAR_CATEGORIES = { - 'nonsilent': [ - 'Missense_Mutation', - 'Nonsense_Mutation', - 'Nonstop_Mutation', - 'Frame_Shift_Del', - 'Frame_Shift_Ins', - 'De_novo_Start_OutOfFrame', - 'In_Frame_Del', - 'In_Frame_Ins', - 'Start_Codon_SNP', - 'Start_Codon_Del', - ] -} - -# Database connection -def sql_connection(): - env = os.getenv('SERVER_SOFTWARE') - database = settings.DATABASES['default'] - connect_options = {} - - db = None - try: - if not settings.IS_DEV: - # Connecting from App Engine - connect_options['host'] = 'localhost' - connect_options['unix_socket'] = database['HOST'] - connect_options['db'] = database['NAME'] - connect_options['user'] = database['USER'] - connect_options['passwd'] = database['PASSWORD'] - else: - connect_options['host'] = '127.0.0.1' - connect_options['port'] = 3306 - connect_options['db'] = database['NAME'] - connect_options['user'] = database['USER'] - connect_options['passwd'] = database['PASSWORD'] - - db = MySQLdb.connect(**connect_options) - - except Exception as e: - print >> sys.stderr, "[ERROR] Exception in sql_connection(): " + e.message - print >> sys.stderr, traceback.format_exc() - - return db - - -def sql_bmi_by_ranges(value): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - if 'None' in value: - result += 'BMI is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == 'underweight': - result += ' (BMI < 18.5)' - elif str(val) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(val) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(val) == 'obese': - result += ' (BMI > 29.9)' - - else: - # value is a single range - if str(value) == 'underweight': - result += ' (BMI < 18.5)' - elif str(value) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(value) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(value) == 'obese': - result += ' (BMI > 29.9)' - - return result - - -def sql_age_by_ranges(value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - #value is a list of ranges - first = True - if 'None' in value: - result += 'age_at_initial_pathologic_diagnosis is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == '10 to 39': - result += ' (age_at_initial_pathologic_diagnosis >= 10 and age_at_initial_pathologic_diagnosis < 40)' - elif str(val) == '40 to 49': - result += ' (age_at_initial_pathologic_diagnosis >= 40 and age_at_initial_pathologic_diagnosis < 50)' - elif str(val) == '50 to 59': - result += ' (age_at_initial_pathologic_diagnosis >= 50 and age_at_initial_pathologic_diagnosis < 60)' - elif str(val) == '60 to 69': - result += ' (age_at_initial_pathologic_diagnosis >= 60 and age_at_initial_pathologic_diagnosis < 70)' - elif str(val) == '70 to 79': - result += ' (age_at_initial_pathologic_diagnosis >= 70 and age_at_initial_pathologic_diagnosis < 80)' - elif str(val).lower() == 'over 80': - result += ' (age_at_initial_pathologic_diagnosis >= 80)' - else: - #value is a single range - if str(value) == '10 to 39': - result += ' (age_at_initial_pathologic_diagnosis >= 10 and age_at_initial_pathologic_diagnosis < 40)' - elif str(value) == '40 to 49': - result += ' (age_at_initial_pathologic_diagnosis >= 40 and age_at_initial_pathologic_diagnosis < 50)' - elif str(value) == '50 to 59': - result += ' (age_at_initial_pathologic_diagnosis >= 50 and age_at_initial_pathologic_diagnosis < 60)' - elif str(value) == '60 to 69': - result += ' (age_at_initial_pathologic_diagnosis >= 60 and age_at_initial_pathologic_diagnosis < 70)' - elif str(value) == '70 to 79': - result += ' (age_at_initial_pathologic_diagnosis >= 70 and age_at_initial_pathologic_diagnosis < 80)' - elif str(value).lower() == 'over 80': - result += ' (age_at_initial_pathologic_diagnosis >= 80)' - elif str(value) == 'None': - result += ' age_at_initial_pathologic_diagnosis is null' - - return result - -def gql_age_by_ranges(q, key, value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - for val in value: - if first: - first = False - else: - result += ' or' - if str(val) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(val) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(val) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(val) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(val) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(val).lower() == 'over80': - result += ' (%s >= 80)' % key - else: - # value is a single range - if str(value) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(value) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(value) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(value) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(value) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(value).lower() == 'over80': - result += ' (%s >= 80)' % key - return result - - -def normalize_bmi(bmis): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - bmi_list = {'underweight': 0, 'normal weight': 0, 'overweight': 0, 'obese': 0, 'None': 0} - for bmi, count in bmis.items(): - if type(bmi) != dict: - if bmi and bmi != 'None': - fl_bmi = float(bmi) - if fl_bmi < 18.5: - bmi_list['underweight'] += int(count) - elif 18.5 <= fl_bmi <= 24.9: - bmi_list['normal weight'] += int(count) - elif 25 <= fl_bmi <= 29.9: - bmi_list['overweight'] += int(count) - elif fl_bmi >= 30: - bmi_list['obese'] += int(count) - else: - bmi_list['None'] += int(count) - - return bmi_list - - -def normalize_ages(ages): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age, count in ages.items(): - if type(age) != dict: - if age and age != 'None': - int_age = float(age) - if int_age < 40: - new_age_list['10 to 39'] += int(count) - elif int_age < 50: - new_age_list['40 to 49'] += int(count) - elif int_age < 60: - new_age_list['50 to 59'] += int(count) - elif int_age < 70: - new_age_list['60 to 69'] += int(count) - elif int_age < 80: - new_age_list['70 to 79'] += int(count) - else: - new_age_list['Over 80'] += int(count) - else: - new_age_list['None'] += int(count) - else: - print age - - return new_age_list - -def applyFilter(field, dict): -# this one gets called a lot... -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - query_dict = dict.copy() - if field in dict: - query_dict.pop(field, None) - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - else: - where_clause = None - else: - where_clause = build_where_clause(dict) - - return where_clause - - -def build_where_clause(filters, alt_key_map=False): -# this one gets called a lot -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - first = True - query_str = '' - big_query_str = '' # todo: make this work for non-string values -- use {}.format - value_tuple = () - key_order = [] - keyType = None - gene = None - - grouped_filters = None - - for key, value in filters.items(): - if isinstance(value, dict) and 'values' in value: - value = value['values'] - - if isinstance(value, list) and len(value) == 1: - value = value[0] - # Check if we need to map to a different column name for a given key - if alt_key_map and key in alt_key_map: - key = alt_key_map[key] - - # Multitable where's will come in with : in the name. Only grab the column piece for now - # TODO: Shouldn't throw away the entire key - elif ':' in key: - keyType = key.split(':')[0] - if keyType == 'MUT': - gene = key.split(':')[1] - key = key.split(':')[-1] - - # Multitable filter lists don't come in as string as they can contain arbitrary text in values - elif isinstance(value, basestring): - # If it's a list of values, split it into an array - if ',' in value: - value = value.split(',') - - key_order.append(key) - - # Bucket the grouped filter types (currently just certain has_ values, could be more) - if 'has_' in key and not key == 'has_Illumina_DNASeq' and not key == 'has_SNP6' and not key == 'has_RPPA': - if grouped_filters is None: - grouped_filters = {} - - if key == 'has_27k' or key == 'has_450k': - if 'DNA_methylation' not in grouped_filters: - grouped_filters['DNA_methylation'] = [] - grouped_filters['DNA_methylation'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_HiSeq_miRnaSeq' or key == 'has_GA_miRNASeq': - if 'miRNA_sequencing' not in grouped_filters: - grouped_filters['miRNA_sequencing'] = [] - grouped_filters['miRNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_UNC_HiSeq_RNASeq' or key == 'has_UNC_GA_RNASeq' or key == 'has_BCGSC_HiSeq_RNASeq' or key == 'has_BCGSC_GA_RNASeq': - if 'RNA_sequencing' not in grouped_filters: - grouped_filters['RNA_sequencing'] = [] - grouped_filters['RNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - # BQ-only format - elif keyType == 'MUT': - # If it's first in the list, don't append an "and" - params = {} - value_tuple += (params,) - - if first: - first = False - else: - big_query_str += ' AND' - - big_query_str += " %s = '{hugo_symbol}' AND " % 'Hugo_Symbol' - params['gene'] = gene - - if(key == 'category'): - if value == 'any': - big_query_str += '%s IS NOT NULL' % 'Variant_Classification' - params['var_class'] = '' - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = MOLECULAR_CATEGORIES[value] - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = value - - if value != 'any': - if isinstance(values, list): - j = 0 - for vclass in values: - if j == 0: - params['var_class'] = "'%s'" % vclass.replace("'", "\\'") - j = 1 - else: - params['var_class'] += ",'%s'" % vclass.replace("'", "\\'") - else: - params['var_class'] = "'%s'" % values.replace("'", "\\'") - - else: - # If it's first in the list, don't append an "and" - if first: - first = False - else: - query_str += ' and' - big_query_str += ' and' - - # If it's age ranges, give it special treament due to normalizations - if key == 'age_at_initial_pathologic_diagnosis': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_age_by_ranges(value) + ') ' - # If it's age ranges, give it special treament due to normalizations - elif key == 'BMI': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_bmi_by_ranges(value) + ') ' - # If it's a list of items for this key, create an or subclause - elif isinstance(value, list): - has_null = False - if 'None' in value: - has_null = True - query_str += ' (%s is null or' % key - big_query_str += ' (%s is null or' % key - value.remove('None') - query_str += ' %s in (' % key - big_query_str += ' %s in (' % key - i = 0 - for val in value: - value_tuple += (val.strip(),) if type(val) is unicode else (val,) - if i == 0: - query_str += '%s' - big_query_str += '"' + str(val) + '"' - i += 1 - else: - query_str += ',%s' - big_query_str += ',' + '"' + str(val) + '"' - query_str += ')' - big_query_str += ')' - if has_null: - query_str += ')' - big_query_str += ')' - - # If it's looking for None values - elif value == 'None': - query_str += ' %s is null' % key - big_query_str += ' %s is null' % key - - # For the general case - else: - if key == 'fl_archive_name': - big_query_str += ' %s like' % key - big_query_str += ' "%' + value + '%"' - elif key == 'fl_data_level': - big_query_str += ' %s=%s' % (key, value) - elif type(value) == bool: - big_query_str += ' %s=%r' % (key, value) - else: - query_str += ' %s=' % key - big_query_str += ' %s=' % key - query_str += '%s' - big_query_str += '"%s"' % value - value_tuple += (value.strip(),) if type(value) is unicode else (value,) - - # Handle our data buckets - if grouped_filters: - for bucket in grouped_filters: - if not query_str == '': - query_str += ' and ' - big_query_str += ' and ' - - query_str += '( ' - big_query_str += '( ' - - first = True - for filter in grouped_filters[bucket]: - if first: - first = False - else: - query_str += ' or ' - big_query_str += ' or ' - - query_str += ' %s=' % filter['filter'] - big_query_str += ' %s=' % filter['filter'] - query_str += '%s' - big_query_str += '"%s"' % filter['value'] - value_tuple += (filter['value'].strip(),) if type(filter['value']) is unicode else (filter['value'],) - - query_str += ' )' - big_query_str += ' )' - - return {'query_str': query_str, 'value_tuple': value_tuple, 'key_order': key_order, 'big_query_str': big_query_str} - - -def possible_future_authorization_function(): - # will put a decorator on this to ensure user has correct authorization before running - # such as if they are dbgap authorized - from oauth2client.client import flow_from_clientsecrets - from oauth2client.file import Storage - from oauth2client import tools - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - flow = flow_from_clientsecrets(settings.CLIENT_SECRETS, scope='https://www.googleapis.com/auth/bigquery') - ## in future, make storage file temporary somehow? - storage = Storage('bigquery_credentials.dat') - credentials = storage.get() - - if credentials is None or credentials.invalid: - credentials = tools.run_flow(flow, storage, tools.argparser.parse_args([])) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - return service - - -def authorize_credentials_with_Google(): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - # credentials = GoogleCredentials.get_application_default().create_scoped(SCOPES) - credentials = GoogleCredentials.from_stream(settings.GOOGLE_APPLICATION_CREDENTIALS).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - if debug: print >> sys.stderr,' big query authorization '+sys._getframe().f_code.co_name - return service - -# TODO refactor to remove duplicate code -def authorize_credentials_with_google_from_file(credentials_path): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - credentials = GoogleCredentials.from_stream(credentials_path).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - - return service - - -def get_user_email_from_token(access_token): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - user_email = None - credentials = AccessTokenCredentials(access_token, 'test-user') - http = credentials.authorize(httplib2.Http()) - user_info_service = build('oauth2', 'v2', http=http) - user_info = user_info_service.userinfo().get().execute() - if 'email' in user_info: - user_email = user_info['email'] - return user_email diff --git a/api/data_access.py b/api/data_access.py deleted file mode 100755 index a482033d..00000000 --- a/api/data_access.py +++ /dev/null @@ -1,542 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import json -import math -import traceback - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import BooleanField, EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.feature_value_types import ValueType, is_log_transformable -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_tcga_only, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess -from bq_data_access.utils import DurationLogged -from bq_data_access.data_access import FeatureIdQueryDescription - -from api.pairwise import PairwiseInputVector, Pairwise -from api.pairwise_api import PairwiseResults, PairwiseResultVector, PairwiseFilterMessage -from api.api_helpers import sql_connection - -from projects.models import Project - -import sys - -logger = logging.getLogger(__name__) - -VIZ_UNIT_DATADICTIONARY = { - 'BMI': 'kg/m^2', -} - -ISB_CGC_PROJECTS = { - 'list': [] -} - -# DUPLICATE METHOD -# Due to the way sql connections are done, it's easiest to duplicate this method and the static variable -# it creates. The original is in Cohorts/views, and all changes will happen there first. -# -# Generate the ISB_CGC_PROJECTS['list'] value set based on the get_isbcgc_project_set sproc -def fetch_isbcgc_project_set(): - try: - cursor = None - db = sql_connection() - if not ISB_CGC_PROJECTS['list'] or len(ISB_CGC_PROJECTS['list']) <= 0: - cursor = db.cursor() - cursor.execute("SELECT COUNT(SPECIFIC_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE SPECIFIC_NAME = 'get_isbcgc_project_set';") - # Only try to fetch the study set if the sproc exists - if cursor.fetchall()[0][0] > 0: - cursor.execute("CALL get_isbcgc_project_set();") - ISB_CGC_PROJECTS['list'] = [] - for row in cursor.fetchall(): - ISB_CGC_PROJECTS['list'].append(row[0]) - else: - # Otherwise just warn - logger.warn("[WARNING] Stored procedure get_isbcgc_project_set was not found!") - - return ISB_CGC_PROJECTS['list'] - except Exception as e: - logger.error(e) - logger.error(traceback.format_exc()) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - -def get_axis_units(xAttr, yAttr): - units = {'x': '', 'y': ''} - - checkUnits = {} - if xAttr is not None: - checkUnits[xAttr] = 'x' - if yAttr is not None: - checkUnits[yAttr] = 'y' - - for attr in checkUnits: - - if '_age' in attr or 'age_' in attr or 'year_' in attr: - units[checkUnits[attr]] = 'years' - elif '_days' in attr or 'days_' in attr: - units[checkUnits[attr]] = 'days' - elif 'percent' in attr: - units[checkUnits[attr]] = 'percent' - elif 'CNVR:' in attr: - units[checkUnits[attr]] = 'log(CN/2)' - elif 'RPPA:' in attr: - units[checkUnits[attr]] = 'protein expression' - elif 'METH:' in attr: - units[checkUnits[attr]] = 'beta value' - elif 'GEXP:' in attr or 'MIRN:' in attr or ('GNAB:' in attr and "num_mutations" in attr): - units[checkUnits[attr]] = 'count' - elif attr.split(':')[1] in VIZ_UNIT_DATADICTIONARY: - units[checkUnits[attr]] = VIZ_UNIT_DATADICTIONARY[attr.split(':')[1]] - - return units - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, required=True) - - -class DataPoint(Message): - patient_id = StringField(1) - sample_id = StringField(2) - aliquot_id = StringField(3) - value = StringField(4) - - -class DataPointList(Message): - type = EnumField(ValueType, 1) - items = MessageField(DataPoint, 2, repeated=True) - - -class PlotDataRequest(Message): - x_id = StringField(1, required=True) - y_id = StringField(2, required=False) - c_id = StringField(3, required=False) - log_transform = StringField(4, required=False) - cohort_id = IntegerField(5, repeated=True) - pairwise = BooleanField(6, required=False) - - -class PlotDataPointCohortMemberships(Message): - ids = IntegerField(1, repeated=True) - - -class PlotDataCohortInfo(Message): - id = IntegerField(1, required=True) - name = StringField(2, required=True) - -DATAPOINT_COHORT_THRESHOLD = 1 - - -class PlotDataPoint(Message): - sample_id = StringField(1) - case_id = StringField(2) - x = StringField(3) - y = StringField(4) - c = StringField(5) - cohort = IntegerField(6, repeated=True) - - -class PlotDataTypes(Message): - x = EnumField(ValueType, 1) - y = EnumField(ValueType, 2) - c = EnumField(ValueType, 3) - - -class PlotDataFeatureLabels(Message): - x = StringField(1) - y = StringField(2) - c = StringField(3) - - -class PlotDatapointCohortSet(Message): - datapoint_id = StringField(1, required=True) - - -class PlotDatapointCount(Message): - total_num_patients = IntegerField(1, required=True) - total_num_samples = IntegerField(2, required=True) - num_patients_w_xy = IntegerField(3, required=True) - num_samples_w_xy = IntegerField(4, required=True) - num_patients_wo_x = IntegerField(5, required=True) - num_samples_wo_x = IntegerField(6, required=True) - num_patients_wo_y = IntegerField(7, required=True) - num_samples_wo_y = IntegerField(8, required=True) - - -class PlotDataResponse(Message): - types = MessageField(PlotDataTypes, 1, required=True) - labels = MessageField(PlotDataFeatureLabels, 2, required=True) - items = MessageField(PlotDataPoint, 3, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - counts = MessageField(PlotDatapointCount, 5) - pairwise_result = MessageField(PairwiseResults, 6, required=False) - xUnits = StringField(7, required=False) - yUnits = StringField(8, required=False) - - -FeatureDataEndpointsAPI = endpoints_api(name='feature_data_api', version='v1', - description='Endpoints for feature data used by the web application.') - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class FeatureDataEndpoints(remote.Service): - def get_counts(self, data): - total_num_patients = [] - total_num_samples = [] - num_samples_w_xy = [] - num_patients_w_xy = [] - num_samples_wo_x = [] - num_samples_wo_y = [] - num_patients_wo_x = [] - num_patients_wo_y = [] - - result = {} - - for item in data: - total_num_samples.append(item['sample_id']) - total_num_patients.append(item['sample_id'][:12]) - - if item['x'] != 'NA' and item['y'] != 'NA': - num_samples_w_xy.append(item['sample_id']) - num_patients_w_xy.append(item['sample_id'][:12]) - else: - if item['x'] == 'NA': - num_samples_wo_x.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_x.append(item['sample_id'][:12]) - elif item['y'] == 'NA': - num_samples_wo_y.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_y.append(item['sample_id'][:12]) - - result['total_num_patients'] = len(set(total_num_patients)) - result['total_num_samples'] = len(set(total_num_samples)) - result['num_patients_w_xy'] = len(set(num_patients_w_xy)) - result['num_samples_w_xy'] = len(set(num_samples_w_xy)) - result['num_patients_wo_x'] = len(set(num_patients_wo_x)) - result['num_samples_wo_x'] = len(set(num_samples_wo_x)) - result['num_patients_wo_y'] = len(set(num_patients_wo_y)) - result['num_samples_wo_y'] = len(set(num_samples_wo_y)) - - return result - - # TODO refactor to separate module - @DurationLogged('PAIRWISE', 'GET') - def get_pairwise_result(self, feature_array): - # Format the feature vectors for pairwise - input_vectors = Pairwise.prepare_feature_vector(feature_array) - outputs = None - results = None - - try: - outputs = Pairwise.run_pairwise(input_vectors) - - if outputs is not None: - results = PairwiseResults(result_vectors=[], filter_messages=[]) - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - except Exception as e: - outputs = None - results = None - logger.error(traceback.format_exc()) - - return results - - @DurationLogged('FEATURE', 'VECTOR_MERGE') - def get_merged_dict_timed(self, vms): - return vms.get_merged_dict() - - # TODO refactor missing value logic out of this module - @DurationLogged('FEATURE', 'GET_VECTORS') - def get_merged_feature_vectors(self, x_id, y_id, c_id, cohort_id_array, logTransform, study_id_array): - """ - Fetches and merges data for two or three feature vectors (see parameter documentation below). - The vectors have to be an array of dictionaries, with each dictionary containing a 'value' field - (other fields are ignored): - [ - { - 'value': 0.5 - }, - { - 'value': 1.0 - } - ] - The merged result: - [ - { - 'patient_id': - 'x': - 'y': - 'c': - }, - { - 'patient_id': - 'x': - 'y': - 'c': - } - ... - ] - - :param x_id: Feature identifier for x-axis e.g. 'CLIN:age_at_initial_pathologic_diagnosis' - :param y_id: Feature identifier for y-axis. If None, values for 'y' in the response will be marked as missing. - :param c_id: Feature identifier for color-by. If None, values for 'c' in the response will be marked as missing. - :param cohort_id_array: Cohort identifier array. - - :return: PlotDataResponse - """ - - async_params = [FeatureIdQueryDescription(x_id, cohort_id_array, study_id_array)] - - c_type, c_vec = ValueType.STRING, [] - y_type, y_vec = ValueType.STRING, [] - - units = get_axis_units(x_id, y_id) - - if c_id is not None: - async_params.append(FeatureIdQueryDescription(c_id, cohort_id_array, study_id_array)) - if y_id is not None: - async_params.append(FeatureIdQueryDescription(y_id, cohort_id_array, study_id_array)) - - async_result = get_feature_vectors_tcga_only(async_params) - - if c_id is not None: - c_type, c_vec = async_result[c_id]['type'], async_result[c_id]['data'] - if y_id is not None: - y_type, y_vec = async_result[y_id]['type'], async_result[y_id]['data'] - if logTransform is not None and logTransform['y'] and y_vec and is_log_transformable(y_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum y-value - ''' - yvals = [] - for yd in y_vec: - if 'value' in yd and yd['value'] is not None and yd['value'] != "NA" and yd['value'] != "None": - yvals.append(float(yd['value'])) - y_min = min(yvals) - ''' - for ydata in y_vec: - if 'value' in ydata and ydata['value'] is not None and ydata['value'] != "NA" and ydata['value'] != "None": - if float(ydata['value']) < 0: - ydata['value'] = "NA" - elif logTransform['yBase'] == 10: - ydata['value'] = str(math.log10((float(ydata['value']) + 1))) - elif logTransform['yBase'] == 'e': - ydata['value'] = str(math.log((float(ydata['value']) + 1))) - elif type(logTransform['yBase']) is int: - ydata['value'] = str(math.log((float(ydata['value']) + 1), logTransform['yBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - x_type, x_vec = async_result[x_id]['type'], async_result[x_id]['data'] - - if logTransform is not None and logTransform['x'] and x_vec and is_log_transformable(x_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum x-value - ''' - xvals = [] - for xd in x_vec: - if 'value' in xd and xd['value'] is not None and xd['value'] != "NA" and xd['value'] != "None": - xvals.append(float(xd['value'])) - x_min = min(xvals) - ''' - - for xdata in x_vec: - if 'value' in xdata and xdata['value'] is not None and xdata['value'] != "NA" and xdata['value'] != "None": - if float(xdata['value']) < 0: - xdata['value'] = "NA" - elif logTransform['xBase'] == 10: - xdata['value'] = str(math.log10((float(xdata['value']) + 1))) - elif logTransform['xBase'] == 'e': - xdata['value'] = str(math.log((float(xdata['value']) + 1))) - elif type(logTransform['xBase']) is int: - xdata['value'] = str(math.log((float(xdata['value']) + 1), logTransform['xBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', ['x', 'y', 'c']) # changed so that it plots per sample not patient - vms.add_dict_array(x_vec, 'x', 'value') - vms.add_dict_array(y_vec, 'y', 'value') - vms.add_dict_array(c_vec, 'c', 'value') - merged = self.get_merged_dict_timed(vms) - - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(id=item['id'], name=item['name'])) - - items = [] - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - - if len(cohort_set) >= DATAPOINT_COHORT_THRESHOLD: - value_bundle['cohort'] = cohort_set - - items.append(PlotDataPoint(**value_bundle)) - - counts = self.get_counts(merged) - count_message = PlotDatapointCount(**counts) - - type_message = PlotDataTypes(x=x_type, y=y_type, c=c_type) - - # TODO assign label for y if y_id is None, as in that case the y-field will be missing from the response - label_message = PlotDataFeatureLabels(x=x_id, y=y_id, c=c_id) - - # TODO Refactor pairwise call to separate function - # Include pairwise results - input_vectors = [PairwiseInputVector(x_id, x_type, x_vec)] - if c_id is not None: - input_vectors.append(PairwiseInputVector(c_id, c_type, c_vec)) - if y_id is not None: - input_vectors.append(PairwiseInputVector(y_id, y_type, y_vec)) - - - pairwise_result = None - - if len(input_vectors) > 1: - pairwise_result = self.get_pairwise_result(input_vectors) - - if pairwise_result is None: - logger.warn("[WARNING] Pairwise results not included in returned object") - - return PlotDataResponse(types=type_message, labels=label_message, items=items, - cohort_set=cohort_info_obj_array, - counts=count_message, pairwise_result=pairwise_result, xUnits=units['x'], yUnits=units['y']) - - def get_feature_id_validity_for_array(self, feature_id_array): - """ - For each feature identifier in an array, check whether or not the identifier is - valid. - - Args: - feature_id_array: - - Returns: - Array of tuples - (feature identifier, ) - """ - result = [] - for feature_id in feature_id_array: - result.append((feature_id, is_valid_feature_identifier(feature_id))) - - return result - - @endpoints_method(PlotDataRequest, PlotDataResponse, - path='feature_data_plot', http_method='GET', name='feature_access.getFeatureDataForPlot') - def data_access_for_plot(self, request): - """ Used by the web application.""" - try: - x_id = request.x_id - y_id = request.y_id - c_id = request.c_id - logTransform = json.loads(request.log_transform) - cohort_id_array = request.cohort_id - - # Check that all requested feature identifiers are valid. Do not check for y_id if it is not - # supplied in the request. - feature_ids_to_check = [x_id] - if c_id is not None: - feature_ids_to_check.append(c_id) - if y_id is not None: - feature_ids_to_check.append(y_id) - - valid_features = self.get_feature_id_validity_for_array(feature_ids_to_check) - - for feature_id, is_valid in valid_features: - logging.info((feature_id, is_valid)) - if not is_valid: - logging.error("Invalid internal feature ID '{}'".format(feature_id)) - raise NotFoundException() - - # Get the project IDs these cohorts' samples come from - cohort_vals = () - cohort_params = "" - - for cohort in cohort_id_array: - cohort_params += "%s," - cohort_vals += (cohort,) - - cohort_params = cohort_params[:-1] - - db = sql_connection() - cursor = db.cursor() - - tcga_studies = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN ("+cohort_params+");",cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_study_ids = [] - unconfirmed_study_ids = [] - - for row in cursor.fetchall(): - if row[0] in tcga_studies: - if row[0] not in confirmed_study_ids: - confirmed_study_ids.append(row[0]) - elif row[0] not in unconfirmed_study_ids: - unconfirmed_study_ids.append(row[0]) - - if len(unconfirmed_study_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_study_ids) - - for project in projects: - if project.get_my_root_and_depth()['root'] in tcga_studies: - confirmed_study_ids.append(project.id) - - return self.get_merged_feature_vectors(x_id, y_id, c_id, cohort_id_array, logTransform, confirmed_study_ids) - except NotFoundException as nfe: - # Pass through NotFoundException so that it is not handled as Exception below. - raise nfe - except Exception as e: - logger.exception(e) - raise InternalServerErrorException() - diff --git a/api/feature_access.py b/api/feature_access.py deleted file mode 100755 index 1e987916..00000000 --- a/api/feature_access.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import BadRequestException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import Message, MessageField, StringField - -from bq_data_access.feature_search.common import BackendException, EmptyQueryException, InvalidFieldException, InvalidDataTypeException -from bq_data_access.gexp_data import GEXP_FEATURE_TYPE -from bq_data_access.clinical_data import CLINICAL_FEATURE_TYPE -from bq_data_access.methylation_data import METH_FEATURE_TYPE -from bq_data_access.copynumber_data import CNVR_FEATURE_TYPE -from bq_data_access.protein_data import RPPA_FEATURE_TYPE -from bq_data_access.mirna_data import MIRN_FEATURE_TYPE -from bq_data_access.gnab_data import GNAB_FEATURE_TYPE -from bq_data_access.feature_search.gexp_searcher import GEXPSearcher -from bq_data_access.feature_search.clinical_searcher import ClinicalSearcher -from bq_data_access.feature_search.methylation_searcher import METHSearcher -from bq_data_access.feature_search.copynumber_search import CNVRSearcher -from bq_data_access.feature_search.protein import RPPASearcher -from bq_data_access.feature_search.microrna_searcher import MIRNSearcher -from bq_data_access.feature_search.gnab_searcher import GNABSearcher - -class ClinicalFeatureType(Message): - feature_type = StringField(1) - gene = StringField(2) - label = StringField(3) - internal_id = StringField(4) - -class FeatureTypesRequest(Message): - keyword = StringField(1, required=True) - -class FeatureDataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = StringField(2, required=True) - -class FeatureTypeSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=False) - gene_name = StringField(3, required=False) - platform = StringField(4, required=False) - center = StringField(5, required=False) - protein_name = StringField(6, required=False) - value_field = StringField(7, required=False) - probe_name = StringField(8, required=False) - relation_to_gene = StringField(9, required=False) - relation_to_island = StringField(10, required=False) - mirna_name = StringField(11, required=False) - -class FeatureSearchResult(Message): - feature_type = StringField(1) - internal_feature_id = StringField(2) - label = StringField(3) - type = StringField(4) - -class FeatureTypeList(Message): - items = MessageField(FeatureSearchResult, 1, repeated=True) - -class FeatureTypeFieldSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=True) - field = StringField(3, required=True) - -class FeatureFieldSearchResult(Message): - values = StringField(1, repeated=True) - - -class FeatureDefinitionSearcherFactory(object): - @classmethod - def build_from_datatype(cls, datatype): - if datatype == CLINICAL_FEATURE_TYPE: - return ClinicalSearcher() - elif datatype == GEXP_FEATURE_TYPE: - return GEXPSearcher() - elif datatype == METH_FEATURE_TYPE: - return METHSearcher() - elif datatype == CNVR_FEATURE_TYPE: - return CNVRSearcher() - elif datatype == RPPA_FEATURE_TYPE: - return RPPASearcher() - elif datatype == MIRN_FEATURE_TYPE: - return MIRNSearcher() - elif datatype == GNAB_FEATURE_TYPE: - return GNABSearcher() - #TODO build a full search on all features - #elif datatype == ALL: - # return FullSearcher() - raise InvalidDataTypeException("Invalid datatype '{datatype}'".format(datatype=datatype)) - -FeatureAccessEndpointsAPI = endpoints_api(name='feature_type_api', version='v1', - description='Endpoints used by the web application to return features.') -@FeatureAccessEndpointsAPI.api_class(resource_name='feature_type_endpoints') -class FeatureAccessEndpoints(remote.Service): - @endpoints_method(FeatureTypeSearchRequest, FeatureTypeList, - path='feature_search', http_method='GET', name='feature_access.FeatureSearch') - def feature_search(self, request): - """ Used by the web application.""" - try: - datatype = request.datatype - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - parameters = {} - for message in request.all_fields(): - field_name = message.name - if field_name != 'datatype': - value = request.get_assigned_value(field_name) - if value is not None: - parameters[field_name] = value - - result = searcher.search(parameters) - items = [] - fields = ['label', 'internal_feature_id', 'feature_type'] - for row in result: - obj = {key: row[key] for key in fields} - if obj['feature_type'] == 'CLIN': - obj['type'] = row['type'] - items.append(obj) - - return FeatureTypeList(items=items) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except EmptyQueryException as e: - logging.error("Empty query: %s", str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error("Invalid field: %s", str(e)) - raise BadRequestException(str(e)) - except BackendException: - logging.exception("feature_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - - @endpoints_method(FeatureTypeFieldSearchRequest, FeatureFieldSearchResult, - path='feature_field_search', http_method='GET', name='feature_access.getFeatureFieldSearch') - def feature_field_search(self, request): - """ Used by the web application.""" - try: - datatype, keyword, field = request.datatype, request.keyword, request.field - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - result = searcher.field_value_search(keyword, field) - return FeatureFieldSearchResult(values=result) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error(str(e)) - raise BadRequestException() - except BackendException: - logging.exception("feature_field_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api/isb_cgc_api/__init__.py b/api/isb_cgc_api/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/api/isb_cgc_api/cohorts_cloudstoragefilepaths.py b/api/isb_cgc_api/cohorts_cloudstoragefilepaths.py deleted file mode 100644 index 849c9477..00000000 --- a/api/isb_cgc_api/cohorts_cloudstoragefilepaths.py +++ /dev/null @@ -1,115 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb -import django - -from django.conf import settings -from django.core.signals import request_finished -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsSamplesFilesQueryBuilder, CohortsSamplesFilesMessageBuilder -from api.api_helpers import sql_connection -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GCSFilePathList(messages.Message): - cloud_storage_file_paths = messages.StringField(1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsCloudStorageFilePathsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True), - limit=messages.IntegerField(2), - platform=messages.StringField(3), - pipeline=messages.StringField(4)) - - @endpoints.method(GET_RESOURCE, GCSFilePathList, http_method='GET', - path='cohorts/{cohort_id}/cloud_storage_file_paths') - def cloud_storage_file_paths(self, request): - """ - Takes a cohort id as a required parameter and returns cloud storage paths to files - associated with all the samples in that cohort, up to a default limit of 10,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - user_email = None - cursor = None - db = None - - limit = request.get_assigned_value('limit') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - cohort_id = request.get_assigned_value('cohort_id') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - Django_Cohort.objects.get(id=cohort_id) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. " \ - "Error: {}".format(user_email, cohort_id, e) - raise endpoints.UnauthorizedException(err_msg) - finally: - request_finished.send(self) - - query_str, query_tuple = CohortsSamplesFilesQueryBuilder().build_query( - platform=platform, pipeline=pipeline, limit=limit, cohort_id=cohort_id) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - cursor_rows = cursor.fetchall() - bad_repo_count, bad_repo_set = CohortsSamplesFilesMessageBuilder().get_GCS_file_paths_and_bad_repos(cursor_rows) - cloud_storage_path_list = [row['cloud_storage_path'] for row in cursor_rows] - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return GCSFilePathList(cloud_storage_file_paths=cloud_storage_path_list, count=len(cloud_storage_path_list)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for cohort {} not found.".format(cohort_id)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving file paths. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_create.py b/api/isb_cgc_api/cohorts_create.py deleted file mode 100644 index f7a2b3e0..00000000 --- a/api/isb_cgc_api/cohorts_create.py +++ /dev/null @@ -1,174 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import re -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages -from datetime import datetime -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsCreatePreviewQueryBuilder, \ - are_there_bad_keys, are_there_no_acceptable_keys, construct_parameter_error_message - -from message_classes import MetadataRangesItem - -from api.api_helpers import sql_connection, WHITELIST_RE -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms, Samples, Filters -from bq_data_access.cohort_bigquery import BigQueryCohortSupport - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -class CreatedCohort(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - filters = messages.MessageField(FilterDetails, 4, repeated=True) - patient_count = messages.IntegerField(5, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(6, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsCreateAPI(remote.Service): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, - name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of patients and the number - of samples in that cohort. - """ - cursor = None - db = None - - user = endpoints.get_current_user() - user_email = user.email() if user else None - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - request_finished.send(self) - raise endpoints.BadRequestException(err_msg) - - query_dict, gte_query_dict, lte_query_dict = CohortsCreatePreviewQueryBuilder().build_query_dictionaries(request) - - patient_query_str, sample_query_str, value_tuple = CohortsCreatePreviewQueryBuilder().build_query( - query_dict, gte_query_dict, lte_query_dict) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, value_tuple) - # TODO: We need to adjust this to pull the correct project ID as well - sample_barcodes = [{'sample_barcode': row['sample_barcode'], 'case_barcode': row['case_barcode'], 'project_id': None,} for row in cursor.fetchall()] - - except (IndexError, TypeError), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("Error retrieving samples or patients") - except MySQLdb.ProgrammingError as e: - logger.warn("Error saving cohort. {}".format(e)) - request_finished.send(self) - raise endpoints.BadRequestException("Error saving cohort. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - cohort_name = request.get_assigned_value('name') - - # Validate the cohort name against a whitelist - whitelist = re.compile(WHITELIST_RE, re.UNICODE) - match = whitelist.search(unicode(cohort_name)) - if match: - # XSS risk, log and fail this cohort save - match = whitelist.findall(unicode(cohort_name)) - logger.error( - '[ERROR] While saving a cohort, saw a malformed name: ' + cohort_name + ', characters: ' + match.__str__()) - raise endpoints.BadRequestException( - "Your cohort's name contains invalid characters (" + match.__str__() + "); please choose another name.") - - if len(sample_barcodes) == 0: - raise endpoints.BadRequestException( - "The cohort could not be saved because no samples meet the specified parameters.") - - # todo: maybe create all objects first, then save them all at the end? - # 1. create new cohorts_cohort with name, active=True, last_date_saved=now - created_cohort = Django_Cohort.objects.create(name=cohort_name, active=True, - last_date_saved=datetime.utcnow()) - created_cohort.save() - - # 2. insert samples into cohort_samples - sample_list = [Samples(cohort=created_cohort, sample_barcode=sample['sample_barcode'], case_barcode=sample['case_barcode'], project_id=sample['project_id']) for sample in sample_barcodes] - Samples.objects.bulk_create(sample_list) - - # 3. Set permission for user to be owner - perm = Cohort_Perms(cohort=created_cohort, user=django_user, perm=Cohort_Perms.OWNER) - perm.save() - - # 4. Create filters applied - filter_data = [] - for key, value_list in query_dict.items(): - for val in value_list: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - for key, val in [(k + '_lte', v) for k, v in lte_query_dict.items()] + [(k + '_gte', v) for k, v in gte_query_dict.items()]: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val).save() - - # 5. Store cohort to BigQuery - project_id = settings.BQ_PROJECT_ID - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - bcs = BigQueryCohortSupport(project_id, cohort_settings.dataset_id, cohort_settings.table_id) - bcs.add_cohort_to_bq(created_cohort.id, sample_barcodes) - - request_finished.send(self) - - return CreatedCohort(id=str(created_cohort.id), - name=cohort_name, - last_date_saved=str(datetime.utcnow()), - filters=filter_data, - patient_count=created_cohort.case_size(), - sample_count=len(sample_barcodes) - ) diff --git a/api/isb_cgc_api/cohorts_delete.py b/api/isb_cgc_api/cohorts_delete.py deleted file mode 100644 index efd6108b..00000000 --- a/api/isb_cgc_api/cohorts_delete.py +++ /dev/null @@ -1,89 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import endpoints -import logging - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class ReturnJSON(messages.Message): - message = messages.StringField(1) - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsDeleteAPI(remote.Service): - DELETE_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - @endpoints.method(DELETE_RESOURCE, ReturnJSON, http_method='DELETE', path='cohorts/{cohort_id}') - def delete(self, request): - """ - Deletes a cohort. User must have owner permissions on the cohort. - """ - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_to_deactivate = Django_Cohort.objects.get(id=cohort_id) - if cohort_to_deactivate.active is True: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - if cohort_perm.perm == 'OWNER': - cohort_to_deactivate.active = False - cohort_to_deactivate.save() - return_message = 'Cohort %d successfully deleted.' % cohort_id - else: - return_message = 'You do not have owner permission on cohort %d.' % cohort_id - else: - return_message = "Cohort %d was already deleted." % cohort_id - return ReturnJSON(message=return_message) - - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Either cohort %d does not have an entry in the database " - "or you do not have owner or reader permissions on this cohort." % cohort_id) - finally: - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_get.py b/api/isb_cgc_api/cohorts_get.py deleted file mode 100644 index 0573f00c..00000000 --- a/api/isb_cgc_api/cohorts_get.py +++ /dev/null @@ -1,161 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import endpoints -import logging -import MySQLdb -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsGetListQueryBuilder, \ - CohortsGetListMessageBuilder, FilterDetails -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class CohortDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - patient_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - patients = messages.StringField(13, repeated=True) - samples = messages.StringField(14, repeated=True) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsGetAPI(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - @endpoints.method(GET_RESOURCE, CohortDetails, http_method='GET', path='cohorts/{cohort_id}') - def get(self, request): - """ - Returns information about a specific cohort the user has READER or OWNER permission on - when given a cohort ID. Authentication is required. - """ - user_email = None - cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - cohort_id = request.get_assigned_value('cohort_id') - query_dict = {'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1'), - 'cohorts_cohort.id': cohort_id} - - query_str, query_tuple = CohortsGetListQueryBuilder().build_cohort_query(query_dict) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - row = cursor.fetchone() - - if row is None: - raise endpoints.NotFoundException( - "Cohort {id} not found. Either it never existed, it was deleted, " - "or {user_email} does not have permission to view it.".format( - id=cohort_id, user_email=user_email)) - - # get the filters used for this cohort - filter_query_str, filter_query_tuple = CohortsGetListQueryBuilder().build_filter_query( - {'cohorts_filters.resulting_cohort_id': str(row['id'])}) - cursor.execute(filter_query_str, filter_query_tuple) - filter_data = CohortsGetListMessageBuilder().make_filter_details_from_cursor( - cursor.fetchall()) - - # getting the parent_id's for this cohort is a separate query - # since a single cohort may have multiple parent cohorts - parent_query_str, parent_query_tuple = CohortsGetListQueryBuilder().build_parent_query( - {'cohort_id': str(row['id'])}) - cursor.execute(parent_query_str, parent_query_tuple) - parent_id_data = CohortsGetListMessageBuilder().make_parent_id_list_from_cursor( - cursor.fetchall(), row) - - # get list of samples and cases in this cohort - sample_query_str, sample_query_tuple = CohortsGetListQueryBuilder().build_samples_query( - {'cohort_id': str(row['id'])}) - cursor.execute(sample_query_str, sample_query_tuple) - sample_list = [] - patient_list = [] - for s_row in cursor.fetchall(): - sample_list.append(s_row['sample_barcode']) - if s_row['case_barcode']: - patient_list.append(s_row['case_barcode']) - - if len(sample_list) == 0: - sample_list = ["None"] - if len(patient_list) == 0: - patient_list = ["None"] - - return CohortDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - patient_count=len(patient_list), - sample_count=len(sample_list), - patients=patient_list, - samples=sample_list - ) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "Cohort {} for user {} not found. {}: {}".format(cohort_id, user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_googlegenomics.py b/api/isb_cgc_api/cohorts_googlegenomics.py deleted file mode 100644 index ec0e6bbb..00000000 --- a/api/isb_cgc_api/cohorts_googlegenomics.py +++ /dev/null @@ -1,127 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb -import django - -from django.conf import settings -from django.core.signals import request_finished -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from api.api_helpers import sql_connection -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsGoogleGenomicssAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='cohorts/{cohort_id}/googlegenomics') - def googlegenomics(self, request): - """ - Returns a list of Google Genomics dataset and readgroupset ids associated with - all the samples in a specified cohort. - Authentication is required. User must have either READER or OWNER permissions on the cohort. - """ - cursor = None - db = None - user_email = None - cohort_id = request.get_assigned_value('cohort_id') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application." - .format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - Django_Cohort.objects.get(id=cohort_id) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}. Error: {}" \ - .format(user_email, cohort_id, e) - request_finished.send(self) - raise endpoints.UnauthorizedException(err_msg) - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'JOIN cohorts_samples ON metadata_data.SampleBarcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (cohort_id,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for cohort {} not found." - .format(cohort_id)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for cohort. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/cohorts_list.py b/api/isb_cgc_api/cohorts_list.py deleted file mode 100644 index e7075d00..00000000 --- a/api/isb_cgc_api/cohorts_list.py +++ /dev/null @@ -1,157 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import django -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages, message_types - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsGetListQueryBuilder, \ - CohortsGetListMessageBuilder, FilterDetails -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class CohortDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - patient_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - - -class CohortDetailsList(messages.Message): - items = messages.MessageField(CohortDetails, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsListAPI(remote.Service): - @endpoints.method(message_types.VoidMessage, CohortDetailsList, http_method='GET', path='cohorts') - def list(self, unused_request): - """ - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. Optionally takes a cohort id as a parameter to - only list information about one cohort. - """ - user_email = None - cursor = None - db = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - - query_str, query_tuple = CohortsGetListQueryBuilder().build_cohort_query({ - 'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1') - }) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - data = [] - - for row in cursor.fetchall(): - # get filters for each cohort - filter_query_str, filter_query_tuple = CohortsGetListQueryBuilder().build_filter_query( - {'cohorts_filters.resulting_cohort_id': str(row['id'])}) - cursor.execute(filter_query_str, filter_query_tuple) - filter_data = CohortsGetListMessageBuilder().make_filter_details_from_cursor( - cursor.fetchall()) - - # getting the parent_id's for each cohort is a separate query - # since a single cohort may have multiple parent cohorts - parent_query_str, parent_query_tuple = CohortsGetListQueryBuilder().build_parent_query( - {'cohort_id': str(row['id'])}) - cursor.execute(parent_query_str, parent_query_tuple) - parent_id_data = CohortsGetListMessageBuilder().make_parent_id_list_from_cursor( - cursor.fetchall(), row) - - # get number of patients for each cohort - patient_query_str, patient_query_tuple = CohortsGetListQueryBuilder().build_patients_query( - {'cohort_id': str(row['id'])}) - cursor.execute(patient_query_str, patient_query_tuple) - patient_count = len(cursor.fetchall()) - - # get number of samples for each cohort - sample_query_str, sample_query_tuple = CohortsGetListQueryBuilder().build_samples_query( - {'cohort_id': str(row['id'])}) - cursor.execute(sample_query_str, sample_query_tuple) - sample_count = len(cursor.fetchall()) - - data.append(CohortDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - patient_count=patient_count, - sample_count=sample_count - )) - - if len(data) == 0: - raise endpoints.NotFoundException("{} has no active cohorts.".format(user_email)) - - return CohortDetailsList(items=data, count=len(data)) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "User {}'s cohorts not found. {}: {}".format(user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api/isb_cgc_api/cohorts_preview.py b/api/isb_cgc_api/cohorts_preview.py deleted file mode 100644 index f272e36c..00000000 --- a/api/isb_cgc_api/cohorts_preview.py +++ /dev/null @@ -1,98 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, CohortsCreatePreviewQueryBuilder, \ - are_there_bad_keys, are_there_no_acceptable_keys, construct_parameter_error_message - -from message_classes import MetadataRangesItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class CohortPatientsSamplesList(messages.Message): - patients = messages.StringField(1, repeated=True) - patient_count = messages.IntegerField(2, variant=messages.Variant.INT32) - samples = messages.StringField(3, repeated=True) - sample_count = messages.IntegerField(4, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsPreviewAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(**{field.name: field for field in MetadataRangesItem.all_fields()}) - - @endpoints.method(GET_RESOURCE, CohortPatientsSamplesList, path='cohorts/preview', http_method='GET') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of participant (aka patient) barcodes, and the list of sample barcodes. - Authentication is not required. - """ - patient_cursor = None - sample_cursor = None - - if are_there_bad_keys(request) or are_there_no_acceptable_keys(request): - err_msg = construct_parameter_error_message(request, True) - raise endpoints.BadRequestException(err_msg) - - query_dict, gte_query_dict, lte_query_dict = CohortsCreatePreviewQueryBuilder().build_query_dictionaries(request) - - patient_query_str, sample_query_str, value_tuple = CohortsCreatePreviewQueryBuilder().build_query( - query_dict, gte_query_dict, lte_query_dict) - - patient_barcodes = [] - sample_barcodes = [] - - try: - db = sql_connection() - patient_cursor = db.cursor(MySQLdb.cursors.DictCursor) - patient_cursor.execute(patient_query_str, value_tuple) - for row in patient_cursor.fetchall(): - patient_barcodes.append(row['case_barcode']) - - sample_cursor = db.cursor(MySQLdb.cursors.DictCursor) - sample_cursor.execute(sample_query_str, value_tuple) - for row in sample_cursor.fetchall(): - sample_barcodes.append(row['sample_barcode']) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("Error retrieving samples or patients: {}".format(e)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tpatient query: {} {}\n\tsample query: {} {}' \ - .format(e, patient_query_str, value_tuple, sample_query_str, value_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error previewing cohort. {}".format(msg)) - finally: - if patient_cursor: patient_cursor.close() - if sample_cursor: sample_cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - return CohortPatientsSamplesList(patients=patient_barcodes, - patient_count=len(patient_barcodes), - samples=sample_barcodes, - sample_count=len(sample_barcodes)) diff --git a/api/isb_cgc_api/isb_cgc_api_helpers.py b/api/isb_cgc_api/isb_cgc_api_helpers.py deleted file mode 100644 index fdc48dca..00000000 --- a/api/isb_cgc_api/isb_cgc_api_helpers.py +++ /dev/null @@ -1,393 +0,0 @@ -import endpoints -from django.conf import settings -from protorpc import messages -import logging - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -BUILTIN_ENDPOINTS_PARAMETERS = [ - 'alt', - 'fields', - 'enum', - 'enumDescriptions', - 'key', - 'oauth_token', - 'prettyPrint', - 'quotaUser', - 'userIp' -] - -ISB_CGC_Endpoints = endpoints.api(name='isb_cgc_api', version='v2', - description="Get information about cohorts, patients, and samples. Create and delete cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v2', - title="ISB-CGC API") - - -def are_there_bad_keys(request): - ''' - Checks for unrecognized fields in an endpoint request - :param request: the request object from the endpoint - :return: boolean indicating True if bad (unrecognized) fields are present in the request - ''' - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - return unrecognized_param_dict != {} - - -def are_there_no_acceptable_keys(request): - """ - Checks for a lack of recognized fields in an endpoints request. Used in save_cohort and preview_cohort endpoints. - :param request: the request object from the endpoint - :return: boolean indicating True if there are no recognized fields in the request. - """ - param_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - } - return param_dict == {} - - -def construct_parameter_error_message(request, filter_required): - err_msg = '' - sorted_acceptable_keys = sorted([k.name for k in request.all_fields()], key=lambda s: s.lower()) - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - if unrecognized_param_dict: - bad_key_str = "'" + "', '".join(unrecognized_param_dict.keys()) + "'" - err_msg += "The following filters were not recognized: {}. ".format(bad_key_str) - if filter_required: - err_msg += "You must specify at least one of the following " \ - "case-sensitive filters: {}".format(sorted_acceptable_keys) - else: - err_msg += "Acceptable filters are: {}".format(sorted_acceptable_keys) - - return err_msg - - -class CohortsGetListQueryBuilder(object): - - def build_cohort_query(self, query_dict): - """ - Builds the query that will select cohort id, name, last_date_saved, - perms, comments, source type, and source notes - :param query_dict: should contain {'cohorts_cohort_perms.user_id': user_id, 'cohorts_cohort.active': unicode('1')} - :return: query_str, query_tuple - """ - query_str = 'SELECT cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'cohorts_cohort_comments.content AS comments, ' \ - 'cohorts_source.type AS source_type, ' \ - 'cohorts_source.notes AS source_notes ' \ - 'FROM cohorts_cohort_perms ' \ - 'JOIN cohorts_cohort ' \ - 'ON cohorts_cohort.id=cohorts_cohort_perms.cohort_id ' \ - 'JOIN auth_user ' \ - 'ON auth_user.id=cohorts_cohort_perms.user_id ' \ - 'LEFT JOIN cohorts_cohort_comments ' \ - 'ON cohorts_cohort_comments.user_id=cohorts_cohort_perms.user_id ' \ - 'AND cohorts_cohort_comments.cohort_id=cohorts_cohort.id ' \ - 'LEFT JOIN cohorts_source ' \ - 'ON cohorts_source.cohort_id=cohorts_cohort_perms.cohort_id ' - - query_tuple = () - if query_dict: - query_str += ' WHERE ' + '=%s and '.join(key for key in query_dict.keys()) + '=%s ' - query_tuple = tuple(value for value in query_dict.values()) - - query_str += 'GROUP BY ' \ - 'cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'comments, ' \ - 'source_type, ' \ - 'source_notes ' - - return query_str, query_tuple - - def build_filter_query(self, filter_query_dict): - """ - Builds the query that selects the filter name and value for a particular cohort - :param filter_query_dict: should be {'cohorts_filters.resulting_cohort_id:': id} - :return: filter_query_str, filter_query_tuple - """ - filter_query_str = 'SELECT name, value ' \ - 'FROM cohorts_filters ' - - filter_query_str += ' WHERE ' + '=%s AND '.join(key for key in filter_query_dict.keys()) + '=%s ' - filter_query_tuple = tuple(value for value in filter_query_dict.values()) - - return filter_query_str, filter_query_tuple - - def build_parent_query(self, parent_query_dict): - """ - Builds the query that selects parent_ids for a particular cohort - :param parent_query_dict: should be {'cohort_id': str(row['id'])} - :return: parent_query_str, parent_query_tuple - """ - parent_query_str = 'SELECT parent_id ' \ - 'FROM cohorts_source ' - parent_query_str += ' WHERE ' + '=%s AND '.join(key for key in parent_query_dict.keys()) + '=%s ' - parent_query_tuple = tuple(value for value in parent_query_dict.values()) - - return parent_query_str, parent_query_tuple - - def build_patients_query(self, patient_query_dict): - """ - Builds the query that selects the case count for a particular cohort - :param patient_query_dict: should be {'cohort_id': str(row['id])} - :return: patient_query_str, patient_query_tuple - """ - patients_query_str = 'SELECT case_barcode ' \ - 'FROM cohorts_samples ' - - patients_query_str += ' WHERE ' + '=%s AND '.join(key for key in patient_query_dict.keys()) + '=%s ' - patient_query_tuple = tuple(value for value in patient_query_dict.values()) - - return patients_query_str, patient_query_tuple - - def build_samples_query(self, sample_query_dict): - """ - Builds the query that selects the sample count for a particular cohort - :param sample_query_dict: should be {'cohort_id': str(row['id])} - :return: sample_query_str, sample_query_tuple - """ - samples_query_str = 'SELECT sample_barcode, case_barcode ' \ - 'FROM cohorts_samples ' - - samples_query_str += ' WHERE ' + '=%s AND '.join(key for key in sample_query_dict.keys()) + '=%s ' - sample_query_tuple = tuple(value for value in sample_query_dict.values()) - - return samples_query_str, sample_query_tuple - - -class CohortsCreatePreviewQueryBuilder(object): - def build_query_dictionaries(self, request): - """ - Builds the query dictionaries for create and preview cohort endpoints. - Returns query_dict, gte_query_dict, lte_query_dict. - """ - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - and k.name is not 'name' - and not k.name.endswith('_gte') - and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - return query_dict, gte_query_dict, lte_query_dict - - def build_query(self, query_dict, gte_query_dict, lte_query_dict): - """ - Builds the queries that selects the patient and sample barcodes - that meet the criteria specified in the request body. - Returns patient query string, sample query string, value tuple. - """ - - patient_query_str = 'SELECT DISTINCT(IF(case_barcode="", LEFT(sample_barcode,12), case_barcode)) ' \ - 'AS case_barcode ' \ - 'FROM metadata_samples ' \ - 'WHERE ' - - sample_query_str = 'SELECT sample_barcode, case_barcode ' \ - 'FROM metadata_samples ' \ - 'WHERE ' - value_tuple = () - - for key, value_list in query_dict.iteritems(): - patient_query_str += ' AND ' if not patient_query_str.endswith('WHERE ') else '' - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - if "None" in value_list: - value_list.remove("None") - patient_query_str += ' ( {key} is null '.format(key=key) - sample_query_str += ' ( {key} is null '.format(key=key) - if len(value_list) > 0: - patient_query_str += ' OR {key} IN ({vals}) '.format( - key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ' OR {key} IN ({vals}) '.format( - key=key, vals=', '.join(['%s'] * len(value_list))) - patient_query_str += ') ' - sample_query_str += ') ' - else: - patient_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - sample_query_str += ' {key} IN ({vals}) '.format(key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - patient_query_str += ' AND ' if not patient_query_str.endswith('WHERE ') else '' - patient_query_str += ' {} >=%s '.format(key) - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} >=%s '.format(key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - patient_query_str += ' AND ' if not patient_query_str.endswith('WHERE ') else '' - patient_query_str += ' {} <=%s '.format(key) - sample_query_str += ' AND ' if not sample_query_str.endswith('WHERE ') else '' - sample_query_str += ' {} <=%s '.format(key) - value_tuple += (value,) - - sample_query_str += ' GROUP BY sample_barcode' - - return patient_query_str, sample_query_str, value_tuple - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -class CohortsGetListMessageBuilder(object): - def make_filter_details_from_cursor(self, filter_cursor_dict): - """ - Returns list of FilterDetails from a dictionary of results - from a filter query. - """ - filter_data = [] - for filter_row in filter_cursor_dict: - filter_data.append(FilterDetails( - name=str(filter_row['name']), - value=str(filter_row['value']) - )) - - if len(filter_data) == 0: - filter_data.append(FilterDetails( - name="None", - value="None" - )) - - return filter_data - - def make_parent_id_list_from_cursor(self, parent_cursor_dict, row): - """ - Returns list of parent_id's from a dictionary of results - from a parent id query. - """ - parent_id_data = [str(p_row['parent_id']) for p_row in parent_cursor_dict if row.get('parent_id')] - if len(parent_id_data) == 0: - parent_id_data.append("None") - - return parent_id_data - - -class CohortsSamplesFilesQueryBuilder(object): - - def build_query(self, platform=None, pipeline=None, limit=None, cohort_id=None, sample_barcode=None): - - query_str = 'SELECT DataFileNameKey, SecurityProtocol, Repository ' \ - 'FROM metadata_data ' - - if cohort_id is None: - query_str += 'WHERE sample_barcode=%s ' - else: - query_str += 'JOIN cohorts_samples ON metadata_data.sample_barcode=cohorts_samples.sample_barcode ' \ - 'WHERE cohorts_samples.cohort_id=%s ' - - query_str += 'AND DataFileNameKey != "" AND DataFileNameKey is not null ' - query_str += ' and metadata_data.Platform=%s ' if platform is not None else '' - query_str += ' and metadata_data.Pipeline=%s ' if pipeline is not None else '' - query_str += ' GROUP BY DataFileNameKey, SecurityProtocol, Repository ' - query_str += ' LIMIT %s' if limit is not None else ' LIMIT 10000' - - query_tuple = (cohort_id,) if cohort_id is not None else (sample_barcode,) - query_tuple += (platform,) if platform is not None else () - query_tuple += (pipeline,) if pipeline is not None else () - query_tuple += (limit,) if limit is not None else () - - return query_str, query_tuple - - -class CohortsSamplesFilesMessageBuilder(object): - - def get_GCS_file_paths_and_bad_repos(self, cursor_rows): - """ - Used in cohorts.datafilenamekeys, samples.datafilenamekeys, samples.get. - Modifies cursor_rows to add the cloud storage path to each row representing a file. - A count of bad repositories and a set of bad repositories is returned in case - there are any errors with the data repository information in the row. - :param cursor_rows: list of dictionaries resulting from a database query. - Each dictionary with the key 'DataFileNameKey' must also have a 'SecurityProtocol' key. - Each dictionary with 'controlled' in the value for 'SecurityProtocol' must also - have the key 'Repository'. - :return: bad_repo_count, bad_repo_set - - """ - bad_repo_count = 0 - bad_repo_set = set() - for row in cursor_rows: - if not row.get('DataFileNameKey'): - continue - - if 'controlled' not in str(row['SecurityProtocol']).lower(): - # this may only be necessary for the vagrant db - path = row.get('DataFileNameKey') if row.get('DataFileNameKey') is None \ - else row.get('DataFileNameKey').replace('gs://' + settings.OPEN_DATA_BUCKET, '') - row['cloud_storage_path'] = "gs://{}{}".format(settings.OPEN_DATA_BUCKET, path) - else: - if row['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif row['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: - bad_repo_count += 1 - bad_repo_set.add(row['Repository']) - continue - # this may only be necessary for the vagrant db - path = row.get('DataFileNameKey') if row.get('DataFileNameKey') is None \ - else row.get('DataFileNameKey').replace('gs://' + bucket_name, '') - - row['cloud_storage_path'] = "gs://{}{}".format(bucket_name, path) - return bad_repo_count, bad_repo_set - - -def build_constructor_dict_for_message(message_class, row): - """ - Takes an instance of a message class and a dictionary of values from a database query - and first validates the values in the dictionary against the message class fields - and then returns a dictionary of all the validated key-value pairs in the database query. - This will only work if the headers in the database query have the same name as the names of - fields in the message class. - """ - constructor_dict = {} - metadata_item_dict = {field.name: field for field in message_class.all_fields()} - for name, field in metadata_item_dict.iteritems(): - if row.get(name) is not None: - try: - field.validate(row[name]) - constructor_dict[name] = row[name] - except messages.ValidationError, e: - constructor_dict[name] = None - logger.warn('{name}: {value} was not validated while constructing kwargs for {message_class}. Error: {e}' - .format(name=name, value=str(row[name]), message_class=str(message_class), e=e)) - else: - constructor_dict[name] = None - - return constructor_dict \ No newline at end of file diff --git a/api/isb_cgc_api/message_classes.py b/api/isb_cgc_api/message_classes.py deleted file mode 100644 index 420b7b6c..00000000 --- a/api/isb_cgc_api/message_classes.py +++ /dev/null @@ -1,350 +0,0 @@ -from protorpc import messages - - -class MetadataRangesItem(messages.Message): - - age_at_initial_pathologic_diagnosis = messages.IntegerField(1, repeated=True, variant=messages.Variant.INT32) - age_at_initial_pathologic_diagnosis_lte = messages.IntegerField(2, variant=messages.Variant.INT32) - age_at_initial_pathologic_diagnosis_gte = messages.IntegerField(3, variant=messages.Variant.INT32) - - anatomic_neoplasm_subdivision = messages.StringField(4, repeated=True) - - avg_percent_lymphocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_lymphocyte_infiltration_lte = messages.FloatField(6) - avg_percent_lymphocyte_infiltration_gte = messages.FloatField(7) - - avg_percent_monocyte_infiltration = messages.FloatField(8, repeated=True) - avg_percent_monocyte_infiltration_lte = messages.FloatField(9) - avg_percent_monocyte_infiltration_gte = messages.FloatField(10) - - avg_percent_necrosis = messages.FloatField(11, repeated=True) - avg_percent_necrosis_lte = messages.FloatField(12) - avg_percent_necrosis_gte = messages.FloatField(13) - - avg_percent_neutrophil_infiltration = messages.FloatField(14, repeated=True) - avg_percent_neutrophil_infiltration_lte = messages.FloatField(15) - avg_percent_neutrophil_infiltration_gte = messages.FloatField(16) - - avg_percent_normal_cells = messages.FloatField(17, repeated=True) - avg_percent_normal_cells_lte = messages.FloatField(18) - avg_percent_normal_cells_gte = messages.FloatField(19) - - avg_percent_stromal_cells = messages.FloatField(20, repeated=True) - avg_percent_stromal_cells_lte = messages.FloatField(21) - avg_percent_stromal_cells_gte = messages.FloatField(22) - - avg_percent_tumor_cells = messages.FloatField(23, repeated=True) - avg_percent_tumor_cells_lte = messages.FloatField(24) - avg_percent_tumor_cells_gte = messages.FloatField(25) - - avg_percent_tumor_nuclei = messages.FloatField(26, repeated=True) - avg_percent_tumor_nuclei_lte = messages.FloatField(27) - avg_percent_tumor_nuclei_gte = messages.FloatField(28) - - batch_number = messages.IntegerField(29, repeated=True, variant=messages.Variant.INT32) - batch_number_lte = messages.IntegerField(30, variant=messages.Variant.INT32) - batch_number_gte = messages.IntegerField(31, variant=messages.Variant.INT32) - - bcr = messages.StringField(32, repeated=True) - - BMI = messages.FloatField(33, repeated=True) - BMI_lte = messages.FloatField(34) - BMI_gte = messages.FloatField(35) - - clinical_M = messages.StringField(36, repeated=True) - clinical_N = messages.StringField(37, repeated=True) - clinical_stage = messages.StringField(38, repeated=True) - clinical_T = messages.StringField(39, repeated=True) - colorectal_cancer = messages.StringField(40, repeated=True) - country = messages.StringField(41, repeated=True) - - days_to_birth = messages.IntegerField(42, repeated=True, variant=messages.Variant.INT32) - days_to_birth_lte = messages.IntegerField(43, variant=messages.Variant.INT32) - days_to_birth_gte = messages.IntegerField(44, variant=messages.Variant.INT32) - - days_to_collection = messages.IntegerField(45, repeated=True, variant=messages.Variant.INT32) - days_to_collection_lte = messages.IntegerField(46, variant=messages.Variant.INT32) - days_to_collection_gte = messages.IntegerField(47, variant=messages.Variant.INT32) - - days_to_death = messages.IntegerField(48, repeated=True, variant=messages.Variant.INT32) - days_to_death_lte = messages.IntegerField(49, variant=messages.Variant.INT32) - days_to_death_gte = messages.IntegerField(50, variant=messages.Variant.INT32) - - days_to_initial_pathologic_diagnosis = messages.IntegerField(51, repeated=True, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_lte = messages.IntegerField(52, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_gte = messages.IntegerField(53, variant=messages.Variant.INT32) - - days_to_last_followup = messages.IntegerField(54, repeated=True, variant=messages.Variant.INT32) - days_to_last_followup_lte = messages.IntegerField(55, variant=messages.Variant.INT32) - days_to_last_followup_gte = messages.IntegerField(56, variant=messages.Variant.INT32) - - days_to_last_known_alive = messages.IntegerField(57, repeated=True, variant=messages.Variant.INT32) - days_to_last_known_alive_lte = messages.IntegerField(58, variant=messages.Variant.INT32) - days_to_last_known_alive_gte = messages.IntegerField(59, variant=messages.Variant.INT32) - - days_to_submitted_specimen_dx = messages.IntegerField(60, repeated=True, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_lte = messages.IntegerField(61, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_gte = messages.IntegerField(62, variant=messages.Variant.INT32) - - ethnicity = messages.StringField(63, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(64, repeated=True) - gender = messages.StringField(65, repeated=True) - - gleason_score_combined = messages.IntegerField(66, repeated=True, variant=messages.Variant.INT32) - gleason_score_combined_lte = messages.IntegerField(67, variant=messages.Variant.INT32) - gleason_score_combined_gte = messages.IntegerField(68, variant=messages.Variant.INT32) - - has_27k = messages.BooleanField(69, repeated=True) - has_450k = messages.BooleanField(70, repeated=True) - has_BCGSC_GA_RNASeq = messages.BooleanField(71, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.BooleanField(72, repeated=True) - has_GA_miRNASeq = messages.BooleanField(73, repeated=True) - has_HiSeq_miRnaSeq = messages.BooleanField(74, repeated=True) - has_Illumina_DNASeq = messages.BooleanField(75, repeated=True) - has_RPPA = messages.BooleanField(76, repeated=True) - has_SNP6 = messages.BooleanField(77, repeated=True) - has_UNC_GA_RNASeq = messages.BooleanField(78, repeated=True) - has_UNC_HiSeq_RNASeq = messages.BooleanField(79, repeated=True) - - height = messages.IntegerField(80, repeated=True, variant=messages.Variant.INT32) - height_lte = messages.IntegerField(81, variant=messages.Variant.INT32) - height_gte = messages.IntegerField(82, variant=messages.Variant.INT32) - - histological_type = messages.StringField(83, repeated=True) - history_of_colon_polyps = messages.StringField(84, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(85, repeated=True) - history_of_prior_malignancy = messages.StringField(86, repeated=True) - hpv_calls = messages.StringField(87, repeated=True) - hpv_status = messages.StringField(88, repeated=True) - icd_10 = messages.StringField(89, repeated=True) - icd_o_3_histology = messages.StringField(90, repeated=True) - icd_o_3_site = messages.StringField(91, repeated=True) - lymphatic_invasion = messages.StringField(92, repeated=True) - lymphnodes_examined = messages.StringField(93, repeated=True) - lymphovascular_invasion_present = messages.StringField(94, repeated=True) - - max_percent_lymphocyte_infiltration = messages.FloatField(95, repeated=True) - max_percent_lymphocyte_infiltration_lte = messages.FloatField(96) - max_percent_lymphocyte_infiltration_gte = messages.FloatField(97) - - max_percent_monocyte_infiltration = messages.FloatField(98, repeated=True) - max_percent_monocyte_infiltration_lte = messages.FloatField(99) - max_percent_monocyte_infiltration_gte = messages.FloatField(100) - - max_percent_necrosis = messages.FloatField(101, repeated=True) - max_percent_necrosis_lte = messages.FloatField(102) - max_percent_necrosis_gte = messages.FloatField(103) - - max_percent_neutrophil_infiltration = messages.FloatField(104, repeated=True) - max_percent_neutrophil_infiltration_lte = messages.FloatField(105) - max_percent_neutrophil_infiltration_gte = messages.FloatField(106) - - max_percent_normal_cells = messages.FloatField(107, repeated=True) - max_percent_normal_cells_lte = messages.FloatField(108) - max_percent_normal_cells_gte = messages.FloatField(109) - - max_percent_stromal_cells = messages.FloatField(110, repeated=True) - max_percent_stromal_cells_lte = messages.FloatField(111) - max_percent_stromal_cells_gte = messages.FloatField(112) - - max_percent_tumor_cells = messages.FloatField(113, repeated=True) - max_percent_tumor_cells_lte = messages.FloatField(114) - max_percent_tumor_cells_gte = messages.FloatField(115) - - max_percent_tumor_nuclei = messages.FloatField(116, repeated=True) - max_percent_tumor_nuclei_lte = messages.FloatField(117) - max_percent_tumor_nuclei_gte = messages.FloatField(118) - - menopause_status = messages.StringField(119, repeated=True) - - min_percent_lymphocyte_infiltration = messages.FloatField(120, repeated=True) - min_percent_lymphocyte_infiltration_lte = messages.FloatField(121) - min_percent_lymphocyte_infiltration_gte = messages.FloatField(122) - - min_percent_monocyte_infiltration = messages.FloatField(123, repeated=True) - min_percent_monocyte_infiltration_lte = messages.FloatField(124) - min_percent_monocyte_infiltration_gte = messages.FloatField(125) - - min_percent_necrosis = messages.FloatField(126, repeated=True) - min_percent_necrosis_lte = messages.FloatField(127) - min_percent_necrosis_gte = messages.FloatField(128) - - min_percent_neutrophil_infiltration = messages.FloatField(129, repeated=True) - min_percent_neutrophil_infiltration_lte = messages.FloatField(130) - min_percent_neutrophil_infiltration_gte = messages.FloatField(131) - - min_percent_normal_cells = messages.FloatField(132, repeated=True) - min_percent_normal_cells_lte = messages.FloatField(133) - min_percent_normal_cells_gte = messages.FloatField(134) - - min_percent_stromal_cells = messages.FloatField(135, repeated=True) - min_percent_stromal_cells_lte = messages.FloatField(136) - min_percent_stromal_cells_gte = messages.FloatField(137) - - min_percent_tumor_cells = messages.FloatField(138, repeated=True) - min_percent_tumor_cells_lte = messages.FloatField(139) - min_percent_tumor_cells_gte = messages.FloatField(140) - - min_percent_tumor_nuclei = messages.FloatField(141, repeated=True) - min_percent_tumor_nuclei_lte = messages.FloatField(142) - min_percent_tumor_nuclei_gte = messages.FloatField(143) - - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(144, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(145, repeated=True) - neoplasm_histologic_grade = messages.StringField(146, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(147, repeated=True) - - number_of_lymphnodes_examined = messages.IntegerField(148, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_lte = messages.IntegerField(149, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_gte = messages.IntegerField(150, variant=messages.Variant.INT32) - - number_of_lymphnodes_positive_by_he = messages.IntegerField(151, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_lte = messages.IntegerField(152, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_gte = messages.IntegerField(153, variant=messages.Variant.INT32) - - number_pack_years_smoked = messages.IntegerField(154, repeated=True, variant=messages.Variant.INT32) - number_pack_years_smoked_lte = messages.IntegerField(155, variant=messages.Variant.INT32) - number_pack_years_smoked_gte = messages.IntegerField(156, variant=messages.Variant.INT32) - - case_barcode = messages.StringField(157, repeated=True) - pathologic_M = messages.StringField(158, repeated=True) - pathologic_N = messages.StringField(159, repeated=True) - pathologic_stage = messages.StringField(160, repeated=True) - pathologic_T = messages.StringField(161, repeated=True) - person_neoplasm_cancer_status = messages.StringField(162, repeated=True) - pregnancies = messages.StringField(163, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(164, repeated=True) - primary_therapy_outcome_success = messages.StringField(165, repeated=True) - prior_dx = messages.StringField(166, repeated=True) - Project = messages.StringField(167, repeated=True) - - psa_value = messages.FloatField(168, repeated=True) - psa_value_lte = messages.FloatField(169) - psa_value_gte = messages.FloatField(170) - - race = messages.StringField(171, repeated=True) - residual_tumor = messages.StringField(172, repeated=True) - sample_barcode = messages.StringField(173, repeated=True) - SampleTypeCode = messages.StringField(174, repeated=True) - disease_code = messages.StringField(175, repeated=True) - tobacco_smoking_history = messages.StringField(176, repeated=True) - TSSCode = messages.StringField(177, repeated=True) - tumor_tissue_site = messages.StringField(178, repeated=True) - tumor_type = messages.StringField(179, repeated=True) - vital_status = messages.StringField(180, repeated=True) - - weight = messages.IntegerField(181, repeated=True, variant=messages.Variant.INT32) - weight_lte = messages.IntegerField(182, variant=messages.Variant.INT32) - weight_gte = messages.IntegerField(183, variant=messages.Variant.INT32) - - weiss_venous_invasion = messages.StringField(184, repeated=True) - - year_of_initial_pathologic_diagnosis = messages.IntegerField(185, repeated=True, variant=messages.Variant.INT32) - year_of_initial_pathologic_diagnosis_lte = messages.IntegerField(186, variant=messages.Variant.INT32) - year_of_initial_pathologic_diagnosis_gte = messages.IntegerField(187, variant=messages.Variant.INT32) - - -class MetadataItem(messages.Message): - age_at_initial_pathologic_diagnosis = messages.IntegerField(1, variant=messages.Variant.INT32) - anatomic_neoplasm_subdivision = messages.StringField(2) - avg_percent_lymphocyte_infiltration = messages.FloatField(3) - avg_percent_monocyte_infiltration = messages.FloatField(4) - avg_percent_necrosis = messages.FloatField(5) - avg_percent_neutrophil_infiltration = messages.FloatField(6) - avg_percent_normal_cells = messages.FloatField(7) - avg_percent_stromal_cells = messages.FloatField(8) - avg_percent_tumor_cells = messages.FloatField(9) - avg_percent_tumor_nuclei = messages.FloatField(10) - batch_number = messages.IntegerField(11, variant=messages.Variant.INT32) - bcr = messages.StringField(12) - BMI = messages.FloatField(13) - clinical_M = messages.StringField(14) - clinical_N = messages.StringField(15) - clinical_stage = messages.StringField(16) - clinical_T = messages.StringField(17) - colorectal_cancer = messages.StringField(18) - country = messages.StringField(19) - days_to_birth = messages.IntegerField(20, variant=messages.Variant.INT32) - days_to_collection = messages.IntegerField(21, variant=messages.Variant.INT32) - days_to_death = messages.IntegerField(22, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis = messages.IntegerField(23, variant=messages.Variant.INT32) - days_to_last_followup = messages.IntegerField(24, variant=messages.Variant.INT32) - days_to_last_known_alive = messages.IntegerField(25, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx = messages.IntegerField(26, variant=messages.Variant.INT32) - ethnicity = messages.StringField(27) - frozen_specimen_anatomic_site = messages.StringField(28) - gender = messages.StringField(29) - gleason_score_combined = messages.IntegerField(30, variant=messages.Variant.INT32) - has_27k = messages.BooleanField(31) - has_450k = messages.BooleanField(32) - has_BCGSC_GA_RNASeq = messages.BooleanField(33) - has_BCGSC_HiSeq_RNASeq = messages.BooleanField(34) - has_GA_miRNASeq = messages.BooleanField(35) - has_HiSeq_miRnaSeq = messages.BooleanField(36) - has_Illumina_DNASeq = messages.BooleanField(37) - has_RPPA = messages.BooleanField(38) - has_SNP6 = messages.BooleanField(39) - has_UNC_GA_RNASeq = messages.BooleanField(40) - has_UNC_HiSeq_RNASeq = messages.BooleanField(41) - height = messages.IntegerField(42, variant=messages.Variant.INT32) - histological_type = messages.StringField(43) - history_of_colon_polyps = messages.StringField(44) - history_of_neoadjuvant_treatment = messages.StringField(45) - history_of_prior_malignancy = messages.StringField(46) - hpv_calls = messages.StringField(47) - hpv_status = messages.StringField(48) - icd_10 = messages.StringField(49) - icd_o_3_histology = messages.StringField(50) - icd_o_3_site = messages.StringField(51) - lymphatic_invasion = messages.StringField(52) - lymphnodes_examined = messages.StringField(53) - lymphovascular_invasion_present = messages.StringField(54) - max_percent_lymphocyte_infiltration = messages.FloatField(55) - max_percent_monocyte_infiltration = messages.FloatField(56) - max_percent_necrosis = messages.FloatField(57) - max_percent_neutrophil_infiltration = messages.FloatField(58) - max_percent_normal_cells = messages.FloatField(59) - max_percent_stromal_cells = messages.FloatField(60) - max_percent_tumor_cells = messages.FloatField(61) - max_percent_tumor_nuclei = messages.FloatField(62) - menopause_status = messages.StringField(63) - min_percent_lymphocyte_infiltration = messages.FloatField(64) - min_percent_monocyte_infiltration = messages.FloatField(65) - min_percent_necrosis = messages.FloatField(66) - min_percent_neutrophil_infiltration = messages.FloatField(67) - min_percent_normal_cells = messages.FloatField(68) - min_percent_stromal_cells = messages.FloatField(69) - min_percent_tumor_cells = messages.FloatField(70) - min_percent_tumor_nuclei = messages.FloatField(71) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(72) - mononucleotide_marker_panel_analysis_status = messages.StringField(73) - neoplasm_histologic_grade = messages.StringField(74) - new_tumor_event_after_initial_treatment = messages.StringField(75) - number_of_lymphnodes_examined = messages.IntegerField(76, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he = messages.IntegerField(77, variant=messages.Variant.INT32) - number_pack_years_smoked = messages.IntegerField(78, variant=messages.Variant.INT32) - case_barcode = messages.StringField(79) - pathologic_M = messages.StringField(80) - pathologic_N = messages.StringField(81) - pathologic_stage = messages.StringField(82) - pathologic_T = messages.StringField(83) - person_neoplasm_cancer_status = messages.StringField(84) - pregnancies = messages.StringField(85) - primary_neoplasm_melanoma_dx = messages.StringField(86) - primary_therapy_outcome_success = messages.StringField(87) - prior_dx = messages.StringField(88) - program_name = messages.StringField(89) - psa_value = messages.FloatField(90) - race = messages.StringField(91) - residual_tumor = messages.StringField(92) - sample_barcode = messages.StringField(93) - SampleTypeCode = messages.StringField(94) - disease_code = messages.StringField(95) - tobacco_smoking_history = messages.StringField(96) - TSSCode = messages.StringField(97) - tumor_tissue_site = messages.StringField(98) - tumor_type = messages.StringField(99) - vital_status = messages.StringField(100) - weight = messages.IntegerField(101, variant=messages.Variant.INT32) - weiss_venous_invasion = messages.StringField(102) - year_of_initial_pathologic_diagnosis = messages.IntegerField(103, variant=messages.Variant.INT32) diff --git a/api/isb_cgc_api/message_generator.py b/api/isb_cgc_api/message_generator.py deleted file mode 100644 index 9063a0a1..00000000 --- a/api/isb_cgc_api/message_generator.py +++ /dev/null @@ -1,116 +0,0 @@ - -from argparse import ArgumentParser -import MySQLdb - -def get_sql_connection(args): - try: - connect_options = { - 'host': args.host, - 'db': args.database, - 'user': args.user, - 'passwd': args.password, - 'ssl': { - 'ca': args.ssl_ca, - 'key': args.ssl_key, - 'cert': args.ssl_cert - } - } - db = MySQLdb.connect(**connect_options) - except: - print 'failed' - raise - return db - -FIELD_TYPES ={ - 'varchar': 'StringField', - 'float': 'FloatField', - 'tinyint': 'BooleanField', - 'int': 'IntegerField' -} - - -def write_metadata_file(rows, write_file=False): - - ranges_text = 'class MetadataRangesItem(messages.Message):\n ' - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if field_type == 'IntegerField' or field_type == 'FloatField' and i > 1: - ranges_text += '\n ' - - ranges_text += '%-65s = messages.%s(%d, repeated=True' % (row['COLUMN_NAME'], field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if field_type == 'IntegerField' or field_type == 'FloatField': - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_lte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_gte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '\n ' - - item_text = '\nclass MetadataItem(messages.Message):\n ' - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - item_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME'], field_type, i) - item_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if write_file is True: - with open('message_classes.py', 'w') as f: - f.write('from protorpc import messages\n\n\n') - f.write(ranges_text) - f.write(item_text) - else: - print ranges_text - print '\n\n' - print item_text - - -def main(args): - db = get_sql_connection(args) - cursor = db.cursor(MySQLdb.cursors.DictCursor) - query_str = 'SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'FROM INFORMATION_SCHEMA.COLUMNS ' - if args.table == 'metadata_samples': - query_str += 'WHERE table_name = "metadata_samples" ' \ - 'AND COLUMN_NAME != "metadata_samples_id" ' - - elif args.table == 'metadata_annotation': - query_str += 'WHERE table_name = "metadata_annotation" ' \ - 'AND COLUMN_NAME != "metadata_annotation_id" ' - - query_str += 'group by COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'ORDER BY column_name' - - cursor.execute(query_str) - rows = cursor.fetchall() - cursor.close() - db.close() - - write_file = not bool(args.dry_run) - - if args.table == 'metadata_samples': - write_metadata_file(rows, write_file=write_file) - if args.table == 'metadata_annotation': - write_annotation_file(rows, write_file=write_file) - -if __name__ == '__main__': - parser = ArgumentParser() - parser.add_argument('--host', help='database host') - parser.add_argument('--database', '-d', help='database name') - parser.add_argument('--table', '-t', help='database table name') - parser.add_argument('--user', '-u', help='database username') - parser.add_argument('--password', '-p', help='database password') - parser.add_argument('--ssl_key') - parser.add_argument('--ssl_cert') - parser.add_argument('--ssl_ca') - parser.add_argument('--dry_run', dest='dry_run', action='store_true') - parser.add_argument('--write_file', dest='dry_run', action='store_false') - parser.set_defaults(feature=True) - args = parser.parse_args() - - main(args) \ No newline at end of file diff --git a/api/isb_cgc_api/patients_get.py b/api/isb_cgc_api/patients_get.py deleted file mode 100644 index c2bc12b9..00000000 --- a/api/isb_cgc_api/patients_get.py +++ /dev/null @@ -1,115 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message -from message_classes import MetadataItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class PatientsGetQueryBuilder(object): - - def build_queries(self): - clinical_query_str = 'select * ' \ - 'from metadata_clinical ' \ - 'where case_barcode=%s' - - sample_query_str = 'select sample_barcode ' \ - 'from metadata_biospecimen ' \ - 'where case_barcode=%s' - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where case_barcode=%s ' \ - 'group by AliquotBarcode' - - return clinical_query_str, sample_query_str, aliquot_query_str - - -class PatientDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - - -@ISB_CGC_Endpoints.api_class(resource_name='patients') -class PatientsGetAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(patient_barcode=messages.StringField(1, required=True)) - - @endpoints.method(GET_RESOURCE, PatientDetails, - path='patients/{patient_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific patient, - including a list of samples and aliquots derived from this patient. - Takes a patient barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - patient_barcode = request.get_assigned_value('patient_barcode') - query_tuple = (str(patient_barcode),) - clinical_query_str, sample_query_str, aliquot_query_str = PatientsGetQueryBuilder().build_queries() - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build clinical data message - cursor.execute(clinical_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - logger.warn("Patient barcode {} not found in metadata_clinical table.".format(patient_barcode)) - raise endpoints.NotFoundException("Patient barcode {} not found".format(patient_barcode)) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - clinical_data_item = MetadataItem(**constructor_dict) - - # get list of samples - cursor.execute(sample_query_str, query_tuple) - sample_list = [row['sample_barcode'] for row in cursor.fetchall()] - - # get list of aliquots - # cursor.execute(aliquot_query_str, query_tuple) - # aliquot_list = [row['AliquotBarcode'] for row in cursor.fetchall()] - aliquot_list = [] - - return PatientDetails(clinical_data=clinical_data_item, samples=sample_list, aliquots=aliquot_list) - - except (IndexError, TypeError), e: - logger.info("Patient {} not found. Error: {}".format(patient_barcode, e)) - raise endpoints.NotFoundException("Patient {} not found.".format(patient_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving patient, sample, or aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving patient, sample, or aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/samples_cloudstoragefilepaths.py b/api/isb_cgc_api/samples_cloudstoragefilepaths.py deleted file mode 100644 index a5faeba2..00000000 --- a/api/isb_cgc_api/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,90 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from django.core.signals import request_finished -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, are_there_bad_keys, construct_parameter_error_message, \ - CohortsSamplesFilesQueryBuilder, CohortsSamplesFilesMessageBuilder -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class GCSFilePathList(messages.Message): - cloud_storage_file_paths = messages.StringField(1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesCloudStorageFilePathsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, GCSFilePathList, - path='samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - platform = request.get_assigned_value('platform') - pipeline = request.get_assigned_value('pipeline') - - if are_there_bad_keys(request): - err_msg = construct_parameter_error_message(request, False) - raise endpoints.BadRequestException(err_msg) - - query_str, query_tuple = CohortsSamplesFilesQueryBuilder().build_query( - platform=platform, pipeline=pipeline, sample_barcode=sample_barcode) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - cursor_rows = cursor.fetchall() - # add 'cloud_storage_path' to cursor_rows - bad_repo_count, bad_repo_set = CohortsSamplesFilesMessageBuilder().get_GCS_file_paths_and_bad_repos(cursor_rows) - cloud_storage_path_list = [row['cloud_storage_path'] for row in cursor_rows] - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - return GCSFilePathList(cloud_storage_file_paths=cloud_storage_path_list, count=len(cloud_storage_path_list)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException("File paths for sample {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\t query: {} {}'.format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) \ No newline at end of file diff --git a/api/isb_cgc_api/samples_get.py b/api/isb_cgc_api/samples_get.py deleted file mode 100644 index 6b7a6619..00000000 --- a/api/isb_cgc_api/samples_get.py +++ /dev/null @@ -1,215 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints, build_constructor_dict_for_message, \ - CohortsSamplesFilesMessageBuilder -from message_classes import MetadataItem -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class SamplesGetQueryBuilder(object): - - def build_aliquot_query(self, platform=None, pipeline=None): - - aliquot_query_str = 'select AliquotBarcode ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' - - aliquot_query_str += ' and platform=%s ' if platform is not None else '' - aliquot_query_str += ' and pipeline=%s ' if pipeline is not None else '' - aliquot_query_str += ' group by AliquotBarcode' - - return aliquot_query_str - - def build_biospecimen_query(self): - - biospecimen_query_str = 'select * ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s' - - return biospecimen_query_str - - def build_data_query(self, platform=None, pipeline=None): - - data_query_str = 'select ' \ - 'sample_barcode, ' \ - 'DataCenterName, ' \ - 'DataCenterType, ' \ - 'DataFileName, ' \ - 'DataFileNameKey, ' \ - 'DatafileUploaded, ' \ - 'DataLevel,' \ - 'Datatype,' \ - 'GenomeReference,' \ - 'Pipeline,' \ - 'Platform,' \ - 'platform_full_name,' \ - 'program_name,' \ - 'Repository,' \ - 'SDRFFileName,' \ - 'SecurityProtocol ' \ - 'from metadata_data ' \ - 'where sample_barcode=%s ' \ - 'and DataFileNameKey is not null and DataFileNameKey !=""' - - data_query_str += ' and platform=%s ' if platform is not None else '' - data_query_str += ' and pipeline=%s ' if pipeline is not None else '' - - return data_query_str - - def build_patient_query(self): - - patient_query_str = 'select case_barcode ' \ - 'from metadata_biospecimen ' \ - 'where sample_barcode=%s ' \ - 'group by case_barcode' - - return patient_query_str - - -class DataDetails(messages.Message): - sample_Barcode = messages.StringField(1) - DataCenterName = messages.StringField(2) - DataCenterType = messages.StringField(3) - DataFileName = messages.StringField(4) - DataFileNameKey = messages.StringField(5) - DatafileUploaded = messages.StringField(6) - DataLevel = messages.StringField(7) - Datatype = messages.StringField(8) - GenomeReference = messages.StringField(9) - Pipeline = messages.StringField(10) - Platform = messages.StringField(11) - platform_full_name = messages.StringField(12) - Project = messages.StringField(13) - Repository = messages.StringField(14) - SDRFFileName = messages.StringField(15) - SecurityProtocol = messages.StringField(16) - cloud_storage_path = messages.StringField(17) - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(MetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - patient = messages.StringField(3) - data_details = messages.MessageField(DataDetails, 4, repeated=True) - data_details_count = messages.IntegerField(5, variant=messages.Variant.INT32) - - -@ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesGetAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True), - platform=messages.StringField(2), - pipeline=messages.StringField(3)) - - @endpoints.method(GET_RESOURCE, SampleDetails, - path='samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated patient barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - pipeline = request.get_assigned_value('pipeline') - platform = request.get_assigned_value('platform') - - aliquot_query_str = SamplesGetQueryBuilder().build_aliquot_query(platform=platform, pipeline=pipeline) - biospecimen_query_str = SamplesGetQueryBuilder().build_biospecimen_query() - data_query_str = SamplesGetQueryBuilder().build_data_query(platform=platform, pipeline=pipeline) - patient_query_str = SamplesGetQueryBuilder().build_patient_query() - - query_tuple = (str(sample_barcode),) - extra_query_tuple = query_tuple - if pipeline is not None: extra_query_tuple += (pipeline,) - if platform is not None: extra_query_tuple += (platform,) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build biospecimen data message - cursor.execute(biospecimen_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - error_message = "Sample barcode {} not found in metadata_biospecimen table.".format(sample_barcode) - raise endpoints.NotFoundException(error_message) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - biospecimen_data_item = MetadataItem(**constructor_dict) - - # get list of aliquots - # cursor.execute(aliquot_query_str, extra_query_tuple) - # aliquot_list = [row['AliquotBarcode'] for row in cursor.fetchall()] - aliquot_list = [] - - # get patient barcode (superfluous?) - cursor.execute(patient_query_str, query_tuple) - row = cursor.fetchone() - patient_barcode = str(row["case_barcode"]) - - # prepare to build list of data details messages - cursor.execute(data_query_str, extra_query_tuple) - cursor_rows = cursor.fetchall() - # update every dictionary in cursor_rows to contain the full cloud_storage_path for each sample - bad_repo_count, bad_repo_set = \ - CohortsSamplesFilesMessageBuilder().get_GCS_file_paths_and_bad_repos(cursor_rows) - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - - # build a data details message for each row returned from metadata_data table - data_details_list = [] - for row in cursor_rows: - constructor_dict = build_constructor_dict_for_message(DataDetails(), row) - data_details_item = DataDetails(**constructor_dict) - data_details_list.append(data_details_item) - - if bad_repo_count > 0: - logger.warn("not returning {count} row(s) in sample_details due to repositories: {bad_repo_list}" - .format(count=bad_repo_count, bad_repo_list=list(bad_repo_set))) - - return SampleDetails(aliquots=aliquot_list, - biospecimen_data=biospecimen_data_item, - data_details=data_details_list, - data_details_count=len(data_details_list), - patient=patient_barcode) - - except (IndexError, TypeError) as e: - logger.info("Sample details for barcode {} not found. Error: {}".format(sample_barcode, e)) - raise endpoints.NotFoundException( - "Sample details for barcode {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn(e) - raise endpoints.BadRequestException("Error retrieving biospecimen, patient, or other data. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api/isb_cgc_api/samples_googlegenomics.py b/api/isb_cgc_api/samples_googlegenomics.py deleted file mode 100644 index ec1b3ea1..00000000 --- a/api/isb_cgc_api/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from api.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api/isb_cgc_api/users_get.py b/api/isb_cgc_api/users_get.py deleted file mode 100644 index 458bd370..00000000 --- a/api/isb_cgc_api/users_get.py +++ /dev/null @@ -1,143 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import django -import pytz -import datetime -import logging -from protorpc import remote, messages, message_types -from googleapiclient.errors import HttpError - -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from django.conf import settings - -from isb_cgc_api_helpers import ISB_CGC_Endpoints -from google_helpers.directory_service import get_directory_resource -from accounts.models import NIH_User - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - - -def is_dbgap_authorized(user_email): - directory_service, http_auth = get_directory_resource() - try: - directory_service.members().get(groupKey="", - memberKey=user_email).execute(http=http_auth) - return True - except HttpError, e: - return False - - -class ReturnJSON(messages.Message): - message = messages.StringField(1) - dbGaP_authorized = messages.BooleanField(2) - - -@ISB_CGC_Endpoints.api_class(resource_name='users') -class UserGetAPI(remote.Service): - - @endpoints.method(message_types.VoidMessage, ReturnJSON, http_method='GET', path='users') - def get(self, request): - ''' - Returns the dbGaP authorization status of the user. - ''' - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException("Authentication unsuccessful.") - - # this checks the controlled-access google group - am_dbgap_authorized = is_dbgap_authorized(user_email) - - if not am_dbgap_authorized: - return ReturnJSON(message="{} is not on the controlled-access google group.".format(user_email), - dbGaP_authorized=False) - - django.setup() - - # all the following five situations should never happen - - # 1. check to make sure they have an entry in auth_user - try: - django_user = Django_User.objects.get(email=user_email) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error("Email {} is in {} group but did not have a unique entry in auth_user table. Error: {}" - .format(user_email, "", e)) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but does not have an entry in the user database." - .format(user_email)) - - # 2. check to make sure they have an entry in accounts_nih_user - try: - nih_user = NIH_User.objects.get(user_id=django_user.id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error("Email {} is in {} group but did not have a unique entry in " - "accounts_nih_user table. Error: {}" - .format(user_email, "", e)) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but does not have an entry in the nih_user database." - .format(user_email)) - - # 3. check if their entry in accounts_nih_user is currently active - if not nih_user.active: - logger.error("Email {} is in {} group but their entry in accounts_nih_user table is inactive." - .format(user_email, "")) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but has an inactive entry in the nih_user database." - .format(user_email)) - - # 4. check if their entry in accounts_nih_user is dbGaP_authorized - if not nih_user.dbGaP_authorized: - logger.error("Email {} is in {} group but their entry in accounts_nih_user table " - "is not dbGaP_authorized." - .format(user_email, "")) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but their entry in the nih_user database is not dbGaP_authorized." - .format(user_email)) - - # 5. check if their dbgap authorization has expired - expire_time = nih_user.NIH_assertion_expiration - expire_time = expire_time if expire_time.tzinfo is not None else pytz.utc.localize(expire_time) - now_in_utc = pytz.utc.localize(datetime.datetime.now()) - - if (expire_time - now_in_utc).total_seconds() <= 0: - logger.error("Email {} is in {} group but their entry in accounts_nih_user table " - "is expired." - .format(user_email, "")) - request_finished.send(self) - raise endpoints.NotFoundException("{} is in the controlled-access google group " - "but their entry in the nih_user database is expired." - .format(user_email)) - - # all checks have passed - request_finished.send(self) - return ReturnJSON(message="{} has dbGaP authorization and is a member of the controlled-access google group." - .format(user_email), - dbGaP_authorized=True) \ No newline at end of file diff --git a/api/maf.py b/api/maf.py deleted file mode 100755 index 3c61ee0d..00000000 --- a/api/maf.py +++ /dev/null @@ -1,104 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -from protorpc import remote -from protorpc.messages import Message, MessageField, IntegerField, StringField, Variant -from MySQLdb.cursors import DictCursor - -from api_helpers import * - - -# Generated code - in scripts/maf_import in Python do: -# -# > import import_maf -# > repr(import_maf.FIELD_NAMES_AND_TYPES) -# -FIELD_NAMES_AND_TYPES = [ - {'type': 'varchar(128)', 'name': 'hugo_symbol'}, - {'type': 'int', 'name': 'amino_acid_position'}, - {'type': 'varchar(128)', 'name': 'tumor_type'}, - {'type': 'varchar(12)', 'name': 'uniprot_id'}, - {'type': 'varchar(128)', 'name': 'variant_classification'}, - {'type': 'varchar(128)', 'name': 'c_position'}, - {'type': 'varchar(128)', 'name': 'tumor_sample_barcode'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'reference_allele'}, - {'type': 'varchar(128)', 'name': 'variant_type'}, - {'type': 'varchar(128)', 'name': 'ucsc_cons'} -] - - -class MAFRecord(Message): - hugo_symbol = StringField(1) - tumor_type = StringField(3) - amino_acid_position = IntegerField(2, variant=Variant.INT32) - uniprot_id = StringField(4) - variant_classification = StringField(5) - c_position = StringField(6) - tumor_sample_barcode = StringField(7) - match_norm_seq_allele2 = StringField(8) - tumor_seq_allele1 = StringField(9) - tumor_seq_allele2 = StringField(10) - match_norm_seq_allele1 = StringField(11) - reference_allele = StringField(12) - variant_type = StringField(13) - ucsc_cons = StringField(14) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - - -class MAFRequest(Message): - gene = StringField(1, required=True) - tumor = StringField(2, repeated=True) - -MAFEndpointsAPI = endpoints.api(name='maf_api', version='v1') - - -@MAFEndpointsAPI .api_class(resource_name='maf_endpoints') -class MAFEndpointsAPI(remote.Service): - @endpoints.method(MAFRequest, MAFRecordList, - path='maf_search', http_method='GET', name='maf.getMAF') - def maf_search(self, request): - gene = request.gene - tumor_type_list = request.tumor - tumor_set_template = ', '.join(['%s' for x in range(len(tumor_type_list))]) - query = 'SELECT * FROM maf WHERE hugo_symbol=%s AND tumor_type IN ({0})'.format(tumor_set_template) - - values = [gene] - values.extend(tumor_type_list) - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - data_list = [] - for row in cursor.fetchall(): - data_list.append(MAFRecord(**row)) - - cursor.close() - db.close() - return MAFRecordList(items=data_list) - - except: - raise endpoints.NotFoundException('MAF query error') \ No newline at end of file diff --git a/api/metadata.py b/api/metadata.py deleted file mode 100755 index a07ec132..00000000 --- a/api/metadata.py +++ /dev/null @@ -1,2753 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import time -from protorpc import messages -from protorpc import message_types -from protorpc import remote -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from allauth.socialaccount.models import SocialToken, SocialAccount -from accounts.models import NIH_User -from cohorts.models import Cohort_Perms, Cohort as Django_Cohort, Samples, Filters -from projects.models import Project, User_Feature_Definitions, User_Feature_Counts, User_Data_Tables -from django.core.signals import request_finished -from time import sleep -import django -import sys -import logging -import re -import json -import traceback -import copy -from uuid import uuid4 - -from api.api_helpers import * - -logger = logging.getLogger(__name__) - -debug = settings.DEBUG - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID -OPEN_DATA_BUCKET = settings.OPEN_DATA_BUCKET -#CONTROLLED_DATA_BUCKET = settings.CONTROLLED_DATA_BUCKET - -METADATA_SHORTLIST = [ - # 'adenocarcinoma_invasion', - 'age_at_initial_pathologic_diagnosis', - # 'anatomic_neoplasm_subdivision', - # 'avg_percent_lymphocyte_infiltration', - # 'avg_percent_monocyte_infiltration', - # 'avg_percent_necrosis', - # 'avg_percent_neutrophil_infiltration', - # 'avg_percent_normal_cells', - # 'avg_percent_stromal_cells', - # 'avg_percent_tumor_cells', - # 'avg_percent_tumor_nuclei', - # 'batch_number', - # 'bcr', - 'BMI', - # 'clinical_M', - # 'clinical_N', - # 'clinical_stage', - # 'clinical_T', - # 'colorectal_cancer', - # 'country', - # 'country_of_procurement', - # 'days_to_birth', - # 'days_to_collection', - # 'days_to_death', - # 'days_to_initial_pathologic_diagnosis', - # 'days_to_last_followup', - # 'days_to_submitted_specimen_dx', - 'Study', - # 'ethnicity', - # 'frozen_specimen_anatomic_site', - 'gender', - # 'gleason_score_combined', - 'has_27k', - 'has_450k', - 'has_BCGSC_GA_RNASeq', - 'has_BCGSC_HiSeq_RNASeq', - 'has_GA_miRNASeq', - 'has_HiSeq_miRnaSeq', - 'has_Illumina_DNASeq', - 'has_RPPA', - 'has_SNP6', - 'has_UNC_GA_RNASeq', - 'has_UNC_HiSeq_RNASeq', - # 'height', - 'histological_type', - # 'history_of_colon_polyps', - # 'history_of_neoadjuvant_treatment', - # 'history_of_prior_malignancy', - # 'hpv_calls', - 'hpv_status', - 'icd_10', - 'icd_o_3_histology', - 'icd_o_3_site', - # 'lymph_node_examined_count', - # 'lymphatic_invasion', - # 'lymphnodes_examined', - # 'lymphovascular_invasion_present', - # 'max_percent_lymphocyte_infiltration', - # 'max_percent_monocyte_infiltration', - # 'max_percent_necrosis', - # 'max_percent_neutrophil_infiltration', - # 'max_percent_normal_cells', - # 'max_percent_stromal_cells', - # 'max_percent_tumor_cells', - # 'max_percent_tumor_nuclei', - # 'menopause_status', - # 'min_percent_lymphocyte_infiltration', - # 'min_percent_monocyte_infiltration', - # 'min_percent_necrosis', - # 'min_percent_neutrophil_infiltration', - # 'min_percent_normal_cells', - # 'min_percent_stromal_cells', - # 'min_percent_tumor_cells', - # 'min_percent_tumor_nuclei', - # 'mononucleotide_and_dinucleotide_marker_panel_analysis_status', - # 'mononucleotide_marker_panel_analysis_status', - 'neoplasm_histologic_grade', - 'new_tumor_event_after_initial_treatment', - # 'number_of_lymphnodes_examined', - # 'number_of_lymphnodes_positive_by_he', - # 'number_pack_years_smoked', - # 'other_dx', - # 'case_barcode', - # 'pathologic_M', - # 'pathologic_N', - 'pathologic_stage', - # 'pathologic_T', - 'person_neoplasm_cancer_status', - # 'pregnancies', - # 'preservation_method', - # 'primary_neoplasm_melanoma_dx', - # 'primary_therapy_outcome_success', - 'Project', - # 'psa_value', - # 'race', - 'residual_tumor', - # 'sample_barcode', - 'SampleTypeCode', - # 'Study', - 'tobacco_smoking_history', - # 'total_number_of_pregnancies', - # 'tumor_pathology', - 'tumor_tissue_site', - 'tumor_type', - # 'weiss_venous_invasion', - 'vital_status' - # 'weight', - # 'year_of_initial_pathologic_diagnosis', -] - -metadata_dict = { - 'age_at_initial_pathologic_diagnosis': 'INTEGER', - 'anatomic_neoplasm_subdivision': 'VARCHAR(63)', - 'avg_percent_lymphocyte_infiltration': 'FLOAT', - 'avg_percent_monocyte_infiltration': 'FLOAT', - 'avg_percent_necrosis': 'FLOAT', - 'avg_percent_neutrophil_infiltration': 'FLOAT', - 'avg_percent_normal_cells': 'FLOAT', - 'avg_percent_stromal_cells': 'FLOAT', - 'avg_percent_tumor_cells': 'FLOAT', - 'avg_percent_tumor_nuclei': 'FLOAT', - 'batch_number': 'INTEGER', - 'bcr': 'VARCHAR(63)', - 'BMI': 'FLOAT', - 'clinical_M': 'VARCHAR(12)', - 'clinical_N': 'VARCHAR(12)', - 'clinical_T': 'VARCHAR(12)', - 'clinical_stage': 'VARCHAR(12)', - 'colorectal_cancer': 'VARCHAR(10)', - 'country': 'VARCHAR(63)', - 'days_to_birth': 'INTEGER', - 'days_to_collection': 'INTEGER', - 'days_to_death': 'INTEGER', - 'days_to_initial_pathologic_diagnosis': 'INTEGER', - 'days_to_last_followup': 'INTEGER', - 'days_to_submitted_specimen_dx': 'INTEGER', - 'Study': 'VARCHAR(4)', - 'ethnicity': 'VARCHAR(20)', - 'frozen_specimen_anatomic_site': 'VARCHAR(63)', - 'gender': 'VARCHAR(15)', - 'gleason_score_combined': 'INTEGER', - 'height': 'INTEGER', - 'histological_type': 'VARCHAR(63)', - 'history_of_colon_polyps': 'VARCHAR(8)', - 'history_of_neoadjuvant_treatment': 'VARCHAR(63)', - 'history_of_prior_malignancy': 'VARCHAR(25)', - 'hpv_calls': 'VARCHAR(20)', - 'hpv_status': 'VARCHAR(20)', - 'icd_10': 'VARCHAR(8)', - 'icd_o_3_histology': 'VARCHAR(10)', - 'icd_o_3_site': 'VARCHAR(8)', - 'lymphatic_invasion': 'VARCHAR(8)', - 'lymphnodes_examined': 'VARCHAR(8)', - 'lymphovascular_invasion_present': 'VARCHAR(63)', - 'max_percent_lymphocyte_infiltration': 'INTEGER', - 'max_percent_monocyte_infiltration': 'INTEGER', - 'max_percent_necrosis': 'INTEGER', - 'max_percent_neutrophil_infiltration': 'INTEGER', - 'max_percent_normal_cells': 'INTEGER', - 'max_percent_stromal_cells': 'INTEGER', - 'max_percent_tumor_cells': 'INTEGER', - 'max_percent_tumor_nuclei': 'INTEGER', - 'menopause_status': 'VARCHAR(30)', - 'min_percent_lymphocyte_infiltration': 'INTEGER', - 'min_percent_monocyte_infiltration': 'INTEGER', - 'min_percent_necrosis': 'INTEGER', - 'min_percent_neutrophil_infiltration': 'INTEGER', - 'min_percent_normal_cells': 'INTEGER', - 'min_percent_stromal_cells': 'INTEGER', - 'min_percent_tumor_cells': 'INTEGER', - 'min_percent_tumor_nuclei': 'INTEGER', - 'mononucleotide_and_dinucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'mononucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'neoplasm_histologic_grade': 'VARCHAR(15)', - 'new_tumor_event_after_initial_treatment': 'VARCHAR(8)', - 'number_of_lymphnodes_examined': 'INTEGER', - 'number_of_lymphnodes_positive_by_he': 'INTEGER', - 'number_pack_years_smoked': 'INTEGER', - 'case_barcode': 'VARCHAR(12)', - 'pathologic_M': 'VARCHAR(5)', - 'pathologic_N': 'VARCHAR(5)', - 'pathologic_T': 'VARCHAR(5)', - 'pathologic_stage': 'VARCHAR(10)', - 'person_neoplasm_cancer_status': 'VARCHAR(15)', - 'pregnancies': 'VARCHAR(35)', - 'primary_neoplasm_melanoma_dx': 'VARCHAR(10)', - 'primary_therapy_outcome_success': 'VARCHAR(35)', - 'prior_dx': 'VARCHAR(50)', - 'Project': 'VARCHAR(4)', - 'psa_value': 'FLOAT', - 'race': 'VARCHAR(30)', - 'residual_tumor': 'VARCHAR(5)', - 'sample_barcode': 'VARCHAR(16)', - 'Study': 'VARCHAR(4)', - 'tobacco_smoking_history': 'VARCHAR(30)', - 'tumor_tissue_site': 'VARCHAR(20)', - 'tumor_type': 'VARCHAR(4)', - 'weiss_venous_invasion': 'VARCHAR(63)', - 'vital_status': 'VARCHAR(63)', - 'weight': 'VARCHAR(63)', - 'year_of_initialPY_pathologic_diagnosis': 'VARCHAR(63)', - 'SampleTypeCode': 'VARCHAR(3)', - 'has_Illumina_DNASeq': 'TINYINT', - 'has_BCGSC_HiSeq_RNASeq': 'TINYINT', - 'has_UNC_HiSeq_RNASeq': 'TINYINT', - 'has_BCGSC_GA_RNASeq': 'TINYINT', - 'has_UNC_GA_RNASeq': 'TINYINT', - 'has_HiSeq_miRnaSeq': 'TINYINT', - 'has_GA_miRNASeq': 'TINYINT', - 'has_RPPA': 'TINYINT', - 'has_SNP6': 'TINYINT', - 'has_27k': 'TINYINT', - 'has_450k': 'TINYINT', -} - - -class MetaValueListCount(messages.Message): - value = messages.StringField(1) # note: this means converting booleans to strings - count = messages.IntegerField(2) - - -class MetaAttrValuesList(messages.Message): - adenocarcinoma_invasion = messages.MessageField(MetaValueListCount, 1, repeated=True) - age_at_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 2, repeated=True) - anatomic_neoplasm_subdivision = messages.MessageField(MetaValueListCount, 3, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_necrosis = messages.FloatField(6, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(7, repeated=True) - avg_percent_normal_cells = messages.FloatField(8, repeated=True) - avg_percent_stromal_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_cells = messages.FloatField(10, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(11, repeated=True) - batch_number = messages.MessageField(MetaValueListCount, 12, repeated=True) - bcr = messages.MessageField(MetaValueListCount, 13, repeated=True) - clinical_M = messages.MessageField(MetaValueListCount, 14, repeated=True) - clinical_N = messages.MessageField(MetaValueListCount, 15, repeated=True) - clinical_stage = messages.MessageField(MetaValueListCount, 16, repeated=True) - clinical_T = messages.MessageField(MetaValueListCount, 17, repeated=True) - colorectal_cancer = messages.MessageField(MetaValueListCount, 18, repeated=True) - country = messages.MessageField(MetaValueListCount, 19, repeated=True) - country_of_procurement = messages.MessageField(MetaValueListCount, 20, repeated=True) - days_to_birth = messages.MessageField(MetaValueListCount, 21, repeated=True) - days_to_collection = messages.MessageField(MetaValueListCount, 22, repeated=True) - days_to_death = messages.MessageField(MetaValueListCount, 23, repeated=True) - days_to_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 24, repeated=True) - days_to_last_followup = messages.MessageField(MetaValueListCount, 25, repeated=True) - days_to_submitted_specimen_dx = messages.MessageField(MetaValueListCount, 26, repeated=True) - Study = messages.MessageField(MetaValueListCount, 27, repeated=True) - ethnicity = messages.MessageField(MetaValueListCount, 28, repeated=True) - frozen_specimen_anatomic_site = messages.MessageField(MetaValueListCount, 29, repeated=True) - gender = messages.MessageField(MetaValueListCount, 30, repeated=True) - height = messages.MessageField(MetaValueListCount, 31, repeated=True) - histological_type = messages.MessageField(MetaValueListCount, 32, repeated=True) - history_of_colon_polyps = messages.MessageField(MetaValueListCount, 33, repeated=True) - history_of_neoadjuvant_treatment = messages.MessageField(MetaValueListCount, 34, repeated=True) - history_of_prior_malignancy = messages.MessageField(MetaValueListCount, 35, repeated=True) - hpv_calls = messages.MessageField(MetaValueListCount, 36, repeated=True) - hpv_status = messages.MessageField(MetaValueListCount, 37, repeated=True) - icd_10 = messages.MessageField(MetaValueListCount, 38, repeated=True) - icd_o_3_histology = messages.MessageField(MetaValueListCount, 39, repeated=True) - icd_o_3_site = messages.MessageField(MetaValueListCount, 40, repeated=True) - lymph_node_examined_count = messages.MessageField(MetaValueListCount, 41, repeated=True) - lymphatic_invasion = messages.MessageField(MetaValueListCount, 42, repeated=True) - lymphnodes_examined = messages.MessageField(MetaValueListCount, 43, repeated=True) - lymphovascular_invasion_present = messages.MessageField(MetaValueListCount, 44, repeated=True) - max_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 45, repeated=True) - max_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 46, repeated=True) - max_percent_necrosis = messages.MessageField(MetaValueListCount, 47, repeated=True) - max_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 48, repeated=True) - max_percent_normal_cells = messages.MessageField(MetaValueListCount, 49, repeated=True) - max_percent_stromal_cells = messages.MessageField(MetaValueListCount, 50, repeated=True) - max_percent_tumor_cells = messages.MessageField(MetaValueListCount, 51, repeated=True) - max_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 52, repeated=True) - menopause_status = messages.MessageField(MetaValueListCount, 53, repeated=True) - min_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 54, repeated=True) - min_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 55, repeated=True) - min_percent_necrosis = messages.MessageField(MetaValueListCount, 56, repeated=True) - min_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 57, repeated=True) - min_percent_normal_cells = messages.MessageField(MetaValueListCount, 58, repeated=True) - min_percent_stromal_cells = messages.MessageField(MetaValueListCount, 59, repeated=True) - min_percent_tumor_cells = messages.MessageField(MetaValueListCount, 60, repeated=True) - min_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 61, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 62, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 63, repeated=True) - neoplasm_histologic_grade = messages.MessageField(MetaValueListCount, 64, repeated=True) - new_tumor_event_after_initial_treatment = messages.MessageField(MetaValueListCount, 65, repeated=True) - number_of_lymphnodes_examined = messages.MessageField(MetaValueListCount, 66, repeated=True) - number_of_lymphnodes_positive_by_he = messages.MessageField(MetaValueListCount, 67, repeated=True) - case_barcode = messages.MessageField(MetaValueListCount, 68, repeated=True) - pathologic_M = messages.MessageField(MetaValueListCount, 69, repeated=True) - pathologic_N = messages.MessageField(MetaValueListCount, 70, repeated=True) - pathologic_stage = messages.MessageField(MetaValueListCount, 71, repeated=True) - pathologic_T = messages.MessageField(MetaValueListCount, 72, repeated=True) - person_neoplasm_cancer_status = messages.MessageField(MetaValueListCount, 73, repeated=True) - pregnancies = messages.MessageField(MetaValueListCount, 74, repeated=True) - preservation_method = messages.MessageField(MetaValueListCount, 75, repeated=True) - primary_neoplasm_melanoma_dx = messages.MessageField(MetaValueListCount, 76, repeated=True) - primary_therapy_outcome_success = messages.MessageField(MetaValueListCount, 77, repeated=True) - prior_dx = messages.MessageField(MetaValueListCount, 78, repeated=True) - Project = messages.MessageField(MetaValueListCount, 79, repeated=True) - psa_value = messages.FloatField(80, repeated=True) - race = messages.MessageField(MetaValueListCount, 81, repeated=True) - residual_tumor = messages.MessageField(MetaValueListCount, 82, repeated=True) - sample_barcode = messages.MessageField(MetaValueListCount, 83, repeated=True) - tobacco_smoking_history = messages.MessageField(MetaValueListCount, 86, repeated=True) - total_number_of_pregnancies = messages.MessageField(MetaValueListCount, 87, repeated=True) - tumor_tissue_site = messages.MessageField(MetaValueListCount, 88, repeated=True) - tumor_pathology = messages.MessageField(MetaValueListCount, 89, repeated=True) - tumor_type = messages.MessageField(MetaValueListCount, 90, repeated=True) - weiss_venous_invasion = messages.MessageField(MetaValueListCount, 91, repeated=True) - vital_status = messages.MessageField(MetaValueListCount, 92, repeated=True) - weight = messages.MessageField(MetaValueListCount, 93, repeated=True) - year_of_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 94, repeated=True) - SampleTypeCode = messages.MessageField(MetaValueListCount, 95, repeated=True) - has_Illumina_DNASeq = messages.MessageField(MetaValueListCount, 96, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 97, repeated=True) - has_UNC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 98, repeated=True) - has_BCGSC_GA_RNASeq = messages.MessageField(MetaValueListCount, 99, repeated=True) - has_UNC_GA_RNASeq = messages.MessageField(MetaValueListCount, 100, repeated=True) - has_HiSeq_miRnaSeq = messages.MessageField(MetaValueListCount, 101, repeated=True) - has_GA_miRNASeq = messages.MessageField(MetaValueListCount, 102, repeated=True) - has_RPPA = messages.MessageField(MetaValueListCount, 103, repeated=True) - has_SNP6 = messages.MessageField(MetaValueListCount, 104, repeated=True) - has_27k = messages.MessageField(MetaValueListCount, 105, repeated=True) - has_450k = messages.MessageField(MetaValueListCount, 106, repeated=True) - BMI = messages.FloatField(107, repeated=True) - - -class MetadataItem(messages.Message): - adenocarcinoma_invasion = messages.StringField(1) - age_at_initial_pathologic_diagnosis = messages.IntegerField(2) - anatomic_neoplasm_subdivision = messages.StringField(3) - avg_percent_lymphocyte_infiltration = messages.FloatField(4) - avg_percent_monocyte_infiltration = messages.FloatField(5) - avg_percent_necrosis = messages.FloatField(6) - avg_percent_neutrophil_infiltration = messages.FloatField(7) - avg_percent_normal_cells = messages.FloatField(8) - avg_percent_stromal_cells = messages.FloatField(9) - avg_percent_tumor_cells = messages.FloatField(10) - avg_percent_tumor_nuclei = messages.FloatField(11) - batch_number = messages.IntegerField(12) - bcr = messages.StringField(13) - clinical_M = messages.StringField(14) - clinical_N = messages.StringField(15) - clinical_stage = messages.StringField(16) - clinical_T = messages.StringField(17) - colorectal_cancer = messages.StringField(18) - country = messages.StringField(19) - country_of_procurement = messages.StringField(20) - days_to_birth = messages.IntegerField(21) - days_to_collection = messages.IntegerField(22) - days_to_death = messages.IntegerField(23) - days_to_initial_pathologic_diagnosis = messages.IntegerField(24) - days_to_last_followup = messages.IntegerField(25) - days_to_submitted_specimen_dx = messages.IntegerField(26) - Study = messages.StringField(27) - ethnicity = messages.StringField(28) - frozen_specimen_anatomic_site = messages.StringField(29) - gender = messages.StringField(30) - height = messages.IntegerField(31) - histological_type = messages.StringField(32) - history_of_colon_polyps = messages.StringField(33) - history_of_neoadjuvant_treatment = messages.StringField(34) - history_of_prior_malignancy = messages.StringField(35) - hpv_calls = messages.StringField(36) - hpv_status = messages.StringField(37) - icd_10 = messages.StringField(38) - icd_o_3_histology = messages.StringField(39) - icd_o_3_site = messages.StringField(40) - lymph_node_examined_count = messages.IntegerField(41) - lymphatic_invasion = messages.StringField(42) - lymphnodes_examined = messages.StringField(43) - lymphovascular_invasion_present = messages.StringField(44) - max_percent_lymphocyte_infiltration = messages.IntegerField(45) - max_percent_monocyte_infiltration = messages.IntegerField(46) - max_percent_necrosis = messages.IntegerField(47) - max_percent_neutrophil_infiltration = messages.IntegerField(48) - max_percent_normal_cells = messages.IntegerField(49) - max_percent_stromal_cells = messages.IntegerField(50) - max_percent_tumor_cells = messages.IntegerField(51) - max_percent_tumor_nuclei = messages.IntegerField(52) - menopause_status = messages.StringField(53) - min_percent_lymphocyte_infiltration = messages.IntegerField(54) - min_percent_monocyte_infiltration = messages.IntegerField(55) - min_percent_necrosis = messages.IntegerField(56) - min_percent_neutrophil_infiltration = messages.IntegerField(57) - min_percent_normal_cells = messages.IntegerField(58) - min_percent_stromal_cells = messages.IntegerField(59) - min_percent_tumor_cells = messages.IntegerField(60) - min_percent_tumor_nuclei = messages.IntegerField(61) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(62) - mononucleotide_marker_panel_analysis_status = messages.StringField(63) - neoplasm_histologic_grade = messages.StringField(64) - new_tumor_event_after_initial_treatment = messages.StringField(65) - number_of_lymphnodes_examined = messages.IntegerField(66) - number_of_lymphnodes_positive_by_he = messages.IntegerField(67) - case_barcode = messages.StringField(68) - pathologic_M = messages.StringField(69) - pathologic_N = messages.StringField(70) - pathologic_stage = messages.StringField(71) - pathologic_T = messages.StringField(72) - person_neoplasm_cancer_status = messages.StringField(73) - pregnancies = messages.StringField(74) - preservation_method = messages.StringField(75) - primary_neoplasm_melanoma_dx = messages.StringField(76) - primary_therapy_outcome_success = messages.StringField(77) - prior_dx = messages.StringField(78) - Project = messages.StringField(79) - psa_value = messages.FloatField(80) - race = messages.StringField(81) - residual_tumor = messages.StringField(82) - sample_barcode = messages.StringField(83) - tobacco_smoking_history = messages.StringField(86) - total_number_of_pregnancies = messages.IntegerField(87) - tumor_tissue_site = messages.StringField(88) - tumor_pathology = messages.StringField(89) - tumor_type = messages.StringField(90) - weiss_venous_invasion = messages.StringField(91) - vital_status = messages.StringField(92) - weight = messages.IntegerField(93) - year_of_initial_pathologic_diagnosis = messages.StringField(94) - SampleTypeCode = messages.StringField(95) - has_Illumina_DNASeq = messages.StringField(96) - has_BCGSC_HiSeq_RNASeq = messages.StringField(97) - has_UNC_HiSeq_RNASeq = messages.StringField(98) - has_BCGSC_GA_RNASeq = messages.StringField(99) - has_UNC_GA_RNASeq = messages.StringField(100) - has_HiSeq_miRnaSeq = messages.StringField(101) - has_GA_miRNASeq = messages.StringField(102) - has_RPPA = messages.StringField(103) - has_SNP6 = messages.StringField(104) - has_27k = messages.StringField(105) - has_450k = messages.StringField(106) - BMI = messages.FloatField(107) - -''' -Incoming object needs to use age and BMI that's a string (eg. 10_to_39) -''' -class IncomingMetadataItem(messages.Message): - age_at_initial_pathologic_diagnosis = messages.StringField(1, repeated=True) - anatomic_neoplasm_subdivision = messages.StringField(2, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(3, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_necrosis = messages.FloatField(5, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(6, repeated=True) - avg_percent_normal_cells = messages.FloatField(7, repeated=True) - avg_percent_stromal_cells = messages.FloatField(8, repeated=True) - avg_percent_tumor_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(10, repeated=True) - batch_number = messages.IntegerField(11, repeated=True) - bcr = messages.StringField(12, repeated=True) - clinical_M = messages.StringField(13, repeated=True) - clinical_N = messages.StringField(14, repeated=True) - clinical_stage = messages.StringField(15, repeated=True) - clinical_T = messages.StringField(16, repeated=True) - colorectal_cancer = messages.StringField(17, repeated=True) - country = messages.StringField(18, repeated=True) - days_to_birth = messages.IntegerField(19, repeated=True) - days_to_collection = messages.IntegerField(20, repeated=True) - days_to_death = messages.IntegerField(21, repeated=True) - days_to_initial_pathologic_diagnosis = messages.IntegerField(22, repeated=True) - days_to_last_followup = messages.IntegerField(23, repeated=True) - days_to_submitted_specimen_dx = messages.IntegerField(24, repeated=True) - Study = messages.StringField(25, repeated=True) - ethnicity = messages.StringField(26, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(27, repeated=True) - gender = messages.StringField(28, repeated=True) - height = messages.IntegerField(29, repeated=True) - histological_type = messages.StringField(30, repeated=True) - history_of_colon_polyps = messages.StringField(31, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(32, repeated=True) - history_of_prior_malignancy = messages.StringField(33, repeated=True) - hpv_calls = messages.StringField(34, repeated=True) - hpv_status = messages.StringField(35, repeated=True) - icd_10 = messages.StringField(36, repeated=True) - icd_o_3_histology = messages.StringField(37, repeated=True) - icd_o_3_site = messages.StringField(38, repeated=True) - lymphatic_invasion = messages.StringField(39, repeated=True) - lymphnodes_examined = messages.StringField(40, repeated=True) - lymphovascular_invasion_present = messages.StringField(41, repeated=True) - max_percent_lymphocyte_infiltration = messages.IntegerField(42, repeated=True) - max_percent_monocyte_infiltration = messages.IntegerField(43, repeated=True) - max_percent_necrosis = messages.IntegerField(44, repeated=True) - max_percent_neutrophil_infiltration = messages.IntegerField(45, repeated=True) - max_percent_normal_cells = messages.IntegerField(46, repeated=True) - max_percent_stromal_cells = messages.IntegerField(47, repeated=True) - max_percent_tumor_cells = messages.IntegerField(48, repeated=True) - max_percent_tumor_nuclei = messages.IntegerField(49, repeated=True) - menopause_status = messages.StringField(50, repeated=True) - min_percent_lymphocyte_infiltration = messages.IntegerField(51, repeated=True) - min_percent_monocyte_infiltration = messages.IntegerField(52, repeated=True) - min_percent_necrosis = messages.IntegerField(53, repeated=True) - min_percent_neutrophil_infiltration = messages.IntegerField(54, repeated=True) - min_percent_normal_cells = messages.IntegerField(55, repeated=True) - min_percent_stromal_cells = messages.IntegerField(56, repeated=True) - min_percent_tumor_cells = messages.IntegerField(57, repeated=True) - min_percent_tumor_nuclei = messages.IntegerField(58, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(59, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(60, repeated=True) - neoplasm_histologic_grade = messages.StringField(61, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(62, repeated=True) - number_of_lymphnodes_examined = messages.IntegerField(63, repeated=True) - number_of_lymphnodes_positive_by_he = messages.IntegerField(64, repeated=True) - case_barcode = messages.StringField(65, repeated=True) - pathologic_M = messages.StringField(66, repeated=True) - pathologic_N = messages.StringField(67, repeated=True) - pathologic_stage = messages.StringField(68, repeated=True) - pathologic_T = messages.StringField(69, repeated=True) - person_neoplasm_cancer_status = messages.StringField(70, repeated=True) - pregnancies = messages.StringField(71, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(72, repeated=True) - primary_therapy_outcome_success = messages.StringField(73, repeated=True) - prior_dx = messages.StringField(74, repeated=True) - Project = messages.StringField(75, repeated=True) - psa_value = messages.FloatField(76, repeated=True) - race = messages.StringField(77, repeated=True) - residual_tumor = messages.StringField(78, repeated=True) - sample_barcode = messages.StringField(79, repeated=True) - tobacco_smoking_history = messages.StringField(80, repeated=True) - tumor_tissue_site = messages.StringField(81, repeated=True) - tumor_type = messages.StringField(82, repeated=True) - weiss_venous_invasion = messages.StringField(83, repeated=True) - vital_status = messages.StringField(84, repeated=True) - weight = messages.IntegerField(85, repeated=True) - year_of_initial_pathologic_diagnosis = messages.StringField(86, repeated=True) - SampleTypeCode = messages.StringField(87, repeated=True) - has_Illumina_DNASeq = messages.StringField(88, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(89, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(90, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(91, repeated=True) - has_UNC_GA_RNASeq = messages.StringField(92, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(93, repeated=True) - has_GA_miRNASeq = messages.StringField(94, repeated=True) - has_RPPA = messages.StringField(95, repeated=True) - has_SNP6 = messages.StringField(96, repeated=True) - has_27k = messages.StringField(97, repeated=True) - has_450k = messages.StringField(98, repeated=True) - BMI = messages.StringField(99, repeated=True) - -class MetadataAttributeValues(messages.Message): - name = messages.StringField(1) - id = messages.StringField(2) - values = messages.MessageField(MetaValueListCount, 3, repeated=True) - total = messages.IntegerField(4) - -class MetadataItemList(messages.Message): - items = messages.MessageField(MetadataItem, 1, repeated=True) - count = messages.MessageField(MetaAttrValuesList, 2) - total = messages.IntegerField(3) - -class MetadataCountsItem(messages.Message): - count = messages.MessageField(MetadataAttributeValues, 1, repeated=True) - total = messages.IntegerField(2) - -class MetaDomainsList(messages.Message): - gender = messages.StringField(1, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(2, repeated=True) - country = messages.StringField(3, repeated=True) - Study = messages.StringField(4, repeated=True) - ethnicity = messages.StringField(5, repeated=True) - histological_type = messages.StringField(6, repeated=True) - icd_10 = messages.StringField(7, repeated=True) - icd_o_3_histology = messages.StringField(8, repeated=True) - icd_o_3_site = messages.StringField(9, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(10, repeated=True) - neoplasm_histologic_grade = messages.StringField(11, repeated=True) - pathologic_N = messages.StringField(12, repeated=True) - pathologic_T = messages.StringField(13, repeated=True) - pathologic_stage = messages.StringField(14, repeated=True) - person_neoplasm_cancer_status = messages.StringField(15, repeated=True) - prior_dx = messages.StringField(16, repeated=True) - Project = messages.StringField(17, repeated=True) - race = messages.StringField(18, repeated=True) - residual_tumor = messages.StringField(19, repeated=True) - SampleTypeCode = messages.StringField(20, repeated=True) - tumor_tissue_site = messages.StringField(21, repeated=True) - tumor_type = messages.StringField(22, repeated=True) - vital_status = messages.StringField(23, repeated=True) - has_Illumina_DNASeq = messages.StringField(24, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(25, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(26, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(27, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(28, repeated=True) - has_GA_miRNASeq = messages.StringField(29, repeated=True) - has_RPPA = messages.StringField(30, repeated=True) - has_SNP6 = messages.StringField(31, repeated=True) - has_27k = messages.StringField(32, repeated=True) - has_450k = messages.StringField(33, repeated=True) - - -class SampleBarcodeItem(messages.Message): - sample_barcode = messages.StringField(1) - study_id = messages.IntegerField(2) - -class MetadataAttr(messages.Message): - attribute = messages.StringField(1) - code = messages.StringField(2) - spec = messages.StringField(3) - key = messages.StringField(4) - - -class MetadataAttrList(messages.Message): - items = messages.MessageField(MetadataAttr, 1, repeated=True) - count = messages.IntegerField(2) - - -class SampleBarcodeList(messages.Message): - items = messages.MessageField(SampleBarcodeItem, 1, repeated=True) - count = messages.IntegerField(2) - - -class MetadataPlatformItem(messages.Message): - DNAseq_data = messages.StringField(1) - cnvrPlatform = messages.StringField(2) - gexpPlatform = messages.StringField(3) - methPlatform = messages.StringField(4) - mirnPlatform = messages.StringField(5) - rppaPlatform = messages.StringField(6) - - -class MetadataPlatformItemList(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - - -class MetadataCountsPlatformItem(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - count = messages.MessageField(MetadataAttributeValues, 2, repeated=True) - participants = messages.IntegerField(3) - total = messages.IntegerField(4) - - -def createDataItem(data, selectors): - if len(selectors): - item = MetadataItem() - for attr in selectors: - attr = attr.encode('utf-8') - if data[attr] is not None: - if type(data[attr]) is not long and type(data[attr]) is not int: - item.__setattr__(attr, data[attr].encode('utf-8')) - if attr.startswith('has_'): - item.__setattr__(attr, str(bool(data[attr]))) - else: - item.__setattr__(attr, data[attr]) - else: - item.__setattr__(attr, None) - return item - - -class MetadataDomainItem(messages.Message): - attribute = messages.StringField(1) - domains = messages.StringField(2, repeated=True) - - -def submit_bigquery_job(bq_service, project_id, query_body, batch=False): - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid4()) - }, - 'configuration': { - 'query': { - 'query': query_body, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - - return bq_service.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=5) - - -def is_bigquery_job_finished(bq_service, project_id, job_id): - job = bq_service.jobs().get(projectId=project_id, - jobId=job_id).execute() - - return job['status']['state'] == 'DONE' - - -def get_bq_job_results(bq_service, job_reference): - result = [] - page_token = None - - while True: - page = bq_service.jobs().getQueryResults( - pageToken=page_token, - **job_reference).execute(num_retries=2) - - if int(page['totalRows']) == 0: - break - - rows = page['rows'] - result.extend(rows) - - page_token = page.get('pageToken') - if not page_token: - break - - return result - - -def generateSQLQuery(request): - db = sql_connection() - query_dict = {} - select = '*' - sample_ids = () - selector_list = [] - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s AND project_id IS NULL;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - cursor.close() - - except (TypeError, IndexError) as e: - print e - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - if request.__getattribute__('selectors') is not None and len(request.__getattribute__('selectors')): - select = ','.join(request.selectors) - selector_list = select.split(',') # request.selectors - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None: - if key.startswith('has_'): - query_dict[key] = 1 if request.__getattribute__(key) == 'True' else 0 - else: - query_dict[key] = request.__getattribute__(key).replace('_', ' ') # values coming in with _ replaced with spaces - - query_str = 'SELECT %s FROM metadata_samples' % select - value_tuple = () - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - if request.__getattribute__('limit') is not None: - query_str += ' LIMIT %s' % request.__getattribute__('limit') - - query_str += ';' - db.close() - - return query_str, value_tuple, selector_list - - -class MetadataDomainList(messages.Message): - items = messages.MessageField(MetadataDomainItem, 1, repeated=True) - - -def normalize_metadata_ages(ages): - result = [] - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age in ages: - if type(age) != dict: - - if age.value != 'None': - int_age = float(age.value) - if int_age < 40: - new_age_list['10 to 39'] += int(age.count) - elif int_age < 50: - new_age_list['40 to 49'] += int(age.count) - elif int_age < 60: - new_age_list['50 to 59'] += int(age.count) - elif int_age < 70: - new_age_list['60 to 69'] += int(age.count) - elif int_age < 80: - new_age_list['70 to 79'] += int(age.count) - else: - new_age_list['Over 80'] += int(age.count) - else: - new_age_list['None'] += int(age.count) - - for key, value in new_age_list.items(): - result.append({'count': value, 'value': key}) - return result - - -class PlatformCount(messages.Message): - platform = messages.StringField(1) - count = messages.IntegerField(2) - - -class FileDetails(messages.Message): - sample = messages.StringField(1) - filename = messages.StringField(2) - pipeline = messages.StringField(3) - platform = messages.StringField(4) - datalevel = messages.StringField(5) - datatype = messages.StringField(6) - gg_readgroupset_id = messages.StringField(7) - cloudstorage_location = messages.StringField(8) - access = messages.StringField(9) - - -class SampleFiles(messages.Message): - total_file_count = messages.IntegerField(1) - page = messages.IntegerField(2) - platform_count_list = messages.MessageField(PlatformCount, 3, repeated=True) - file_list = messages.MessageField(FileDetails, 4, repeated=True) - - -class SampleFileCount(messages.Message): - sample_id = messages.StringField(1) - count = messages.IntegerField(2) - - -class CohortFileCountSampleList(messages.Message): - sample_list = messages.MessageField(SampleFileCount, 1, repeated=True) - - -class IncomingPlatformSelection(messages.Message): - ABSOLiD_DNASeq = messages.StringField(1) - Genome_Wide_SNP_6 = messages.StringField(2) - HumanMethylation27 = messages.StringField(3) - HumanMethylation450 = messages.StringField(4) - IlluminaGA_DNASeq = messages.StringField(5) - IlluminaGA_DNASeq_automated = messages.StringField(6) - IlluminaGA_DNASeq_Cont_automated = messages.StringField(7) - IlluminaGA_DNASeq_curated = messages.StringField(8) - IlluminaGA_miRNASeq = messages.StringField(9) - IlluminaGA_None = messages.StringField(10) - IlluminaGA_RNASeq = messages.StringField(11) - IlluminaGA_RNASeqV2 = messages.StringField(12) - IlluminaHiSeq_DNASeq = messages.StringField(13) - IlluminaHiSeq_DNASeq_automated = messages.StringField(14) - IlluminaHiSeq_DNASeq_Cont_automated = messages.StringField(15) - IlluminaHiSeq_miRNASeq = messages.StringField(16) - IlluminaHiSeq_None = messages.StringField(17) - IlluminaHiSeq_RNASeq = messages.StringField(18) - IlluminaHiSeq_RNASeqV2 = messages.StringField(19) - IlluminaHiSeq_TotalRNASeqV2 = messages.StringField(20) - IlluminaMiSeq_DNASeq = messages.StringField(21) - IlluminaMiSeq_None = messages.StringField(22) - LifeIonTorrentPGM_None = messages.StringField(23) - LifeIonTorrentProton_None = messages.StringField(24) - MDA_RPPA_Core = messages.StringField(25) - microsat_i = messages.StringField(26) - Mixed_DNASeq_Cont = messages.StringField(27) - Mixed_DNASeq_Cont_curated = messages.StringField(28) - Mixed_DNASeq_curated = messages.StringField(29) - RocheGSFLX_DNASeq = messages.StringField(30) - - -class IncomingMetadataCount(messages.Message): - filters = messages.StringField(1) - cohort_id = messages.IntegerField(2) - token = messages.StringField(3) - - -def get_current_user(request): - """ - Returns a Django_User object from either endpoints.get_current_user() or an access token. - Anytime this is used in an endpoint, request_finished.send(self) must be used - """ - user_email = None - access_token = request.__getattribute__('token') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - django.setup() - try: - if access_token is not None and user_email is None: - social_account_id = SocialToken.objects.get(token=access_token).account_id - user_id = SocialAccount.objects.get(id=social_account_id).user_id - return Django_User.objects.get(id=user_id) - elif user_email is not None: - return Django_User.objects.get(email=user_email) - else: - return None - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - return None - - -# TODO: needs to be refactored to use other samples tables -def get_participant_list(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - return results - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_list') - -# TODO: needs to be refactored to use other samples tables -def get_participant_count(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT COUNT(DISTINCT case_barcode) AS ParticipantCount FROM metadata_samples WHERE sample_barcode IN (' - first = True - samples = () - for barcode in sample_ids: - samples += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - count = 0; - cursor.execute(participant_query, samples) - for row in cursor.fetchall(): - count = row['ParticipantCount'] - - return count - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_count') - - -def count_metadata(user, cohort_id=None, sample_ids=None, filters=None): - counts_and_total = {} - sample_tables = {} - valid_attrs = {} - project_ids = () - table_key_map = {} - - if filters is None: - filters = {} - - if sample_ids is None: - sample_ids = {} - - for key in sample_ids: - samples_by_project = sample_ids[key] - sample_ids[key] = { - 'sample_barcode': build_where_clause({'sample_barcode': samples_by_project}), - } - - db = sql_connection() - django.setup() - - try: - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'sample_ids': None} - if sample_ids and None in sample_ids: - sample_tables['metadata_samples']['sample_ids'] = sample_ids[None] - - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - if row['attribute'] in METADATA_SHORTLIST: - valid_attrs[row['spec'] + ':' + row['attribute']] = { - 'name': row['attribute'], - 'tables': ('metadata_samples',), - 'sample_ids': None - } - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = {'sample_ids': None} - if sample_ids and project.id in sample_ids: - sample_tables[tables.metadata_samples_table]['sample_ids'] = sample_ids[project.id] - - features = User_Feature_Definitions.objects.filter(project__in=project_ids) - for feature in features: - if ' ' in feature.feature_name: - # It is not a column name and comes from molecular data, ignore it - continue - - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name, 'tables': (), 'sample_ids': None} - - for tables in User_Data_Tables.objects.filter(project_id=feature.project_id): - valid_attrs[key]['tables'] += (tables.metadata_samples_table,) - - if tables.metadata_samples_table not in table_key_map: - table_key_map[tables.metadata_samples_table] = {} - table_key_map[tables.metadata_samples_table][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - - if sample_ids and feature.project_id in sample_ids: - valid_attrs[key]['sample_ids'] = sample_ids[feature.project_id] - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - # For filters with no tables at this point, assume its the TCGA metadata_samples table - for key, obj in filters.items(): - if not obj['tables']: - filters[key]['tables'].append('metadata_samples') - - resulting_samples = {} - - # Loop through the features - for key, feature in valid_attrs.items(): - # Get a count for each feature - table_values = {} - feature['total'] = 0 - for table in feature['tables']: - # Check if the filters make this table 0 anyway - # We do this to avoid SQL errors for columns that don't exist - should_be_queried = True - if cohort_id and sample_tables[table]['sample_ids'] is None: - should_be_queried = False - - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - # Build Filter Where Clause - filter_copy = copy.deepcopy(filters) - key_map = table_key_map[table] if table in table_key_map else False - where_clause = build_where_clause(filter_copy, alt_key_map=key_map) - col_name = feature['name'] - if key_map and key in key_map: - col_name = key_map[key] - - cursor = db.cursor() - if should_be_queried: - # Query the table for counts and values - query = ('SELECT DISTINCT %s, COUNT(1) as count FROM %s') % (col_name, table) - sample_query = ('SELECT DISTINCT %s AS sample_id FROM %s') % ('sample_barcode', table) - query_clause = '' - if where_clause['query_str']: - query_clause = ' WHERE ' + where_clause['query_str'] - if sample_tables[table]['sample_ids']: - barcode_key = 'sample_barcode' - addt_cond = sample_tables[table]['sample_ids'][barcode_key]['query_str'] - if addt_cond and where_clause['query_str']: - query_clause += ' AND ' + addt_cond - elif addt_cond: - query_clause = ' WHERE ' + addt_cond - where_clause['value_tuple'] += sample_tables[table]['sample_ids'][barcode_key]['value_tuple'] - query += query_clause + (' GROUP BY %s ' % col_name) - sample_query += query_clause - - logger.debug("In api count_metadata, executing query "+query) - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - table_values[row[0]] += int(row[1]) - feature['total'] += int(row[1]) - - cursor.execute(sample_query, where_clause['value_tuple']) - for row in cursor.fetchall(): - resulting_samples[row[0]] = 1 - else: - # Just get the values so we can have them be 0 - cursor.execute(('SELECT DISTINCT %s FROM %s') % (col_name, table)) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - - cursor.close() - - feature['values'] = table_values - - sample_set = () - for sample in resulting_samples: - sample_set += (sample,) - - counts_and_total['participants'] = get_participant_count(sample_set) if sample_set.__len__() > 0 else 0 - counts_and_total['counts'] = [] - counts_and_total['total'] = 0 - for key, feature in valid_attrs.items(): - value_list = [] - - # Special case for age ranges and BMI - if key == 'CLIN:age_at_initial_pathologic_diagnosis': - feature['values'] = normalize_ages(feature['values']) - elif key == 'CLIN:BMI': - feature['values'] = normalize_BMI(feature['values']) - - - for value, count in feature['values'].items(): - if feature['name'].startswith('has_'): - value = 'True' if value else 'False' - - value_list.append(MetaValueListCount(value=str(value), count=count)) - - counts_and_total['counts'].append( - MetadataAttributeValues(name=feature['name'], values=value_list, id=key, total=feature['total'])) - if feature['total'] > counts_and_total['total']: - counts_and_total['total'] = feature['total'] - - db.close() - - return counts_and_total - - except (Exception) as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in count_metadata.') - - -def query_samples_and_studies(parameter, bucket_by=None): - - query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - if bucket_by is not None and bucket_by not in query_str: - logging.error("Cannot group barcodes: column '" + bucket_by + - "' not found in query string '"+query_str+"'. Barcodes will not be grouped.") - bucket_by = None - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, (parameter,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs for cohort '" + parameter + "': " + (stop-start).__str__()) - - samples = () - - if bucket_by is not None: - samples = {} - - for row in cursor.fetchall(): - if bucket_by is not None: - if row[bucket_by] not in samples: - samples[row[bucket_by]] = [] - samples[row[bucket_by]].append(row['sample_barcode']) - else: - samples += ({"sample_id": row['sample_barcode'], "project_id": row['project_id']},) - cursor.close() - db.close() - - return samples - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - -Meta_Endpoints = endpoints.api(name='meta_api', version='v1', description='Metadata endpoints used by the web application.', - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - - -@Meta_Endpoints.api_class(resource_name='meta_endpoints') -class Meta_Endpoints_API(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(IncomingMetadataItem, - limit=messages.IntegerField(2), - cohort_id=messages.IntegerField(3)) - @endpoints.method(GET_RESOURCE, MetadataPlatformItemList, - path='metadata_platform_list', http_method='GET', - name='meta.metadata_platform_list') - def metadata_platform_list(self, request): - """ Used by the web application.""" - query_dict = {} - sample_ids = None - - db = sql_connection() - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - - except (TypeError, IndexError) as e: - print e - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None and len(request.__getattribute__(key)) > 0: - values = [] - for val in request.__getattribute__(key): - values.append(val) - if key.startswith('has_'): - query_dict[key] = '1' if values[0] == 'True' else '0' - else: - query_dict[key] = ','.join(values).replace('_', ' ') - # combinations: has_UNC_HiSeq_RNASeq and has_UNC_GA_RNASeq 20 rows - # has_UNC_HiSeq_RNASeq and has_BCGSC_GA_RNASeq 209 rows - # has_BCGSC_HiSeq_RNASeq and has_UNC_HiSeq_RNASeq 919 rows - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - if request.__getattribute__('limit') is not None: - query_str += ' LIMIT %s' % request.__getattribute__('limit') - - query_str += ';' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataPlatformItemList(items=data) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - - - # UNUSED ENDPOINT - GET_RESOURCE = endpoints.ResourceContainer(IncomingMetadataItem, - limit=messages.IntegerField(2), - cohort_id=messages.IntegerField(3), - selectors=messages.StringField(4, repeated=True)) - @endpoints.method(GET_RESOURCE, MetadataItemList, - path='metadata_list', http_method='GET', - name='meta.metadata_list') - def metadata_list(self, request): - """ Used by the web application.""" - select = '*' - query_dict = {} - selector_list = [] # todo: determine use or delete this - sample_ids = None - - db = sql_connection() - query_str, value_tuple, selector_list = generateSQLQuery(request) - if debug: print >> sys.stderr,query_str - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - - for row in cursor.fetchall(): - if selector_list: - item = createDataItem(row, selector_list) - else: - item = MetadataItem( - age_at_initial_pathologic_diagnosis=None if "age_at_initial_pathologic_diagnosis" not in row or row["age_at_initial_pathologic_diagnosis"] is None else int(row["age_at_initial_pathologic_diagnosis"]), - anatomic_neoplasm_subdivision=str(row["anatomic_neoplasm_subdivision"]), - avg_percent_lymphocyte_infiltration=None if "avg_percent_lymphocyte_infiltration" not in row or row["avg_percent_lymphocyte_infiltration"] is None else float(row["avg_percent_lymphocyte_infiltration"]), - avg_percent_monocyte_infiltration=None if "avg_percent_monocyte_infiltration" not in row or row["avg_percent_monocyte_infiltration"] is None else float(row["avg_percent_monocyte_infiltration"]), - avg_percent_necrosis=None if "avg_percent_necrosis" not in row or row["avg_percent_necrosis"] is None else float(row["avg_percent_necrosis"]), - avg_percent_neutrophil_infiltration=None if "avg_percent_neutrophil_infiltration" not in row or row["avg_percent_neutrophil_infiltration"] is None else float(row["avg_percent_neutrophil_infiltration"]), - avg_percent_normal_cells=None if "avg_percent_normal_cells" not in row or row["avg_percent_normal_cells"] is None else float(row["avg_percent_normal_cells"]), - avg_percent_stromal_cells=None if "avg_percent_stromal_cells" not in row or row["avg_percent_stromal_cells"] is None else float(row["avg_percent_stromal_cells"]), - avg_percent_tumor_cells=None if "avg_percent_tumor_cells" not in row or row["avg_percent_tumor_cells"] is None else float(row["avg_percent_tumor_cells"]), - avg_percent_tumor_nuclei=None if "avg_percent_tumor_nuclei" not in row or row["avg_percent_tumor_nuclei"] is None else float(row["avg_percent_tumor_nuclei"]), - batch_number=None if "batch_number" not in row or row["batch_number"] is None else int(row["batch_number"]), - bcr=str(row["bcr"]), - clinical_M=str(row["clinical_M"]), - clinical_N=str(row["clinical_N"]), - clinical_stage=str(row["clinical_stage"]), - clinical_T=str(row["clinical_T"]), - colorectal_cancer=str(row["colorectal_cancer"]), - country=str(row["country"]), - days_to_birth=None if "days_to_birth" not in row or row['days_to_birth'] is None else int(row["days_to_birth"]), - days_to_collection=None if "days_to_collection" not in row or row['days_to_collection'] is None else int(row["days_to_collection"]), - days_to_death=None if "days_to_death" not in row or row['days_to_death'] is None else int(row["days_to_death"]), - days_to_initial_pathologic_diagnosis=None if "days_to_initial_pathologic_diagnosis" not in row or row['days_to_initial_pathologic_diagnosis'] is None else int(row["days_to_initial_pathologic_diagnosis"]), - days_to_last_followup=None if "days_to_last_followup" not in row or row['days_to_last_followup'] is None else int(row["days_to_last_followup"]), - days_to_submitted_specimen_dx=None if "days_to_submitted_specimen_dx" not in row or row['days_to_submitted_specimen_dx'] is None else int(row["days_to_submitted_specimen_dx"]), - Study=str(row["Study"]), - ethnicity=str(row["ethnicity"]), - frozen_specimen_anatomic_site=str(row["frozen_specimen_anatomic_site"]), - gender=str(row["gender"]), - height=None if "height" not in row or row['height'] is None else int(row["height"]), - histological_type=str(row["histological_type"]), - history_of_colon_polyps=str(row["history_of_colon_polyps"]), - history_of_neoadjuvant_treatment=str(row["history_of_neoadjuvant_treatment"]), - history_of_prior_malignancy=str(row["history_of_prior_malignancy"]), - hpv_calls=str(row["hpv_calls"]), - hpv_status=str(row["hpv_status"]), - icd_10=str(row["icd_10"]), - icd_o_3_histology=str(row["icd_o_3_histology"]), - icd_o_3_site=str(row["icd_o_3_site"]), - lymphatic_invasion=str(row["lymphatic_invasion"]), - lymphnodes_examined=str(row["lymphnodes_examined"]), - lymphovascular_invasion_present=str(row["lymphovascular_invasion_present"]), - max_percent_lymphocyte_infiltration=None if "max_percent_lymphocyte_infiltration" not in row or row["max_percent_lymphocyte_infiltration"] is None else int(row["max_percent_lymphocyte_infiltration"]), # 46) - max_percent_monocyte_infiltration=None if "max_percent_monocyte_infiltration" not in row or row["max_percent_monocyte_infiltration"] is None else int(row["max_percent_monocyte_infiltration"]), # 47) - max_percent_necrosis=None if "max_percent_necrosis" not in row or row["max_percent_necrosis"] is None else int(row["max_percent_necrosis"]), # 48) - max_percent_neutrophil_infiltration=None if "max_percent_neutrophil_infiltration" not in row or row["max_percent_neutrophil_infiltration"] is None else int(row["max_percent_neutrophil_infiltration"]), # 49) - max_percent_normal_cells=None if "max_percent_normal_cells" not in row or row["max_percent_normal_cells"] is None else int(row["max_percent_normal_cells"]), # 50) - max_percent_stromal_cells=None if "max_percent_stromal_cells" not in row or row["max_percent_stromal_cells"] is None else int(row["max_percent_stromal_cells"]), # 51) - max_percent_tumor_cells=None if "max_percent_tumor_cells" not in row or row["max_percent_tumor_cells"] is None else int(row["max_percent_tumor_cells"]), # 52) - max_percent_tumor_nuclei=None if "max_percent_tumor_nuclei" not in row or row["max_percent_tumor_nuclei"] is None else int(row["max_percent_tumor_nuclei"]), # 53) - menopause_status=str(row["menopause_status"]), - min_percent_lymphocyte_infiltration=None if "min_percent_lymphocyte_infiltration" not in row or row["min_percent_lymphocyte_infiltration"] is None else int(row["min_percent_lymphocyte_infiltration"]), # 55) - min_percent_monocyte_infiltration=None if "min_percent_monocyte_infiltration" not in row or row["min_percent_monocyte_infiltration"] is None else int(row["min_percent_monocyte_infiltration"]), # 56) - min_percent_necrosis=None if "min_percent_necrosis" not in row or row["min_percent_necrosis"] is None else int(row["min_percent_necrosis"]), # 57) - min_percent_neutrophil_infiltration=None if "min_percent_neutrophil_infiltration" not in row or row["min_percent_neutrophil_infiltration"] is None else int(row["min_percent_neutrophil_infiltration"]), # 58) - min_percent_normal_cells=None if "min_percent_normal_cells" not in row or row["min_percent_normal_cells"] is None else int(row["min_percent_normal_cells"]), # 59) - min_percent_stromal_cells=None if "min_percent_stromal_cells" not in row or row["min_percent_stromal_cells"] is None else int(row["min_percent_stromal_cells"]), # 60) - min_percent_tumor_cells=None if "min_percent_tumor_cells" not in row or row["min_percent_tumor_cells"] is None else int(row["min_percent_tumor_cells"]), # 61) - min_percent_tumor_nuclei=None if "min_percent_tumor_nuclei" not in row or row["min_percent_tumor_nuclei"] is None else int(row["min_percent_tumor_nuclei"]), # 62) - mononucleotide_and_dinucleotide_marker_panel_analysis_status=str(row["mononucleotide_and_dinucleotide_marker_panel_analysis_status"]), - mononucleotide_marker_panel_analysis_status=str(row["mononucleotide_marker_panel_analysis_status"]), - neoplasm_histologic_grade=str(row["neoplasm_histologic_grade"]), - new_tumor_event_after_initial_treatment=str(row["new_tumor_event_after_initial_treatment"]), - number_of_lymphnodes_examined=None if "number_of_lymphnodes_examined" not in row or row['number_of_lymphnodes_examined'] is None else int(row["number_of_lymphnodes_examined"]), - number_of_lymphnodes_positive_by_he=None if "number_of_lymphnodes_positive_by_he" not in row or row['number_of_lymphnodes_positive_by_he'] is None else int(row["number_of_lymphnodes_positive_by_he"]), - case_barcode=str(row["case_barcode"]), - pathologic_M=str(row["pathologic_M"]), - pathologic_N=str(row["pathologic_N"]), - pathologic_stage=str(row["pathologic_stage"]), - pathologic_T=str(row["pathologic_T"]), - person_neoplasm_cancer_status=str(row["person_neoplasm_cancer_status"]), - pregnancies=str(row["pregnancies"]), - primary_neoplasm_melanoma_dx=str(row["primary_neoplasm_melanoma_dx"]), - primary_therapy_outcome_success=str(row["primary_therapy_outcome_success"]), - prior_dx=str(row["prior_dx"]), - Project=str(row["Project"]), - psa_value=None if "psa_value" not in row or row["psa_value"] is None else float(row["psa_value"]), - race=str(row["race"]), - residual_tumor=str(row["residual_tumor"]), - sample_barcode=str(row["sample_barcode"]), - tobacco_smoking_history=str(row["tobacco_smoking_history"]), - tumor_tissue_site=str(row["tumor_tissue_site"]), - tumor_type=str(row["tumor_type"]), - weiss_venous_invasion=str(row["weiss_venous_invasion"]), - vital_status=str(row["vital_status"]), - weight=None if "weight" not in row or row["weight"] is None else int(float(row["weight"])), - year_of_initial_pathologic_diagnosis=str(row["year_of_initial_pathologic_diagnosis"]), - SampleTypeCode=str(row["SampleTypeCode"]), - has_Illumina_DNASeq=str(bool(row["has_Illumina_DNASeq"])), - has_BCGSC_HiSeq_RNASeq=str(bool(row["has_BCGSC_HiSeq_RNASeq"])), - has_UNC_HiSeq_RNASeq=str(bool(row["has_UNC_HiSeq_RNASeq"])), - has_BCGSC_GA_RNASeq=str(bool(row["has_BCGSC_GA_RNASeq"])), - has_UNC_GA_RNASeq=str(bool(row["has_UNC_GA_RNASeq"])), - has_HiSeq_miRnaSeq=str(bool(row["has_HiSeq_miRnaSeq"])), - has_GA_miRNASeq=str(bool(row["has_GA_miRNASeq"])), - has_RPPA=str(bool(row["has_RPPA"])), - has_SNP6=str(bool(row["has_SNP6"])), - has_27k=str(bool(row["has_27k"])), - has_450k=str(bool(row["has_450k"])) - ) - - data.append(item) - - cursor.close() - db.close() - - return MetadataItemList(items=data, total=len(data)) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - # UNUSED ENDPOINT - GET_RESOURCE = endpoints.ResourceContainer(IncomingMetadataItem, - is_landing=messages.BooleanField(2), - cohort_id=messages.IntegerField(3)) - @endpoints.method(GET_RESOURCE, MetadataItemList, - path='metadata_counts', http_method='GET', - name='meta.metadata_counts') - def metadata_counts(self, request): - """ Used by the web application.""" - query_dict = {} - sample_ids = None - is_landing = False - - if request.__getattribute__('is_landing') is not None: - is_landing = request.__getattribute__('is_landing') - - if is_landing: - try: - db = sql_connection() - cursor = db.cursor() - cursor.execute('SELECT Study, COUNT(Study) as disease_count FROM metadata_samples GROUP BY Study;') - data = [] - for row in cursor.fetchall(): - value_list_count = MetaValueListCount( - value=row[0], - count=int(row[1]) - ) - data.append(value_list_count) - - attr_values_list = MetaAttrValuesList(Study=data) - - cursor.close() - db.close() - - return MetadataItemList(count=attr_values_list) - - except (IndexError, TypeError) as e: - - raise endpoints.NotFoundException('Error in getting landing data.') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - cursor.close() - db.close() - - except (TypeError, IndexError) as e: - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None: - if key.startswith('has_'): - query_dict[key] = '1' if request.__getattribute__(key) == 'True' else '0' - else: - query_dict[key] = request.__getattribute__(key).replace('_', ' ') - - value_list = {} - total = 0 - for key in METADATA_SHORTLIST: # fix metadata_shortlist - # counts[key] = {} - domain_query_str = 'SELECT %s FROM metadata_samples GROUP BY %s' % (key, key) - value_count_query_str = 'SELECT %s, COUNT(*) FROM metadata_samples' % key - value_count_tuple = () - if len(query_dict) > 0: - - where_clause = applyFilter(key, query_dict) - - if where_clause is not None: - value_count_query_str += ' WHERE' + where_clause['query_str'] - value_count_tuple += where_clause['value_tuple'] - - if sample_ids: - if value_count_query_str.rfind('WHERE') >= 0: - value_count_query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - value_count_query_str += ' where sample_barcode in %s' % (sample_ids,) - - value_count_query_str += ' GROUP BY %s;' % key - - try: - db = sql_connection() - cursor = db.cursor() - cursor.execute(value_count_query_str, value_count_tuple) - - value_list[key] = [] - count = 0 - data = [] - data_domain = [] - # Real data counts - key_total = 0 - for row in cursor.fetchall(): - - # print row - - key_total += row[1] - # note: assumes no null values for data availability fields (which start with 'has_') - if key.startswith('has_'): - value_list[key].append(MetaValueListCount(value=str(bool(row[0])), count=row[1])) - data_domain.append(str(bool(row[0]))) - elif type(row[0]) == long: - value_list[key].append(MetaValueListCount(value=str(int(row[0])), count=row[1])) - data_domain.append(int(row[0])) - else: - value_list[key].append(MetaValueListCount(value=str(row[0]), count=row[1])) - data_domain.append(str(row[0])) - # Find the total number of samples - if key_total > total: - total = key_total - - # Fill in other values for each feature with 0 - cursor.execute(domain_query_str) - for row in cursor.fetchall(): - # note: assumes no null values for data availability fields (which start with 'has_') - if key.startswith('has_'): - if str(bool(row[0])) not in data_domain: - value_list[key].append(MetaValueListCount(value=str(bool(row[0])), count=0)) - data_domain.append(bool(row[0])) - elif str(row[0]) not in data_domain: - if type(row[0]) == long: - value_list[key].append(MetaValueListCount(value=str(int(row[0])), count=0)) - data_domain.append(int(row[0])) - else: - value_list[key].append(MetaValueListCount(value=str(row[0]), count=0)) - data_domain.append(str(row[0])) - - if key == 'age_at_initial_pathologic_diagnosis': - value_list['age_at_initial_pathologic_diagnosis'] = normalize_metadata_ages(value_list['age_at_initial_pathologic_diagnosis']) - cursor.close() - db.close() - except (KeyError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in getting value counts.') - - value_list_item = MetaAttrValuesList() - for key in METADATA_SHORTLIST: - value_list_item.__setattr__(key, None if key not in value_list else value_list[key]) - - return MetadataItemList(count=value_list_item, total=total) - - # UNUSED ENDPOINT - GET_RESOURCE = endpoints.ResourceContainer( - MetadataAttr) - @endpoints.method(GET_RESOURCE, MetadataAttrList, - path='metadata_attr_list', http_method='GET', - name='meta.metadata_attr_list') - def metadata_attr_list(self, request): - """ Used by the web application.""" - query_dict = {} - value_tuple = () - for key, value in MetadataAttr.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) != None: - query_dict[key] = request.__getattribute__(key) - - if len(query_dict) == 0: - query_str = 'SELECT * FROM metadata_attr' - else: - query_str = 'SELECT * FROM metadata_attr where' - - where_clause = build_where_clause(query_dict) - query_str += where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - data.append(MetadataAttr(attribute=str(row['attribute']), - code=str(row['code']), - spec=str(row['spec']), - )) - - cursor.close() - db.close() - return MetadataAttrList(items=data, count=len(data)) - - except (IndexError, TypeError): - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample %s not found.' % (request.id,)) - - # UNUSED ENDPOINT - @endpoints.method(message_types.VoidMessage, MetaDomainsList, - path='metadata_domains', http_method='GET', - name='meta.metadata_domains') - def domains_list(self, request): - """ Used by the web application.""" - db = sql_connection() - cursor = db.cursor() - items = {} - feature_list = MetaDomainsList.__dict__.keys() - meta_categorical_attributes = [ - 'gender', - 'history_of_neoadjuvant_treatment', - 'icd_o_3_histology', - 'prior_dx', - 'vital_status', - 'country', - 'Study', - 'histological_type', - 'icd_10', - 'icd_o_3_site', - 'tumor_tissue_site', - 'tumor_type', - 'person_neoplasm_cancer_status', - 'pathologic_N', - 'pathologic_T', - 'race', - 'ethnicity', - 'SampleTypeCode', - 'has_Illumina_DNASeq', - 'has_BCGSC_HiSeq_RNASeq', - 'has_UNC_HiSeq_RNASeq', - 'has_BCGSC_GA_RNASeq', - 'has_HiSeq_miRnaSeq', - 'has_GA_miRNASeq', - 'has_RPPA', - 'has_SNP6', - 'has_27k', - 'has_450k' - ] - - try: - for feature in feature_list: - if '__' not in feature: - query_str = 'SELECT DISTINCT %s from metadata_samples;' % feature - cursor.execute(query_str) - item_list = [] - for item in cursor.fetchall(): - if feature.startswith('has_'): - item_list.append(str(bool(int(item[0])))) - else: - item_list.append(str(item[0])) - items[feature] = item_list - items['age_at_initial_pathologic_diagnosis'] = ['10 to 39', '40 to 49', '50 to 59', '60 to 69', '70 to 79', 'Over 80', 'None'] - cursor.close() - db.close() - return MetaDomainsList( - gender = items['gender'], - history_of_neoadjuvant_treatment = items['history_of_neoadjuvant_treatment'], - icd_o_3_histology = items['icd_o_3_histology'], - prior_dx = items['prior_dx'], - vital_status = items['vital_status'], - country = items['country'], - Study = items['Study'], - histological_type = items['histological_type'], - icd_10 = items['icd_10'], - icd_o_3_site = items['icd_o_3_site'], - tumor_tissue_site = items['tumor_tissue_site'], - tumor_type = items['tumor_type'], - person_neoplasm_cancer_status = items['person_neoplasm_cancer_status'], - pathologic_N = items['pathologic_N'], - pathologic_T = items['pathologic_T'], - race = items['race'], - ethnicity = items['ethnicity'], - SampleTypeCode = items['SampleTypeCode'], - has_Illumina_DNASeq = items['has_Illumina_DNASeq'], - has_BCGSC_HiSeq_RNASeq = items['has_BCGSC_HiSeq_RNASeq'], - has_UNC_HiSeq_RNASeq = items['has_UNC_HiSeq_RNASeq'], - has_BCGSC_GA_RNASeq = items['has_BCGSC_GA_RNASeq'], - has_HiSeq_miRnaSeq = items['has_HiSeq_miRnaSeq'], - has_GA_miRNASeq = items['has_GA_miRNASeq'], - has_RPPA = items['has_RPPA'], - has_SNP6 = items['has_SNP6'], - has_27k = items['has_27k'], - has_450k = items['has_450k'] - ) - - except (IndexError, TypeError): - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in meta_domains') - - GET_RESOURCE = endpoints.ResourceContainer(IncomingPlatformSelection, - cohort_id=messages.IntegerField(1, required=True), - page=messages.IntegerField(2), - limit=messages.IntegerField(3), - token=messages.StringField(4), - platform_count_only=messages.StringField(5), - offset=messages.IntegerField(6) - ) - - @endpoints.method(GET_RESOURCE, SampleFiles, - path='cohort_files', http_method='GET', - name='meta.cohort_files') - def cohort_files(self, request): - """ Used by the web application.""" - limit = 20 - page = 1 - offset = 0 - cohort_id = request.cohort_id - - platform_count_only = request.__getattribute__('platform_count_only') - is_dbGaP_authorized = False - user_email = None - user_id = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - # users have the option of pasting the access token in the query string - # or in the 'token' field in the api explorer - # but this is not required - access_token = request.__getattribute__('token') - if access_token: - user_email = get_user_email_from_token(access_token) - - if user_email or user_id: - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.error(e) - logger.error(traceback.format_exc()) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - logger.error(traceback.format_exc()) - request_finished.send(self) - raise endpoints.UnauthorizedException("%s does not have permission to view cohort %d." % (user_email, cohort_id)) - - try: - nih_user = NIH_User.objects.get(user_id=user_id) - is_dbGaP_authorized = nih_user.dbGaP_authorized and nih_user.active - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn("[STATUS] %s does not have an entry in NIH_User: %s" % (user_email, str(e))) - else: - logger.error("[ERROR] Authentication required for cohort_files endpoint.") - raise endpoints.UnauthorizedException("No user email found.") - - if request.__getattribute__('page') is not None: - page = request.page - offset = (page - 1) * 20 - elif request.__getattribute__('offset') is not None: - offset = request.offset - - if request.__getattribute__('limit') is not None: - limit = request.limit - - sample_query = 'select sample_barcode from cohorts_samples where cohort_id=%s;' - sample_list = () - db = None - cursor = None - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query, (cohort_id,)) - in_clause = '(' - first = True - for row in cursor.fetchall(): - sample_list += (row['sample_barcode'],) - if first: - in_clause += '%s' - first = False - else: - in_clause += ',%s' - in_clause += ')' - except (IndexError, TypeError): - logger.error("Error obtaining list of samples in cohort file list") - logger.error(traceback.format_exc()) - raise endpoints.ServiceException('Error obtaining list of samples in cohort file list') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - platform_count_query = 'select Platform, count(Platform) as platform_count from metadata_data where sample_barcode in {0} and DatafileUploaded="true" group by Platform order by sample_barcode;'.format(in_clause) - query = 'select sample_barcode, DatafileName, DatafileNameKey, SecurityProtocol, Pipeline, Platform, DataLevel, Datatype, GG_readgroupset_id, Repository, SecurityProtocol from metadata_data where sample_barcode in {0} and DatafileUploaded="true" '.format(in_clause) - - # Check for incoming platform selectors - platform_selector_list = [] - for key, value in IncomingPlatformSelection.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None and request.__getattribute__(key) == 'True': - platform_selector_list.append(key) - - if len(platform_selector_list): - query += ' and Platform in ("' + '","'.join(platform_selector_list) + '")' - - query_tuple = sample_list - - if limit > 0: - query += ' limit %s' - query_tuple += (limit,) - # Offset is only valid when there is a limit - if offset > 0: - query += ' offset %s' - query_tuple += (offset,) - - query += ';' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(platform_count_query, sample_list) - platform_count_list = [] - count = 0 - if cursor.rowcount > 0: - for row in cursor.fetchall(): - if len(platform_selector_list): - if row['Platform'] in platform_selector_list: - count += int(row['platform_count']) - else: - count += int(row['platform_count']) - platform_count_list.append(PlatformCount(platform=row['Platform'], count=row['platform_count'])) - else: - platform_count_list.append(PlatformCount(platform='None', count=0)) - - file_list = [] - if not platform_count_only: - cursor.execute(query, query_tuple) - if cursor.rowcount > 0: - for item in cursor.fetchall(): - # If there's a datafilenamekey - if 'DatafileNameKey' in item and item['DatafileNameKey'] != '': - # Find protected bucket it should belong to - bucket_name = '' - if item['Repository'] and item['Repository'].lower() == 'dcc': - bucket_name = settings.DCC_CONTROLLED_DATA_BUCKET - elif item['Repository'] and item['Repository'].lower() == 'cghub': - bucket_name = settings.CGHUB_CONTROLLED_DATA_BUCKET - else: - bucket_name = settings.OPEN_DATA_BUCKET - - item['DatafileNameKey'] = "gs://{}{}".format(bucket_name, item['DatafileNameKey']) - - file_list.append(FileDetails(sample=item['sample_barcode'], cloudstorage_location=item['DatafileNameKey'], access=(item['SecurityProtocol'] or 'N/A'), filename=item['DatafileName'], pipeline=item['Pipeline'], platform=item['Platform'], datalevel=item['DataLevel'], datatype=(item['Datatype'] or " "), gg_readgroupset_id=item['GG_readgroupset_id'])) - else: - file_list.append(FileDetails(sample='None', filename='', pipeline='', platform='', datalevel='')) - return SampleFiles(total_file_count=count, page=page, platform_count_list=platform_count_list, file_list=file_list) - - except Exception as e: - logger.error("Error obtaining platform counts") - logger.error(traceback.format_exc()) - raise endpoints.ServiceException('Error getting counts') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - GET_RESOURCE = endpoints.ResourceContainer(sample_id=messages.StringField(1, required=True)) - @endpoints.method(GET_RESOURCE, SampleFiles, - path='sample_files', http_method='GET', - name='meta.sample_files') - def sample_files(self, request): - ''' - Takes a sample_barcode and returns a list of filenames and their associated cloudstorage locations, pipelines, and platforms. - Also returns counts for the number of files derived from each platform. - Checks user for dbGaP authorization and returns "Restricted" if a filename is controlled access and the user is not dbGaP authorized - :param sample_id: sample_barcode - :return SampleFiles: a list of filenames and their associated cloudstorage locations, pipelines, and platforms - as well as counts for the number of files for each platform. - ''' - - global cloudstorage_location - sample_id = request.sample_id - dbGaP_authorized = False - - query = "select sample_barcode, " \ - "DatafileName, " \ - "Pipeline, " \ - "Platform, " \ - "IF(SecurityProtocol LIKE '%%open%%', DatafileNameKey, 'Restricted') as DatafileNameKey, " \ - "SecurityProtocol " \ - "from metadata_data " \ - "where sample_barcode=%s;" - - if endpoints.get_current_user(): - user_email = endpoints.get_current_user().email() - try: - user_id = Django_User.objects.get(email=user_email) - nih_user = NIH_User.objects.get(user_id=user_id) - dbGaP_authorized = nih_user.dbGaP_authorized and nih_user.active - if dbGaP_authorized: - query = "select sample_barcode, " \ - "DatafileName, " \ - "Pipeline, " \ - "Platform, " \ - "DatafileNameKey, " \ - "SecurityProtocol, " \ - "Repository " \ - "from metadata_data " \ - "where sample_barcode=%s;" - except (ObjectDoesNotExist, MultipleObjectsReturned) as e: - if type(e) is MultipleObjectsReturned: - logger.error("Meta.sample_files endpoint with user {} gave error: {}".format(user_email, str(e))) - - platform_query = "select Platform, " \ - "count(Platform) as platform_count " \ - "from metadata_data " \ - "where sample_barcode=%s " \ - "group by Platform;" - - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - try: - cursor.execute(query, (sample_id,)) - file_list = [] - platform_list = [] - for item in cursor.fetchall(): - if len(item.get('DatafileNameKey', '')) == 0: - cloudstorage_location = 'File location not found.' - elif 'open' in item['SecurityProtocol']: - cloudstorage_location = 'gs://{}{}'.format(OPEN_DATA_BUCKET, item['DatafileNameKey']) - elif dbGaP_authorized: - # hard-coding mock bucket names for now --testing purposes only - if item['Repository'].lower() == 'dcc': - cloudstorage_location = 'gs://{}{}'.format( - 'gs://62f2c827-mock-mock-mock-1cde698a4f77', item['DatafileNameKey']) - elif item['Repository'].lower() == 'cghub': - cloudstorage_location = 'gs://{}{}'.format( - 'gs://360ee3ad-mock-mock-mock-52f9a5e7f99a', item['DatafileNameKey']) - else: - cloudstorage_location = item.get('DatafileNameKey') # this will return "Restricted" - - file_list.append( - FileDetails( - filename=item['DatafileName'], - pipeline=item['Pipeline'], - platform=item['Platform'], - cloudstorage_location=cloudstorage_location - ) - ) - cursor.execute(platform_query, (sample_id,)) - for item in cursor.fetchall(): - platform_list.append(PlatformCount(platform=item['Platform'], count=item['platform_count'])) - cursor.close() - db.close() - return SampleFiles(total_file_count=len(file_list), page=1, platform_count_list=platform_list, file_list=file_list) - except Exception as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error getting file details: {}'.format(str(e))) - -""" -Metadata Endpoints v2 - -Includes User Uploaded Data -""" -Meta_Endpoints_v2 = endpoints.api(name='meta_api', version='v2', - description='Retrieve metadata information relating to projects, cohorts, and other data', - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - -@Meta_Endpoints_v2.api_class(resource_name='meta_endpoints') -class Meta_Endpoints_API_v2(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer( - MetadataAttr, - token=messages.StringField(4), - ) - @endpoints.method(GET_RESOURCE, MetadataAttrList, - path='attributes', http_method='GET', - name='meta.attr_list') - def metadata_attr_list(self, request): - - cursor = None - db = None - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - query_dict = {} - value_tuple = () - for key, value in MetadataAttr.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) != None: - query_dict[key] = request.__getattribute__(key) - - if len(query_dict) == 0: - query_str = 'SELECT * FROM metadata_attr' - else: - query_str = 'SELECT * FROM metadata_attr where' - where_clause = build_where_clause(query_dict) - query_str += where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - data.append(MetadataAttr(attribute=str(row['attribute']), - code=str(row['code']), - spec=str(row['spec']), - key=str(row['spec']) + ':' + str(row['attribute']) - )) - - if user: - studies = Project.get_user_studies(user) - feature_defs = User_Feature_Definitions.objects.filter(project__in=studies) - for feature in feature_defs: - data_table = User_Data_Tables.objects.get(project=feature.project).metadata_samples_table - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - - data.append(MetadataAttr(attribute=name, - code='N' if feature.is_numeric else 'C', - spec='USER', - key=key - )) - - cursor.close() - db.close() - request_finished.send(self) - return MetadataAttrList(items=data, count=len(data)) - - except (IndexError, TypeError): - raise endpoints.InternalServerErrorException('Error retrieving attribute list') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsItem, - path='metadata_counts', http_method='POST', - name='meta.metadata_counts') - def metadata_counts(self, request): - filters = {} - sample_ids = None - cohort_id = None - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for reqFilter in tmp: - key = reqFilter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(reqFilter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - start = time.time() - counts_and_totals = count_metadata(user, cohort_id, sample_ids, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts" - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - request_finished.send(self) - return MetadataCountsItem(count=counts_and_totals['counts'], total=counts_and_totals['total']) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - @endpoints.method(POST_RESOURCE, SampleBarcodeList, - path='metadata_sample_list', http_method='POST', - name='meta.metadata_sample_list') - def metadata_list(self, request): - filters = {} - valid_attrs = {} - sample_tables = {} - table_key_map = {} - sample_ids = None - project_ids = () - cohort_id = None - cursor = None - db = None - mutation_filters = None - mutation_results = {} - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if 'MUT:' in key: - if not mutation_filters: - mutation_filters = {} - if not key in mutation_filters: - mutation_filters[key] = [] - mutation_filters[key].append(filter['value']) - else: - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - django.setup() - - # TODO enable filtering based off of this - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - if mutation_filters: - mutation_where_clause = build_where_clause(mutation_filters) - print >> sys.stdout, mutation_where_clause - cohort_join_str = '' - cohort_where_str = '' - bq_cohort_table = '' - bq_cohort_dataset = '' - cohort = '' - query_template = None - - if cohort_id is not None: - query_template = \ - ("SELECT ct.sample_barcode" - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] ct" - " JOIN (SELECT Tumor_SampleBarcode AS barcode " - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY barcode) mt" - " ON mt.barcode = ct.sample_barcode" - " WHERE ct.cohort_id = {cohort};") - bq_cohort_table = settings.BIGQUERY_COHORT_TABLE_ID - bq_cohort_dataset = settings.COHORT_DATASET_ID - cohort = cohort_id - else: - query_template = \ - ("SELECT Tumor_SampleBarcode" - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY Tumor_SampleBarcode; ") - - params = mutation_where_clause['value_tuple'][0] - - query = query_template.format(dataset_name=settings.BIGQUERY_DATASET, - project_name=settings.BIGQUERY_PROJECT_NAME, - table_name="Somatic_Mutation_calls", hugo_symbol=str(params['gene']), - var_class=params['var_class'], cohort_dataset=bq_cohort_dataset, - cohort_table=bq_cohort_table, cohort=cohort) - - bq_service = authorize_credentials_with_Google() - query_job = submit_bigquery_job(bq_service, settings.BQ_PROJECT_ID, query) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - retries = 0 - - while not job_is_done and retries < 10: - retries += 1 - sleep(1) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - results = get_bq_job_results(bq_service, query_job['jobReference']) - - # for-each result, add to list - - if results.__len__() > 0: - for barcode in results: - mutation_results[str(barcode['f'][0]['v'])] = 1 - - else: - print >> sys.stdout, "Mutation filter result was empty!" - # Put in one 'not found' entry to zero out the rest of the queries - barcodes = ['NONE_FOUND', ] - - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'features': {}, 'barcode': 'sample_barcode', 'project_id': None} - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - key = row['spec'] + ':' + row['attribute'] - valid_attrs[key] = {'name': row['attribute']} - sample_tables['metadata_samples']['features'][key] = row['attribute'] - if key in filters: - filters[key]['tables'] += ('metadata_samples',) - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - # Add all tables from each project - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = { - 'features':{}, - 'barcode':'sample_barcode', - 'project_id': project.id - } - - # Record features that should be in each sample table so we can know how and when we need to query - for feature in User_Feature_Definitions.objects.filter(project=project): - name = feature.feature_name - key = 'study:' + str(project.id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name} - - for tables in User_Data_Tables.objects.filter(project=feature.project_id): - sample_tables[tables.metadata_samples_table]['features'][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - results = [] - # Loop through the sample tables - for table, table_settings in sample_tables.items(): - # Make sure we should run the query here, or if we have filters that won't return anything, skip - should_be_queried = True - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - if not should_be_queried: - continue - - filter_copy = copy.deepcopy(filters) - where_clause = build_where_clause(filter_copy, table_settings['features']) - query = 'SELECT DISTINCT %s FROM %s' % (table_settings['barcode'], table) - if where_clause['query_str']: - query += ' WHERE ' + where_clause['query_str'] - cursor = db.cursor() - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - project_id = table_settings['project_id'] - if cohort_id and (project_id not in sample_ids or row[0] not in sample_ids[project_id]): - # This barcode was not in our cohort's list of barcodes, skip it - continue - if mutation_filters: - if row[0] in mutation_results: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id'])) - else: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id']) ) - cursor.close() - - db.close() - request_finished.send(self) - return SampleBarcodeList( items=results, count=len(results) ) - - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - ) - @endpoints.method(GET_RESOURCE, SampleBarcodeList, - path='metadata_participant_list', http_method='GET', - name='meta.metadata_participant_list') - def metadata_participant_list(self, request): - cursor = None - db = None - - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = [] - - for row in cursor.fetchall(): - sample_ids.append(row['sample_barcode']) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - cursor.close() - db.close() - return SampleBarcodeList(items=results, count=len(results)) - - except (TypeError, IndexError) as e: - raise endpoints.NotFoundException('Error in retrieving barcodes.') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataPlatformItemList, - path='metadata_platform_list', http_method='POST', - name='meta.metadata_platform_list') - def metadata_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - cursor = None - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(sample_query_str, (cohort_id,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs in metadata_platform_list for cohort '" + cohort_id + "': " + (stop - start).__str__()) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - - except (TypeError, IndexError) as e: - print e - cursor.close() - db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_platform_list for cohort '" + str(request.cohort_id) + "': " + (stop - start).__str__()) - data = [] - for row in cursor.fetchall(): - - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataPlatformItemList(items=data) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsPlatformItem, - path='metadata_counts_platform_list', http_method='POST', - name='meta.metadata_counts_platform_list') - def metadata_counts_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - samples_by_project = None - cohort_id = None - participants = 0 - user = get_current_user(request) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - samples = query_samples_and_studies(cohort_id, ) - - sample_ids = () - samples_by_project = {} - - for sample in samples: - sample_ids += (sample['sample_id'],) - if sample['project_id'] not in samples_by_project: - samples_by_project[sample['project_id']] = [] - samples_by_project[sample['project_id']].append(sample['sample_id']) - - participants = get_participant_count(sample_ids) - - start = time.time() - counts_and_total = count_metadata(user, cohort_id, samples_by_project, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts " - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - db = sql_connection() - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - data = [] - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_counts_platform_list for cohort '" + str( - request.cohort_id) + "': " + (stop - start).__str__()) - for row in cursor.fetchall(): - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataCountsPlatformItem(items=data, count=counts_and_total['counts'], - participants=counts_and_total['participants'], - total=counts_and_total['total']) - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Exception in metadata_counts_platforms_list.') diff --git a/api/pairwise.py b/api/pairwise.py deleted file mode 100755 index acb7f8ba..00000000 --- a/api/pairwise.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import json -import base64 -import logging -import urllib -import traceback - -from google.appengine.api import urlfetch -from django.conf import settings - -from bq_data_access.data_access import get_feature_vector -from bq_data_access.feature_value_types import ValueType -from bq_data_access.utils import VectorMergeSupport - -logger = logging.getLogger(__name__) - -class PairwiseInputVector(object): - def __init__(self, feature_id, value_type, data): - self.feature_id = feature_id - self.value_type = value_type - self.data = data - - -class Pairwise(object): - def __init__(self): - pass - - @classmethod - def prepare_features(self, cohort_id, features): - # Get the feature data - feature_vector_mapping = {} - vectors = [] - for feature in features: - value_type, vector = get_feature_vector(feature, cohort_id) - - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - vms = VectorMergeSupport('NA', 'sample_id', row_ids=features) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def prepare_feature_vector(self, input_vectors): - feature_vector_mapping = {} - vectors = [] - for item in input_vectors: - feature_id, value_type, vector = item.feature_id, item.value_type, item.data - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature_id] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - feature_ids = [v.feature_id for v in input_vectors] - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', row_ids=feature_ids) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def run_pairwise(self, feature_rows): - url = settings.PAIRWISE_SERVICE_URL - - data_dict = {} - row_count = 1 - for row in feature_rows: - label = "row_{count}".format(count=row_count) - data_dict[label] = row - row_count += 1 - - # Encode the data to be sent to the service - data = urllib.urlencode(data_dict) - decoded_response = None - - try: - pairwise_response = urlfetch.fetch(url=url, payload=data, method=urlfetch.POST) - response = pairwise_response.content - decoded_response = json.loads(base64.b64decode(response)) - except Exception as e: - decoded_response = None - logger.error(traceback.format_exc()) - - return decoded_response diff --git a/api/pairwise_api.py b/api/pairwise_api.py deleted file mode 100644 index d089916c..00000000 --- a/api/pairwise_api.py +++ /dev/null @@ -1,166 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -import endpoints -from protorpc import messages, remote, message_types -from django.conf import settings - -from api.api_helpers import authorize_credentials_with_Google -from api.pairwise import Pairwise - -package = 'pairwise' - - -class PairwiseJobRequest(messages.Message): - cohort_id = messages.StringField(1, required=True) - feature = messages.StringField(2, repeated=True) - - -class PairwiseResultVector(messages.Message): - feature_1 = messages.StringField(1, required=True) - feature_2 = messages.StringField(2, required=True) - comparison_type = messages.StringField(3, required=True) - correlation_coefficient = messages.StringField(4, required=True) - n = messages.IntegerField(5, required=True) - _logp = messages.FloatField(6, required=True) - n_A = messages.IntegerField(7, required=True) - p_A = messages.FloatField(8, required=True) - n_B = messages.IntegerField(9, required=True) - p_B = messages.FloatField(10, required=True) - exclusion_rules = messages.StringField(11, required=True) - - -class PairwiseFilterMessage(messages.Message): - filter_message = messages.StringField(1, required=True) - - -class PairwiseResults(messages.Message): - result_vectors = messages.MessageField(PairwiseResultVector, 1, repeated=True) - filter_messages = messages.MessageField(PairwiseFilterMessage, 2, repeated=True) - - -class Feature(messages.Message): - annotated_type = messages.StringField(1) - chr = messages.StringField(2) - start = messages.IntegerField(3) - end = messages.IntegerField(4) - label = messages.StringField(5) - mutation_count = messages.IntegerField(6) - source = messages.StringField(7) - - -class Association(messages.Message): - node1 = messages.MessageField(Feature, 1) - node2 = messages.MessageField(Feature, 2) - logged_pvalue = messages.FloatField(3) - - -class CircvizOutput(messages.Message): - items = messages.MessageField(Association, 1, repeated=True) - - -Pairwise_Endpoints = endpoints.api(name='pairwise', version='v1') - - -@Pairwise_Endpoints.api_class(resource_name='pairwise_api') -class PairwiseApi(remote.Service): - """Pairwise API v1""" - - @endpoints.method(PairwiseJobRequest, PairwiseResults, name="run", http_method="POST") - def run_job(self, request): - """ Used by the web application.""" - features = [] - count = len(request.feature) - 1 - while count >= 0: - features.append(str(request.feature[count])) - count -= 1 - - prepped_features = Pairwise.prepare_features(request.cohort_id, features) - outputs = Pairwise.run_pairwise(prepped_features) - - results = PairwiseResults(result_vectors=[], filter_messages=[]) - logging.info(results) - - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - - return results - - @endpoints.method(message_types.VoidMessage, CircvizOutput, - path='precomp', http_method='GET', name='precomputed') - def precomputed_results(self, request): - """ Used by the web application.""" - bq_table = 'brca_pwpv' - query = 'SELECT A_valueType, A_chr, A_startPos, A_endPos, A_featureName, A_N, A_dataType,' \ - 'B_valueType, B_chr, B_startPos, B_endPos, B_featureName, B_N, B_dataType,' \ - 'logP FROM [isb-cgc:test.brca_pwpv] ' \ - 'where B_chr != "null" ' \ - 'and A_chr != "null"' \ - 'and A_startPos != "null" and A_endPos != "null"' \ - 'and B_startPos != "null" and B_endPos != "null"' \ - 'LIMIT 50;' - query_body = { - 'query': query - } - bigquery_service = authorize_credentials_with_Google() - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=settings.BQ_PROJECT_ID, body=query_body).execute() - association_list = [] - feature_list = [] - for row in query_response['rows']: - node1 = Feature( - annotated_type=row['f'][0]['v'].encode('utf-8') if row['f'][0]['v'] else None, - chr=row['f'][1]['v'].encode('utf-8').replace('chr','') if row['f'][1]['v'] else None, - start=int(row['f'][2]['v']) if row['f'][2]['v'] else None, - end=int(row['f'][3]['v']) if row['f'][3]['v'] else None, - label=row['f'][4]['v'].encode('utf-8') if row['f'][4]['v'] else '', - mutation_count=int(row['f'][5]['v']) if row['f'][5]['v'] else None, - source=row['f'][6]['v'].encode('utf-8') if row['f'][6]['v'] else None - ) - node2 = Feature( - annotated_type=row['f'][7]['v'].encode('utf-8') if row['f'][7]['v'] else None, - chr=row['f'][8]['v'].encode('utf-8').replace('chr','') if row['f'][8]['v'] else None, - start=int(row['f'][9]['v']) if row['f'][9]['v'] else None, - end=int(row['f'][10]['v']) if row['f'][10]['v'] else None, - label=row['f'][11]['v'].encode('utf-8') if row['f'][11]['v'] else '', - mutation_count=int(row['f'][12]['v']) if row['f'][12]['v'] else None, - source=row['f'][13]['v'].encode('utf-8') if row['f'][13]['v'] else None - ) - logP = float(row['f'][14]['v']) - association_list.append(Association(node1=node1, node2=node2, logged_pvalue=logP)) - feature_list.append(node1) - feature_list.append(node2) - return CircvizOutput(items=association_list) - - -APPLICATION = endpoints.api_server([PairwiseApi]) diff --git a/api/schema/__init__.py b/api/schema/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api/schema/tcga_clinical.py b/api/schema/tcga_clinical.py deleted file mode 100755 index 0ec1191b..00000000 --- a/api/schema/tcga_clinical.py +++ /dev/null @@ -1,283 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -# Updated from -# https://github.com/isb-cgc/data-prototyping/blob/537c5c24646f87bda804ca95dee6cf479f0b1fb9/tcga_etl_pipeline/schemas/clinical.json - -schema = [ - { - "type": "STRING", - "name": "ParticipantBarcode" - }, - { - "type": "STRING", - "name": "Study" - }, - { - "type": "STRING", - "name": "Project" - }, - { - "type": "STRING", - "name": "ParticipantUUID" - }, - { - "type": "STRING", - "name": "TSSCode" - }, - { - "type": "INTEGER", - "name": "age_at_initial_pathologic_diagnosis" - }, - { - "type": "STRING", - "name": "anatomic_neoplasm_subdivision" - }, - { - "type": "INTEGER", - "name": "batch_number" - }, - { - "type": "STRING", - "name": "bcr" - }, - { - "type": "STRING", - "name": "clinical_M" - }, - { - "type": "STRING", - "name": "clinical_N" - }, - { - "type": "STRING", - "name": "clinical_T" - }, - { - "type": "STRING", - "name": "clinical_stage" - }, - { - "type": "STRING", - "name": "colorectal_cancer" - }, - { - "type": "STRING", - "name": "country" - }, - { - "type": "STRING", - "name": "vital_status" - }, - { - "type": "INTEGER", - "name": "days_to_birth" - }, - { - "type": "INTEGER", - "name": "days_to_death" - }, - { - "type": "INTEGER", - "name": "days_to_last_known_alive" - }, - { - "type": "INTEGER", - "name": "days_to_last_followup" - }, - { - "type": "INTEGER", - "name": "days_to_initial_pathologic_diagnosis" - }, - { - "type": "INTEGER", - "name": "days_to_submitted_specimen_dx" - }, - { - "type": "STRING", - "name": "ethnicity" - }, - { - "type": "STRING", - "name": "frozen_specimen_anatomic_site" - }, - { - "type": "STRING", - "name": "gender" - }, - { - "type": "FLOAT", - "name": "gleason_score_combined" - }, - { - "type": "STRING", - "name": "histological_type" - }, - { - "type": "STRING", - "name": "history_of_colon_polyps" - }, - { - "type": "STRING", - "name": "history_of_neoadjuvant_treatment" - }, - { - "type": "STRING", - "name": "hpv_calls" - }, - { - "type": "STRING", - "name": "hpv_status" - }, - { - "type": "STRING", - "name": "icd_10" - }, - { - "type": "STRING", - "name": "icd_o_3_histology" - }, - { - "type": "STRING", - "name": "icd_o_3_site" - }, - { - "type": "STRING", - "name": "lymphatic_invasion" - }, - { - "type": "STRING", - "name": "lymphnodes_examined" - }, - { - "type": "STRING", - "name": "lymphovascular_invasion_present" - }, - { - "type": "STRING", - "name": "menopause_status" - }, - { - "type": "STRING", - "name": "mononucleotide_and_dinucleotide_marker_panel_analysis_status" - }, - { - "type": "FLOAT", - "name": "mononucleotide_marker_panel_analysis_status" - }, - { - "type": "STRING", - "name": "neoplasm_histologic_grade" - }, - { - "type": "STRING", - "name": "new_tumor_event_after_initial_treatment" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_examined" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_positive_by_he" - }, - { - "type": "FLOAT", - "name": "number_pack_years_smoked" - }, - { - "type": "INTEGER", - "name": "year_of_initial_pathologic_diagnosis" - }, - { - "type": "STRING", - "name": "pathologic_M" - }, - { - "type": "STRING", - "name": "pathologic_N" - }, - { - "type": "STRING", - "name": "pathologic_T" - }, - { - "type": "STRING", - "name": "pathologic_stage" - }, - { - "type": "STRING", - "name": "person_neoplasm_cancer_status" - }, - { - "type": "STRING", - "name": "pregnancies" - }, - { - "type": "STRING", - "name": "primary_neoplasm_melanoma_dx" - }, - { - "type": "STRING", - "name": "primary_therapy_outcome_success" - }, - { - "type": "STRING", - "name": "prior_dx" - }, - { - "type": "FLOAT", - "name": "psa_value" - }, - { - "type": "STRING", - "name": "race" - }, - { - "type": "STRING", - "name": "residual_tumor" - }, - { - "type": "STRING", - "name": "tobacco_smoking_history" - }, - { - "type": "STRING", - "name": "tumor_tissue_site" - }, - { - "type": "STRING", - "name": "tumor_type" - }, - { - "type": "STRING", - "name": "venous_invasion" - }, - { - "type": "FLOAT", - "name": "weight" - }, - { - "type": "FLOAT", - "name": "height" - }, - { - "type": "FLOAT", - "name": "BMI" - } -] \ No newline at end of file diff --git a/api/seqpeek_api.py b/api/seqpeek_api.py deleted file mode 100644 index 2c8e2c4a..00000000 --- a/api/seqpeek_api.py +++ /dev/null @@ -1,94 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from api.data_access import PlotDataCohortInfo - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class MAFRecord(Message): - sample_id = StringField(1) - patient_id = StringField(2) - aliquot_id = StringField(3) - hugo_symbol = StringField(4) - uniprot_aapos = IntegerField(5, variant=Variant.INT32) - uniprot_id = StringField(6) - variant_classification = StringField(7) - cohort = IntegerField(8, repeated=True) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 2, repeated=True) - -SeqPeekDataEndpointsAPI = endpoints_api(name='seqpeek_data_api', version='v1', - description='Endpoints used by the seqpeek visualization in the web application.') - - -def maf_array_to_record(maf_array): - data_points = [] - for item in maf_array: - data_points.append(MAFRecord(**item)) - - return data_points - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekDataAccessAPI(remote.Service): - - def create_response(self, maf_with_cohorts): - - data_points = maf_array_to_record(maf_with_cohorts.maf_vector) - - cohort_info_obj_array = [] - for item in maf_with_cohorts.cohort_info: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return MAFRecordList(items=data_points, cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, MAFRecordList, - path='by_gnab_feature', http_method='GET', name='seqpeek.getMAFDataWithCohorts') - def data_access_by_gnab_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - - maf_with_cohorts = SeqPeekMAFDataFormatter().format_maf_vector_for_view(feature_id, cohort_id_array) - response = self.create_response(maf_with_cohorts) - return response - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api/seqpeek_view_api.py b/api/seqpeek_view_api.py deleted file mode 100644 index b7dbabe2..00000000 --- a/api/seqpeek_view_api.py +++ /dev/null @@ -1,238 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from endpoints import method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant - -from bq_data_access.seqpeek.seqpeek_view import SeqPeekViewDataBuilder -from bq_data_access.data_access import get_feature_vectors_tcga_only -from api.seqpeek_api import SeqPeekDataEndpointsAPI, MAFRecord, maf_array_to_record -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter -from bq_data_access.seqpeek_maf_data import SeqPeekDataProvider -from bq_data_access.data_access import ProviderClassQueryDescription -from api.data_access import fetch_isbcgc_project_set -from api.api_helpers import sql_connection - -from projects.models import Project - -class SeqPeekViewDataRequest(Message): - hugo_symbol = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class InterproMatchLocation(Message): - # TODO this should likely be a float - score = IntegerField(1, variant=Variant.INT32) - start = IntegerField(2, variant=Variant.INT32) - end = IntegerField(3, variant=Variant.INT32) - - -class InterproMatch(Message): - status = StringField(1) - name = StringField(2) - evd = StringField(3) - locations = MessageField(InterproMatchLocation, 4, repeated=True) - dbname = StringField(5) - id = StringField(6) - - -class InterproJson(Message): - matches = MessageField(InterproMatch, 1, repeated=True) - uniprot_id = StringField(2) - length = IntegerField(3, variant=Variant.INT32) - name = StringField(4) - - -class InterproItem(Message): - uniprot_id = StringField(1) - interpro_json = MessageField(InterproJson, 2, repeated=False) - - -class SeqPeekRegionRecord(Message): - type = StringField(1, required=True) - start = IntegerField(2, required=True, variant=Variant.INT32) - end = IntegerField(3, required=True, variant=Variant.INT32) - - -class SeqPeekTrackRecord(Message): - mutations = MessageField(MAFRecord, 1, repeated=True) - type = StringField(2, required=True) - label = StringField(3, required=True) - number_of_samples = IntegerField(4, required=True) - mutated_positions = IntegerField(5, required=True) - cohort_size = IntegerField(6, required=False) - row_id = StringField(7, required=True) - - -class SeqPeekViewPlotDataRecord(Message): - tracks = MessageField(SeqPeekTrackRecord, 1, repeated=True) - protein = MessageField(InterproItem, 2, required=False) - regions = MessageField(SeqPeekRegionRecord, 3, repeated=True) - - -class SeqPeekRemovedRow(Message): - name = StringField(1, required=True) - num = IntegerField(2, required=True) - - -class SeqPeekViewRecord(Message): - cohort_id_list = StringField(1, repeated=True) - hugo_symbol = StringField(2, required=True) - plot_data = MessageField(SeqPeekViewPlotDataRecord, 3, required=True) - removed_row_statistics = MessageField(SeqPeekRemovedRow, 4, repeated=True) - - -def create_interpro_record(interpro_literal): - match_data = [] - for match in interpro_literal['matches']: - - match_location_data = [] - for location in match['locations']: - match_location_data.append(InterproMatchLocation( - score=int(location['score']), - start=int(location['start']), - end=int(location['end']) - )) - - match_data.append(InterproMatch( - status=str(match['status']), - name=str(match['name']), - evd=str(match['evd']), - locations=match_location_data, - dbname=str(match['dbname']), - id=str(match['id']), - )) - - interpro_json = InterproJson( - matches=match_data, - uniprot_id=str(interpro_literal['uniprot_id']), - length=int(interpro_literal['length']), - name=str(interpro_literal['name']) - ) - - return InterproItem( - interpro_json=interpro_json - ) - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekViewDataAccessAPI(remote.Service): - def build_gnab_feature_id(self, gene_label): - return "GNAB:{gene_label}:variant_classification".format(gene_label=gene_label) - - def create_response(self, seqpeek_view_data): - plot_data = seqpeek_view_data['plot_data'] - tracks = [] - for track in plot_data['tracks']: - mutations = maf_array_to_record(track['mutations']) - tracks.append(SeqPeekTrackRecord(mutations=mutations, label=track['label'], type=track["type"], - row_id=track['render_info']['row_id'], - number_of_samples=track['statistics']['samples']['numberOf'], - mutated_positions=track['statistics']['samples']['mutated_positions'], - cohort_size=track['statistics']['cohort_size'])) - - region_records = [] - for region in plot_data['regions']: - region_records.append(SeqPeekRegionRecord(**region)) - - protein = create_interpro_record(plot_data['protein']) - plot_data_record = SeqPeekViewPlotDataRecord(tracks=tracks, protein=protein, regions=region_records) - - removed_row_statistics = [] - for item in seqpeek_view_data['removed_row_statistics']: - removed_row_statistics.append(SeqPeekRemovedRow(**item)) - - return SeqPeekViewRecord(plot_data=plot_data_record, hugo_symbol=seqpeek_view_data['hugo_symbol'], - cohort_id_list=seqpeek_view_data['cohort_id_list'], - removed_row_statistics=removed_row_statistics) - - @endpoints_method(SeqPeekViewDataRequest, SeqPeekViewRecord, - path='view_data', http_method='GET', name='seqpeek.getViewData') - def seqpeek_view_data(self, request): - try: - hugo_symbol = request.hugo_symbol - cohort_id_array = request.cohort_id - - gnab_feature_id = self.build_gnab_feature_id(hugo_symbol) - logging.debug("GNAB feature ID for SeqPeke: {0}".format(gnab_feature_id)) - - # Get the project IDs these cohorts' samples come from - cohort_vals = tuple(int(i) for i in cohort_id_array) - cohort_params = ('%s,' * len(cohort_id_array))[:-1] - - db = sql_connection() - cursor = db.cursor() - - isbcgc_projects = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN (%s);" % cohort_params, - cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_project_ids = [] - unconfirmed_project_ids = [] - - for row in cursor.fetchall(): - if row[0] in isbcgc_projects: - if row[0] not in confirmed_project_ids: - confirmed_project_ids.append(row[0]) - elif row[0] not in unconfirmed_project_ids: - unconfirmed_project_ids.append(row[0]) - - if len(unconfirmed_project_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_project_ids) - - for proj in projects: - if proj.get_my_root_and_depth()['root'] in isbcgc_projects: - confirmed_project_ids.append(proj.id) - - async_params = [ProviderClassQueryDescription(SeqPeekDataProvider, gnab_feature_id, cohort_id_array, confirmed_project_ids)] - maf_data_result = get_feature_vectors_tcga_only(async_params, skip_formatting_for_plot=True) - - maf_data_vector = maf_data_result[gnab_feature_id]['data'] - - if len(maf_data_vector) > 0: - # Since the gene (hugo_symbol) parameter is part of the GNAB feature ID, - # it will be sanity-checked in the SeqPeekMAFDataAccess instance. - seqpeek_data = SeqPeekMAFDataFormatter().format_maf_vector_for_view(maf_data_vector, cohort_id_array) - - seqpeek_maf_vector = seqpeek_data.maf_vector - seqpeek_cohort_info = seqpeek_data.cohort_info - removed_row_statistics_dict = seqpeek_data.removed_row_statistics - - seqpeek_view_data = SeqPeekViewDataBuilder().build_view_data(hugo_symbol, - seqpeek_maf_vector, - seqpeek_cohort_info, - cohort_id_array, - removed_row_statistics_dict) - - response = self.create_response(seqpeek_view_data) - return response - else: - # No data found - return SeqPeekViewRecord(plot_data=SeqPeekViewPlotDataRecord(tracks=[], protein=None, regions=[]), - hugo_symbol=hugo_symbol, cohort_id_list=[str(i) for i in cohort_id_array], - removed_row_statistics=[]) - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - diff --git a/api/single_feature_access.py b/api/single_feature_access.py deleted file mode 100644 index e270610f..00000000 --- a/api/single_feature_access.py +++ /dev/null @@ -1,131 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import time - -from api.data_access import FeatureDataEndpointsAPI, PlotDataCohortInfo - -from endpoints import method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class DataPointList(Message): - type = EnumField(ValueType, 1, required=True) - items = MessageField(DataPoint, 2, repeated=True) - label = StringField(3, required=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class SingleFeatureDataAccess(remote.Service): - def get_feature_vector(self, feature_id, cohort_id_array): - start = time.time() - - async_params = [(feature_id, cohort_id_array)] - async_result = get_feature_vectors_with_user_data(async_params) - - feature_type, feature_vec = async_result[feature_id]['type'], async_result[feature_id]['data'] - - end = time.time() - time_elapsed = end-start - logging.info('Time elapsed: ' + str(time_elapsed)) - - vms = VectorMergeSupport('NA', 'sample_id', [feature_id]) - vms.add_dict_array(feature_vec, feature_id, 'value') - - merged = vms.get_merged_dict() - - return feature_type, merged - - def annotate_vector_with_cohorts(self, cohort_id_array, merged): - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - value_bundle['cohort'] = cohort_set - - def get_cohort_information(self, cohort_id_array): - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - - return cohort_info_array - - def create_response(self, feature_id, vector_type, vector, cohort_info_array): - data_points = [] - for item in vector: - data_points.append(DataPoint( - sample_id=item['sample_id'], - value=item[feature_id], - cohort=item['cohort'] - )) - - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return DataPointList(type=vector_type, items=data_points, label=feature_id, - cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, DataPointList, - path='feature_data', http_method='GET', name='feature_access.getFeatureData') - def data_access_by_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - vector_type, vector = self.get_feature_vector(feature_id, cohort_id_array) - - self.annotate_vector_with_cohorts(cohort_id_array, vector) - - cohort_info = self.get_cohort_information(cohort_id_array) - response = self.create_response(feature_id, vector_type, vector, cohort_info) - return response - except FeatureNotFoundException as fnf: - logging.error("Invalid internal feature ID '{}'".format(str(fnf))) - raise NotFoundException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/README.md b/api_3/README.md deleted file mode 100755 index d1213d73..00000000 --- a/api_3/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# ISB-CGC-API/api_3 -Google Endpoints API Used to Drive CGC Webapp. -The endpoints defined here are intended for the switch over to the GDC -metadata schema and to migrate to the Cloud Endpoints Frameworks for App Engine -The original `api` directory was copied and named `api_3` and all the endpoint versions -set to '3' diff --git a/api_3/__init__.py b/api_3/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/api_helpers.py b/api_3/api_helpers.py deleted file mode 100755 index 6ad2483d..00000000 --- a/api_3/api_helpers.py +++ /dev/null @@ -1,483 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import sys -import os -import MySQLdb -import httplib2 -from oauth2client.client import GoogleCredentials, AccessTokenCredentials -from django.conf import settings -from googleapiclient.discovery import build - -from cohorts.metadata_helpers import get_sql_connection - -debug = settings.DEBUG - -WHITELIST_RE = ur'([^\\\_\|\"\+~@:#\$%\^&\*=\-\.,\(\)0-9a-zA-Z\s\xc7\xfc\xe9\xe2\xe4\xe0\xe5\xe7\xea\xeb\xe8\xef\xee\xed\xec\xc4\xc5\xc9\xe6\xc6\xf4\xf6\xf2\xfb\xf9\xd6\xdc\xe1\xf3\xfa\xf1\xd1\xc0\xc1\xc2\xc3\xc8\xca\xcb\xcc\xcd\xce\xcf\xd0\xd2\xd3\xd4\xd5\xd8\xd9\xda\xdb\xdd\xdf\xe3\xf0\xf5\xf8\xfd\xfe\xff])' - -MOLECULAR_CATEGORIES = { - 'nonsilent': [ - 'Missense_Mutation', - 'Nonsense_Mutation', - 'Nonstop_Mutation', - 'Frame_Shift_Del', - 'Frame_Shift_Ins', - 'De_novo_Start_OutOfFrame', - 'In_Frame_Del', - 'In_Frame_Ins', - 'Start_Codon_SNP', - 'Start_Codon_Del', - ] -} - -# Database connection -def sql_connection(): - return get_sql_connection() - -def sql_bmi_by_ranges(value): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - if 'None' in value: - result += 'BMI is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == 'underweight': - result += ' (BMI < 18.5)' - elif str(val) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(val) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(val) == 'obese': - result += ' (BMI > 29.9)' - - else: - # value is a single range - if str(value) == 'underweight': - result += ' (BMI < 18.5)' - elif str(value) == 'normal weight': - result += ' (BMI >= 18.5 and BMI <= 24.9)' - elif str(value) == 'overweight': - result += ' (BMI > 24.9 and BMI <= 29.9)' - elif str(value) == 'obese': - result += ' (BMI > 29.9)' - - return result - - -def sql_age_by_ranges(value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - #value is a list of ranges - first = True - if 'None' in value: - result += 'age_at_diagnosis is null or ' - value.remove('None') - for val in value: - if first: - result += '' - first = False - else: - result += ' or' - if str(val) == '10 to 39': - result += ' (age_at_diagnosis >= 10 and age_at_diagnosis < 40)' - elif str(val) == '40 to 49': - result += ' (age_at_diagnosis >= 40 and age_at_diagnosis < 50)' - elif str(val) == '50 to 59': - result += ' (age_at_diagnosis >= 50 and age_at_diagnosis < 60)' - elif str(val) == '60 to 69': - result += ' (age_at_diagnosis >= 60 and age_at_diagnosis < 70)' - elif str(val) == '70 to 79': - result += ' (age_at_diagnosis >= 70 and age_at_diagnosis < 80)' - elif str(val).lower() == 'over 80': - result += ' (age_at_diagnosis >= 80)' - else: - #value is a single range - if str(value) == '10 to 39': - result += ' (age_at_diagnosis >= 10 and age_at_diagnosis < 40)' - elif str(value) == '40 to 49': - result += ' (age_at_diagnosis >= 40 and age_at_diagnosis < 50)' - elif str(value) == '50 to 59': - result += ' (age_at_diagnosis >= 50 and age_at_diagnosis < 60)' - elif str(value) == '60 to 69': - result += ' (age_at_diagnosis >= 60 and age_at_diagnosis < 70)' - elif str(value) == '70 to 79': - result += ' (age_at_diagnosis >= 70 and age_at_diagnosis < 80)' - elif str(value).lower() == 'over 80': - result += ' (age_at_diagnosis >= 80)' - elif str(value) == 'None': - result += ' age_at_diagnosis is null' - - return result - -def gql_age_by_ranges(q, key, value): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - result = '' - if not isinstance(value, basestring): - # value is a list of ranges - first = True - for val in value: - if first: - first = False - else: - result += ' or' - if str(val) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(val) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(val) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(val) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(val) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(val).lower() == 'over80': - result += ' (%s >= 80)' % key - else: - # value is a single range - if str(value) == '10to39': - result += ' (%s >= 10 and %s < 40)' % (key, key) - elif str(value) == '40to49': - result += ' (%s >= 40 and %s < 50)' % (key, key) - elif str(value) == '50to59': - result += ' (%s >= 50 and %s < 60)' % (key, key) - elif str(value) == '60to69': - result += ' (%s >= 60 and %s < 70)' % (key, key) - elif str(value) == '70to79': - result += ' (%s >= 70 and %s < 80)' % (key, key) - elif str(value).lower() == 'over80': - result += ' (%s >= 80)' % key - return result - - -def normalize_bmi(bmis): - if debug: print >> sys.stderr, 'Called ' + sys._getframe().f_code.co_name - bmi_list = {'underweight': 0, 'normal weight': 0, 'overweight': 0, 'obese': 0, 'None': 0} - for bmi, count in bmis.items(): - if type(bmi) != dict: - if bmi and bmi != 'None': - fl_bmi = float(bmi) - if fl_bmi < 18.5: - bmi_list['underweight'] += int(count) - elif 18.5 <= fl_bmi <= 24.9: - bmi_list['normal weight'] += int(count) - elif 25 <= fl_bmi <= 29.9: - bmi_list['overweight'] += int(count) - elif fl_bmi >= 30: - bmi_list['obese'] += int(count) - else: - bmi_list['None'] += int(count) - - return bmi_list - - -def normalize_ages(ages): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age, count in ages.items(): - if type(age) != dict: - if age and age != 'None': - int_age = float(age) - if int_age < 40: - new_age_list['10 to 39'] += int(count) - elif int_age < 50: - new_age_list['40 to 49'] += int(count) - elif int_age < 60: - new_age_list['50 to 59'] += int(count) - elif int_age < 70: - new_age_list['60 to 69'] += int(count) - elif int_age < 80: - new_age_list['70 to 79'] += int(count) - else: - new_age_list['Over 80'] += int(count) - else: - new_age_list['None'] += int(count) - else: - print age - - return new_age_list - -def applyFilter(field, dict): -# this one gets called a lot... -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - query_dict = dict.copy() - if field in dict: - query_dict.pop(field, None) - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - else: - where_clause = None - else: - where_clause = build_where_clause(dict) - - return where_clause - - -def build_where_clause(filters, alt_key_map=False): -# this one gets called a lot -# if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - first = True - query_str = '' - big_query_str = '' # todo: make this work for non-string values -- use {}.format - value_tuple = () - key_order = [] - keyType = None - gene = None - - grouped_filters = None - - for key, value in filters.items(): - if isinstance(value, dict) and 'values' in value: - value = value['values'] - - if isinstance(value, list) and len(value) == 1: - value = value[0] - # Check if we need to map to a different column name for a given key - if alt_key_map and key in alt_key_map: - key = alt_key_map[key] - - # Multitable where's will come in with : in the name. Only grab the column piece for now - # TODO: Shouldn't throw away the entire key - elif ':' in key: - keyType = key.split(':')[0] - if keyType == 'MUT': - gene = key.split(':')[1] - key = key.split(':')[-1] - - # Multitable filter lists don't come in as string as they can contain arbitrary text in values - elif isinstance(value, basestring): - # If it's a list of values, split it into an array - if ',' in value: - value = value.split(',') - - key_order.append(key) - - # Bucket the grouped filter types (currently just certain has_ values, could be more) - if 'has_' in key and not key == 'has_Illumina_DNASeq' and not key == 'has_SNP6' and not key == 'has_RPPA': - if grouped_filters is None: - grouped_filters = {} - - if key == 'has_27k' or key == 'has_450k': - if 'DNA_methylation' not in grouped_filters: - grouped_filters['DNA_methylation'] = [] - grouped_filters['DNA_methylation'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_HiSeq_miRnaSeq' or key == 'has_GA_miRNASeq': - if 'miRNA_sequencing' not in grouped_filters: - grouped_filters['miRNA_sequencing'] = [] - grouped_filters['miRNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - elif key == 'has_UNC_HiSeq_RNASeq' or key == 'has_UNC_GA_RNASeq' or key == 'has_BCGSC_HiSeq_RNASeq' or key == 'has_BCGSC_GA_RNASeq': - if 'RNA_sequencing' not in grouped_filters: - grouped_filters['RNA_sequencing'] = [] - grouped_filters['RNA_sequencing'].append({'filter': str(key), 'value': str(value)}) - # BQ-only format - elif keyType == 'MUT': - # If it's first in the list, don't append an "and" - params = {} - value_tuple += (params,) - - if first: - first = False - else: - big_query_str += ' AND' - - big_query_str += " %s = '{hugo_symbol}' AND " % 'Hugo_Symbol' - params['gene'] = gene - - if(key == 'category'): - if value == 'any': - big_query_str += '%s IS NOT NULL' % 'Variant_Classification' - params['var_class'] = '' - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = MOLECULAR_CATEGORIES[value] - else: - big_query_str += '%s IN ({var_class})' % 'Variant_Classification' - values = value - - if value != 'any': - if isinstance(values, list): - j = 0 - for vclass in values: - if j == 0: - params['var_class'] = "'%s'" % vclass.replace("'", "\\'") - j = 1 - else: - params['var_class'] += ",'%s'" % vclass.replace("'", "\\'") - else: - params['var_class'] = "'%s'" % values.replace("'", "\\'") - - else: - # If it's first in the list, don't append an "and" - if first: - first = False - else: - query_str += ' and' - big_query_str += ' and' - - # If it's age ranges, give it special treament due to normalizations - if key == 'age_at_diagnosis': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_age_by_ranges(value) + ') ' - # If it's age ranges, give it special treament due to normalizations - elif key == 'BMI': - if value == 'None': - query_str += ' %s IS NULL' % key - else: - query_str += ' (' + sql_bmi_by_ranges(value) + ') ' - # If it's a list of items for this key, create an or subclause - elif isinstance(value, list): - has_null = False - if 'None' in value: - has_null = True - query_str += ' (%s is null or' % key - big_query_str += ' (%s is null or' % key - value.remove('None') - query_str += ' %s in (' % key - big_query_str += ' %s in (' % key - i = 0 - for val in value: - value_tuple += (val.strip(),) if type(val) is unicode else (val,) - if i == 0: - query_str += '%s' - big_query_str += '"' + str(val) + '"' - i += 1 - else: - query_str += ',%s' - big_query_str += ',' + '"' + str(val) + '"' - query_str += ')' - big_query_str += ')' - if has_null: - query_str += ')' - big_query_str += ')' - - # If it's looking for None values - elif value == 'None': - query_str += ' %s is null' % key - big_query_str += ' %s is null' % key - - # For the general case - else: - if key == 'fl_archive_name': - big_query_str += ' %s like' % key - big_query_str += ' "%' + value + '%"' - elif key == 'fl_data_level': - big_query_str += ' %s=%s' % (key, value) - elif type(value) == bool: - big_query_str += ' %s=%r' % (key, value) - else: - query_str += ' %s=' % key - big_query_str += ' %s=' % key - query_str += '%s' - big_query_str += '"%s"' % value - value_tuple += (value.strip(),) if type(value) is unicode else (value,) - - # Handle our data buckets - if grouped_filters: - for bucket in grouped_filters: - if not query_str == '': - query_str += ' and ' - big_query_str += ' and ' - - query_str += '( ' - big_query_str += '( ' - - first = True - for filter in grouped_filters[bucket]: - if first: - first = False - else: - query_str += ' or ' - big_query_str += ' or ' - - query_str += ' %s=' % filter['filter'] - big_query_str += ' %s=' % filter['filter'] - query_str += '%s' - big_query_str += '"%s"' % filter['value'] - value_tuple += (filter['value'].strip(),) if type(filter['value']) is unicode else (filter['value'],) - - query_str += ' )' - big_query_str += ' )' - - return {'query_str': query_str, 'value_tuple': value_tuple, 'key_order': key_order, 'big_query_str': big_query_str} - - -def possible_future_authorization_function(): - # will put a decorator on this to ensure user has correct authorization before running - # such as if they are dbgap authorized - from oauth2client.client import flow_from_clientsecrets - from oauth2client.file import Storage - from oauth2client import tools - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - flow = flow_from_clientsecrets(settings.CLIENT_SECRETS, scope='https://www.googleapis.com/auth/bigquery') - ## in future, make storage file temporary somehow? - storage = Storage('bigquery_credentials.dat') - credentials = storage.get() - - if credentials is None or credentials.invalid: - credentials = tools.run_flow(flow, storage, tools.argparser.parse_args([])) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - return service - - -def authorize_credentials_with_Google(): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - # credentials = GoogleCredentials.get_application_default().create_scoped(SCOPES) - credentials = GoogleCredentials.from_stream(settings.GOOGLE_APPLICATION_CREDENTIALS).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - if debug: print >> sys.stderr,' big query authorization '+sys._getframe().f_code.co_name - return service - -# TODO refactor to remove duplicate code -def authorize_credentials_with_google_from_file(credentials_path): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - # documentation: https://developers.google.com/accounts/docs/application-default-credentials - SCOPES = ['https://www.googleapis.com/auth/bigquery'] - credentials = GoogleCredentials.from_stream(credentials_path).create_scoped(SCOPES) - http = httplib2.Http() - http = credentials.authorize(http) - service = build('bigquery', 'v2', http=http) - - return service - - -def get_user_email_from_token(access_token): - if debug: print >> sys.stderr,'Called '+sys._getframe().f_code.co_name - user_email = None - credentials = AccessTokenCredentials(access_token, 'test-user') - http = credentials.authorize(httplib2.Http()) - user_info_service = build('oauth2', 'v2', http=http) - user_info = user_info_service.userinfo().get().execute() - if 'email' in user_info: - user_email = user_info['email'] - return user_email diff --git a/api_3/cloudstoragefilepaths_helper.py b/api_3/cloudstoragefilepaths_helper.py deleted file mode 100755 index 0b52d6f5..00000000 --- a/api_3/cloudstoragefilepaths_helper.py +++ /dev/null @@ -1,212 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -from collections import OrderedDict -import logging -import MySQLdb - -try: - import endpoints -except Exception as e: - print 'couldn\'t import google endpoints, using mock for testing: %s' % (e) -import django - -from django.conf import settings -from django.core.signals import request_finished -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from sharing.models import User -# from django.contrib.auth.models import User as Django_User -try: - from protorpc import remote, messages -except Exception as e: - print 'couldn\'t import google protorpc, using mock for testing: %s' % (e) - -from api_3.api_helpers import sql_connection -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - -class GCSFilePathList(messages.Message): - cloud_storage_file_paths = messages.StringField(1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -class CloudStorageFilePathsAPI(remote.Service): - def setup_param_map(self, request, param_names): - param_map = OrderedDict() - for param_name in param_names: - param_map[param_name] = request.get_assigned_value(param_name) - return param_map - - - def get_genomic_builds(self, param_map, program): - if 'CCLE' == program: - builds = ['HG19'] - else: - builds = ['HG19', 'HG38'] - - if 'genomic_build' in param_map and param_map['genomic_build']: - if 'HG19' == param_map['genomic_build'].upper(): - builds = ['HG19'] - elif 'HG38' == param_map['genomic_build'].upper(): - builds = ['HG38'] - else: - msg = 'Unknown genomic build: {}. Acceptable genomic builds are HG19 and HG38.'.format(param_map['genomic_build']) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for cohort. {}".format(msg)) - return builds - - def get_cloud_storage_file_paths(self, param_map, program): - """ - Uses the param_map to pass to the query builder then executes the query to obtain the cloud - storage file paths. if cohort_id is on the parameter list, verifies that the calling user has - permission for the cohort. - """ - cursor = None - db = None - - query_str, query_tuple = self.build_query(param_map, program) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - cursor_rows = cursor.fetchall() - cloud_storage_path_list = [row['file_name_key'] for row in cursor_rows] - return GCSFilePathList(cloud_storage_file_paths=cloud_storage_path_list, count=len(cloud_storage_path_list)) - except (IndexError, TypeError), e: - logger.warn('problem getting file paths: {}\nSQL: {}\nparams: {}'.format(e, query_str, query_tuple)) - raise endpoints.NotFoundException("File paths for {} {} not found." \ - .format('cohort', param_map['cohort_id'] if 'cohort_id' in param_map else 'sample', param_map['sample_barcode'])) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving file paths. {}\nSQL: {}\nparams: {}".format(e, query_str, query_tuple)) - raise endpoints.BadRequestException("Error retrieving file paths. {}".format(e)) - finally: - if cursor: - cursor.close() - if db and db.open: - db.close() - -class SamplesCloudStorageFilePathsHelper(CloudStorageFilePathsAPI): - - GET_RESOURCE = endpoints.ResourceContainer( - sample_barcode=messages.StringField(1, required=True), - data_type=messages.StringField(2), - data_category=messages.StringField(3), - experimental_strategy=messages.StringField(4), - data_format=messages.StringField(5), - platform=messages.StringField(6), - genomic_build=messages.StringField(7), - analysis_workflow_type=messages.StringField(8) - ) - - def build_query(self, param_map, program): - builds = self.get_genomic_builds(param_map, program) - final_query_str = '' - query_tuple = [] - for build in builds: - query_str = 'SELECT md.file_name_key, md.access ' \ - 'FROM {}_metadata_data_{}_r14 md '.format(program, build) - - query_str += 'WHERE sample_barcode=%s ' - query_tuple += [param_map['sample_barcode']] - query_str += 'AND file_name_key != "" AND file_name_key is not null ' - for field, value in param_map.iteritems(): - if field not in ['limit', 'cohort_id', 'sample_barcode', 'genomic_build'] and value: - query_str += ' and md.{}=%s '.format(field) - query_tuple += [value] - query_str += ' GROUP BY md.file_name_key, md.access ' - if 0 < len(final_query_str): - final_query_str += ' UNION ' - final_query_str += query_str - - if 'limit' in param_map and param_map['limit']: - final_query_str += ' LIMIT %s' - query_tuple += [param_map['limit']] - else: - final_query_str += ' LIMIT 10000' - return final_query_str, query_tuple - - def cloud_storage_file_paths(self, request, program): - param_map = self.setup_param_map(request, [ - 'data_type', - 'data_category', - 'experimental_strategy', - 'data_format', - 'platform', - 'genomic_build', - 'analysis_workflow_type', - 'sample_barcode' - ] - ) - return self.get_cloud_storage_file_paths(param_map, program) - -class CohortsCloudStorageFilePathsHelper(CloudStorageFilePathsAPI): - - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - limit=messages.IntegerField(2), - data_type=messages.StringField(3), - data_category=messages.StringField(4), - experimental_strategy=messages.StringField(5), - data_format=messages.StringField(6), - platform=messages.StringField(7), - genomic_build=messages.StringField(8), - analysis_workflow_type=messages.StringField(9) - ) - - def validate_user(self, cohort_id): - user_email = None - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(BASE_URL)) - - django.setup() - try: - user_id = User.objects.get(email=user_email).id - Django_Cohort.objects.get(id=cohort_id) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}.".format(user_email, cohort_id) - raise endpoints.UnauthorizedException(err_msg) - finally: - request_finished.send(self) - - def cloud_storage_file_paths(self, request): - param_map = self.setup_param_map(request, [ - 'data_type', - 'data_category', - 'experimental_strategy', - 'data_format', - 'platform', - 'genomic_build', - 'analysis_workflow_type', - 'limit', - 'cohort_id' - ] - ) - self.validate_user(param_map['cohort_id']) - return self.get_cloud_storage_file_paths(param_map, None) diff --git a/api_3/cohort_create_preview_helper.py b/api_3/cohort_create_preview_helper.py deleted file mode 100755 index 227656d3..00000000 --- a/api_3/cohort_create_preview_helper.py +++ /dev/null @@ -1,396 +0,0 @@ -''' -Created on Mar 30, 2017 - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import django -import re -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages -from datetime import datetime -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from api_3.cohort_endpoint_helpers import are_there_bad_keys, are_there_no_acceptable_keys, construct_parameter_error_message - -from api_3.api_helpers import sql_connection, WHITELIST_RE -from bq_data_access.cohort_bigquery import BigQueryCohortSupport -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms, Samples, Filters -from projects.models import Program, Project - -logger = logging.getLogger(__name__) - -class CohortsCreatePreviewAPI(remote.Service): - def build_query_dictionaries(self, request): - """ - Builds the query dictionaries for create and preview cohort endpoints. - Returns query_dict, gte_query_dict, lte_query_dict. - """ - query_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - and k.name is not 'name' - and not k.name.endswith('_gte') - and not k.name.endswith('_lte') - } - - gte_query_dict = { - k.name.replace('_gte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_gte') - } - - lte_query_dict = { - k.name.replace('_lte', ''): request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) and k.name.endswith('_lte') - } - - return query_dict, gte_query_dict, lte_query_dict - - def build_query(self, program, table, query_dict, gte_query_dict, lte_query_dict): - """ - Builds the queries that selects the case and sample barcodes - that meet the criteria specified in the request body. - Returns case query string, sample query string, value tuple. - """ - if (0 == len(query_dict) and 0 == len(gte_query_dict) and 0 == len(lte_query_dict)): - return None, None - - if 'Clinical' == table: - query_str = 'SELECT sample_barcode, c.case_barcode, c.project_short_name ' \ - 'FROM {0}_metadata_clinical c join {0}_metadata_biospecimen b on c.case_barcode = b.case_barcode ' \ - 'WHERE '.format(program) - else: - query_str = 'SELECT sample_barcode, case_barcode, project_short_name ' \ - 'FROM {0}_metadata_{1} ' \ - 'WHERE '.format(program, table[:1].lower() + table[1:]) - value_tuple = () - - alias = 'c.' if 'Clinical' == table else '' - for key, value_list in query_dict.iteritems(): - query_str += ' AND ' if not query_str.endswith('WHERE ') else '' - if "None" in value_list: - value_list.remove("None") - query_str += ' ( {alias}{key} is null '.format(alias=alias, key=key) - if len(value_list) > 0: - query_str += ' OR {alias}{key} IN ({vals}) '.format( - alias=alias, key=key, vals=', '.join(['%s'] * len(value_list))) - query_str += ') ' - else: - query_str += ' {alias}{key} IN ({vals}) '.format(alias=alias, key=key, vals=', '.join(['%s'] * len(value_list))) - value_tuple += tuple(value_list) - - for key, value in gte_query_dict.iteritems(): - query_str += ' AND ' if not query_str.endswith('WHERE ') else '' - query_str += ' {}{} >=%s '.format(alias, key) - value_tuple += (value,) - - for key, value in lte_query_dict.iteritems(): - query_str += ' AND ' if not query_str.endswith('WHERE ') else '' - query_str += ' {}{} <=%s '.format(alias, key) - value_tuple += (value,) - - query_str += ' GROUP BY sample_barcode, {0}case_barcode, {0}project_short_name'.format(alias) - - return query_str, value_tuple - - def query(self, request): - try: - db = None - try: - db = sql_connection() - except Exception as e: - logger.exception(e) - raise endpoints.NotFoundException("Error connecting to database: {}".format(e)) - - fields = request.get_assigned_value('Common') - common_query_dict = {} - - if fields: - common_query_dict, _, _ = self.build_query_dictionaries(fields) - - # Our return values - ret_query_dict = {} - ret_lte_query_dict = {} - ret_gte_query_dict = {} - ret_rows = None - - for table in ('Clinical', 'Biospecimen', 'data_HG19_r14', 'data_HG38_r14'): - if 'CCLE' == self.program and 'data_HG38_r14' == table: - continue - - fields = request.get_assigned_value(table) - - if fields and (are_there_bad_keys(fields) or are_there_no_acceptable_keys(fields)): - err_msg = construct_parameter_error_message(request, True) - raise endpoints.BadRequestException(err_msg) - - query_dict = {} - gte_query_dict = {} - lte_query_dict = {} - - if fields: - query_dict, gte_query_dict, lte_query_dict = self.build_query_dictionaries(fields) - - logger.info("query_dict: {}".format(str(query_dict))) - logger.info("common_query_dict: {}".format(str(common_query_dict))) - - query_dict.update(common_query_dict) - - ret_query_dict.update(query_dict) - ret_lte_query_dict.update(lte_query_dict) - ret_gte_query_dict.update(gte_query_dict) - - query_str, value_tuple = self.build_query(self.program, table, query_dict, gte_query_dict, lte_query_dict) - - if not query_str: - continue - - cursor = None - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - rows = list(cursor.fetchall()) - if 0 == len(rows): - # if any query returns no rows, then, overall, no sample_barcode will match - return [], '', '', '' - - if ret_rows is None: - ret_rows = rows - else: - cur_samples = set() - for row in rows: - cur_samples.add(row['sample_barcode']) - not_in_sample = [] - for row in ret_rows: - if row['sample_barcode'] not in cur_samples: - not_in_sample += [row] - for row in not_in_sample: - ret_rows.remove(row) - except (IndexError, TypeError) as e: - logger.exception(e) - raise endpoints.NotFoundException("Error retrieving samples and cases: {}\n{} {}".format(e, query_str, value_tuple)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tsample query: {} {}'.format(e, query_str, value_tuple) - logger.exception(msg) - raise endpoints.BadRequestException("Error creating/previewing cohort. {}".format(e)) - finally: - if cursor: - cursor.close() - finally: - if db and db.open: - db.close() - request_finished.send(self) - - return ret_rows, ret_query_dict, ret_lte_query_dict, ret_gte_query_dict - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - -class CreatedCohort(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - filters = messages.MessageField(FilterDetails, 4, repeated=True) - case_count = messages.IntegerField(5, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(6, variant=messages.Variant.INT32) - -class CohortsCreateHelper(CohortsCreatePreviewAPI): - BASE_URL = settings.BASE_URL - - def get_django_program(self, program_name): - try: - # get the ISB superuser - isb_superuser = Django_User.objects.get(username='isb', is_staff=True, is_superuser=True, is_active=True) - # get the program - program = Program.objects.get(name=program_name, is_public=True, active=True, owner=isb_superuser) - finally: - request_finished.send(self) - return program - - def get_django_project(self, project_short_name): - try: - # get the ISB superuser - isb_superuser = Django_User.objects.get(username='isb', is_staff=True, is_superuser=True, is_active=True) - # get the program - program = self.get_django_program(project_short_name.split('-')[0]) - # get the project - project = Project.objects.get(name=project_short_name[project_short_name.find('-') + 1:], active=True, owner=isb_superuser, program=program) - finally: - request_finished.send(self) - return project - - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - user = endpoints.get_current_user() - user_email = user.email() if user else None - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.".format(self.BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - finally: - request_finished.send(self) - - # get the sample barcode information for use in creating the sample list for the cohort - rows, query_dict, lte_query_dict, gte_query_dict = self.query(request) - project2django = {} - if rows: - for row in rows: - if row['project_short_name'] not in project2django: - project2django[row['project_short_name']] = self.get_django_project(row['project_short_name']) - else: - raise endpoints.BadRequestException("No samples meet the specified parameters.") - - sample_barcodes = [{'sample_barcode': row['sample_barcode'], 'case_barcode': row['case_barcode'], 'project': project2django[row['project_short_name']]} for row in rows] - cohort_name = request.get_assigned_value('name') - - # Validate the cohort name against a whitelist - whitelist = re.compile(WHITELIST_RE, re.UNICODE) - match = whitelist.search(unicode(cohort_name)) - if match: - # XSS risk, log and fail this cohort save - match = whitelist.findall(unicode(cohort_name)) - logger.error( - '[ERROR] While saving a cohort, saw a malformed name: ' + cohort_name + ', characters: ' + match.__str__()) - raise endpoints.BadRequestException( - "Your cohort's name contains invalid characters (" + match.__str__() + "); please choose another name.") - - if len(sample_barcodes) == 0: - raise endpoints.BadRequestException( - "The cohort could not be saved because no samples meet the specified parameters.") - - # todo: maybe create all objects first, then save them all at the end? - # 1. create new cohorts_cohort with name, active=True, last_date_saved=now - try: - created_cohort = Django_Cohort.objects.create(name=cohort_name, active=True, last_date_saved=datetime.utcnow()) - created_cohort.save() - finally: - request_finished.send(self) - - # 2. insert samples into cohort_samples - try: - sample_list = [Samples(cohort=created_cohort, sample_barcode=sample['sample_barcode'], case_barcode=sample['case_barcode'], project=sample['project']) for sample in sample_barcodes] - Samples.objects.bulk_create(sample_list) - finally: - request_finished.send(self) - - # 3. Set permission for user to be owner - try: - perm = Cohort_Perms(cohort=created_cohort, user=django_user, perm=Cohort_Perms.OWNER) - perm.save() - finally: - request_finished.send(self) - - # 4. Create filters applied - filter_data = [] - django_program = self.get_django_program(self.program) - try: - # special case sample barcode since the list can be ALL the sample barcodes in the program - edit_barcodes = set() - for key, value_list in query_dict.items(): - if 'sample_barcode' == key: - edit_barcodes |= set(value_list) - continue - for val in value_list: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val, program=django_program).save() - if 0 < len(edit_barcodes): - if len(edit_barcodes) < 6: - val = 'barcodes: {}'.format(', '.join(sorted(list(edit_barcodes)))) - else: - val = '{} barcodes beginning with {}'.format(len(edit_barcodes), ', '.join(sorted(list(edit_barcodes))[:5])) - filter_data.append(FilterDetails(name='sample_barcode', value=val)) - Filters.objects.create(resulting_cohort=created_cohort, name='sample_barcode', value=val, program=django_program).save() - - for key, val in [(k + '_lte', v) for k, v in lte_query_dict.items()] + [(k + '_gte', v) for k, v in gte_query_dict.items()]: - filter_data.append(FilterDetails(name=key, value=str(val))) - Filters.objects.create(resulting_cohort=created_cohort, name=key, value=val, program=django_program).save() - finally: - request_finished.send(self) - - # 5. Store cohort to BigQuery - project_id = settings.BQ_PROJECT_ID - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - bcs = BigQueryCohortSupport(project_id, cohort_settings.dataset_id, cohort_settings.table_id) - batch = 5000 - start = 0 - while start < len(sample_barcodes): - bcs.add_cohort_to_bq(created_cohort.id, sample_barcodes[start:start + batch]) - start += batch - - request_finished.send(self) - - return CreatedCohort( - id=str(created_cohort.id), - name=cohort_name, - last_date_saved=str(datetime.utcnow()), - filters=filter_data, - case_count=created_cohort.case_size(), - sample_count=len(sample_barcodes) - ) - -class CohortsPreviewHelper(CohortsCreatePreviewAPI): - class CohortCasesSamplesList(messages.Message): - cases = messages.StringField(1, repeated=True) - case_count = messages.IntegerField(2, variant=messages.Variant.INT32) - samples = messages.StringField(3, repeated=True) - sample_count = messages.IntegerField(4, variant=messages.Variant.INT32) - - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - rows, _, _, _ = self.query(request) - - case_barcodes = set() - sample_barcodes = [] - - if rows: - for row in rows: - case_barcodes.add(row['case_barcode']) - sample_barcodes.append(row['sample_barcode']) - case_barcodes = list(case_barcodes) - - if len(sample_barcodes) == 0: - raise endpoints.BadRequestException("No samples meet the specified parameters.") - - return self.CohortCasesSamplesList(cases=case_barcodes, - case_count=len(case_barcodes), - samples=sample_barcodes, - sample_count=len(sample_barcodes)) diff --git a/api_3/cohort_endpoint_helpers.py b/api_3/cohort_endpoint_helpers.py deleted file mode 100755 index 2406d732..00000000 --- a/api_3/cohort_endpoint_helpers.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -from django.conf import settings -from protorpc import messages -import logging - -logger = logging.getLogger(__name__) - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -BUILTIN_ENDPOINTS_PARAMETERS = [ - 'alt', - 'fields', - 'enum', - 'enumDescriptions', - 'key', - 'oauth_token', - 'prettyPrint', - 'quotaUser', - 'userIp' -] - - -def are_there_bad_keys(request): - ''' - Checks for unrecognized fields in an endpoint request - :param request: the request object from the endpoint - :return: boolean indicating True if bad (unrecognized) fields are present in the request - ''' - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - return unrecognized_param_dict != {} - - -def are_there_no_acceptable_keys(request): - """ - Checks for a lack of recognized fields in an endpoints request. Used in save_cohort and preview_cohort endpoints. - :param request: the request object from the endpoint - :return: boolean indicating True if there are no recognized fields in the request. - """ - param_dict = { - k.name: request.get_assigned_value(k.name) - for k in request.all_fields() - if request.get_assigned_value(k.name) - } - return param_dict == {} - - -def construct_parameter_error_message(request, filter_required): - err_msg = '' - sorted_acceptable_keys = sorted([k.name for k in request.all_fields()], key=lambda s: s.lower()) - unrecognized_param_dict = { - k: request.get_unrecognized_field_info(k)[0] - for k in request.all_unrecognized_fields() - if k not in BUILTIN_ENDPOINTS_PARAMETERS - } - if unrecognized_param_dict: - bad_key_str = "'" + "', '".join(unrecognized_param_dict.keys()) + "'" - err_msg += "The following filters were not recognized: {}. ".format(bad_key_str) - if filter_required: - err_msg += "You must specify at least one of the following " \ - "case-sensitive filters: {}".format(sorted_acceptable_keys) - else: - err_msg += "Acceptable filters are: {}".format(sorted_acceptable_keys) - - return err_msg - - -class FilterDetails(messages.Message): - name = messages.StringField(1) - value = messages.StringField(2) - - -def build_constructor_dict_for_message(message_class, row): - """ - Takes an instance of a message class and a dictionary of values from a database query - and first validates the values in the dictionary against the message class fields - and then returns a dictionary of all the validated key-value pairs in the database query. - This will only work if the headers in the database query have the same name as the names of - fields in the message class. - """ - constructor_dict = {} - metadata_item_dict = {field.name: field for field in message_class.all_fields()} - for name, field in metadata_item_dict.iteritems(): - if row.get(name) is not None: - try: - field.validate(row[name]) - constructor_dict[name] = row[name] - except messages.ValidationError, e: - constructor_dict[name] = None - logger.warn('{name}: {value} was not validated while constructing kwargs for {message_class}. Error: {e}' - .format(name=name, value=str(row[name]), message_class=str(message_class), e=e)) - else: - constructor_dict[name] = None - - return constructor_dict - - -def build_constructor_dicts_for_message(message_class, rows): - """ - Takes an instance of a message class and a dictionary of values from a database query - and first validates the values in the dictionary against the message class fields - and then returns a dictionary of all the validated key-value pairs in the database query. - This will only work if the headers in the database query have the same name as the names of - fields in the message class. - """ - constructor_dicts = [] - metadata_item_dict = {field.name: field for field in message_class.all_fields()} - - for row in rows: - constructor_dict = {} - for name, field in metadata_item_dict.iteritems(): - if row.get(name) is not None: - try: - field.validate(row[name]) - constructor_dict[name] = row[name] - except messages.ValidationError, e: - constructor_dict[name] = None - logger.warn('{name}: {value} was not validated while constructing kwargs for {message_class}. Error: {e}' - .format(name=name, value=str(row[name]), message_class=str(message_class), e=e)) - else: - constructor_dict[name] = None - - if len(constructor_dict): - constructor_dicts.append(constructor_dict) - - return constructor_dicts \ No newline at end of file diff --git a/api_3/data_access.py b/api_3/data_access.py deleted file mode 100755 index b9abc7b9..00000000 --- a/api_3/data_access.py +++ /dev/null @@ -1,542 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import json -import math -import traceback - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import BooleanField, EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.feature_value_types import ValueType, is_log_transformable -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_tcga_only, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess -from bq_data_access.utils import DurationLogged -from bq_data_access.data_access import FeatureIdQueryDescription - -from api_3.pairwise import PairwiseInputVector, Pairwise -from api_3.pairwise_api import PairwiseResults, PairwiseResultVector, PairwiseFilterMessage -from api_3.api_helpers import sql_connection - -from projects.models import Project - -import sys - -logger = logging.getLogger(__name__) - -VIZ_UNIT_DATADICTIONARY = { - 'BMI': 'kg/m^2', -} - -ISB_CGC_PROJECTS = { - 'list': [] -} - -# DUPLICATE METHOD -# Due to the way sql connections are done, it's easiest to duplicate this method and the static variable -# it creates. The original is in Cohorts/views, and all changes will happen there first. -# -# Generate the ISB_CGC_PROJECTS['list'] value set based on the get_isbcgc_project_set sproc -def fetch_isbcgc_project_set(): - try: - cursor = None - db = sql_connection() - if not ISB_CGC_PROJECTS['list'] or len(ISB_CGC_PROJECTS['list']) <= 0: - cursor = db.cursor() - cursor.execute("SELECT COUNT(SPECIFIC_NAME) FROM INFORMATION_SCHEMA.ROUTINES WHERE SPECIFIC_NAME = 'get_isbcgc_project_set';") - # Only try to fetch the study set if the sproc exists - if cursor.fetchall()[0][0] > 0: - cursor.execute("CALL get_isbcgc_project_set();") - ISB_CGC_PROJECTS['list'] = [] - for row in cursor.fetchall(): - ISB_CGC_PROJECTS['list'].append(row[0]) - else: - # Otherwise just warn - logger.warn("[WARNING] Stored procedure get_isbcgc_project_set was not found!") - - return ISB_CGC_PROJECTS['list'] - except Exception as e: - logger.error(e) - logger.error(traceback.format_exc()) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - -def get_axis_units(xAttr, yAttr): - units = {'x': '', 'y': ''} - - checkUnits = {} - if xAttr is not None: - checkUnits[xAttr] = 'x' - if yAttr is not None: - checkUnits[yAttr] = 'y' - - for attr in checkUnits: - - if '_age' in attr or 'age_' in attr or 'year_' in attr: - units[checkUnits[attr]] = 'years' - elif '_days' in attr or 'days_' in attr: - units[checkUnits[attr]] = 'days' - elif 'percent' in attr: - units[checkUnits[attr]] = 'percent' - elif 'CNVR:' in attr: - units[checkUnits[attr]] = 'log(CN/2)' - elif 'RPPA:' in attr: - units[checkUnits[attr]] = 'protein expression' - elif 'METH:' in attr: - units[checkUnits[attr]] = 'beta value' - elif 'GEXP:' in attr or 'MIRN:' in attr or ('GNAB:' in attr and "num_mutations" in attr): - units[checkUnits[attr]] = 'count' - elif attr.split(':')[1] in VIZ_UNIT_DATADICTIONARY: - units[checkUnits[attr]] = VIZ_UNIT_DATADICTIONARY[attr.split(':')[1]] - - return units - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, required=True) - - -class DataPoint(Message): - patient_id = StringField(1) - sample_id = StringField(2) - aliquot_id = StringField(3) - value = StringField(4) - - -class DataPointList(Message): - type = EnumField(ValueType, 1) - items = MessageField(DataPoint, 2, repeated=True) - - -class PlotDataRequest(Message): - x_id = StringField(1, required=True) - y_id = StringField(2, required=False) - c_id = StringField(3, required=False) - log_transform = StringField(4, required=False) - cohort_id = IntegerField(5, repeated=True) - pairwise = BooleanField(6, required=False) - - -class PlotDataPointCohortMemberships(Message): - ids = IntegerField(1, repeated=True) - - -class PlotDataCohortInfo(Message): - id = IntegerField(1, required=True) - name = StringField(2, required=True) - -DATAPOINT_COHORT_THRESHOLD = 1 - - -class PlotDataPoint(Message): - sample_id = StringField(1) - case_id = StringField(2) - x = StringField(3) - y = StringField(4) - c = StringField(5) - cohort = IntegerField(6, repeated=True) - - -class PlotDataTypes(Message): - x = EnumField(ValueType, 1) - y = EnumField(ValueType, 2) - c = EnumField(ValueType, 3) - - -class PlotDataFeatureLabels(Message): - x = StringField(1) - y = StringField(2) - c = StringField(3) - - -class PlotDatapointCohortSet(Message): - datapoint_id = StringField(1, required=True) - - -class PlotDatapointCount(Message): - total_num_patients = IntegerField(1, required=True) - total_num_samples = IntegerField(2, required=True) - num_patients_w_xy = IntegerField(3, required=True) - num_samples_w_xy = IntegerField(4, required=True) - num_patients_wo_x = IntegerField(5, required=True) - num_samples_wo_x = IntegerField(6, required=True) - num_patients_wo_y = IntegerField(7, required=True) - num_samples_wo_y = IntegerField(8, required=True) - - -class PlotDataResponse(Message): - types = MessageField(PlotDataTypes, 1, required=True) - labels = MessageField(PlotDataFeatureLabels, 2, required=True) - items = MessageField(PlotDataPoint, 3, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - counts = MessageField(PlotDatapointCount, 5) - pairwise_result = MessageField(PairwiseResults, 6, required=False) - xUnits = StringField(7, required=False) - yUnits = StringField(8, required=False) - - -FeatureDataEndpointsAPI = endpoints_api(name='feature_data_api', version='v1', - description='Endpoints for feature data used by the web application.') - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class FeatureDataEndpoints(remote.Service): - def get_counts(self, data): - total_num_patients = [] - total_num_samples = [] - num_samples_w_xy = [] - num_patients_w_xy = [] - num_samples_wo_x = [] - num_samples_wo_y = [] - num_patients_wo_x = [] - num_patients_wo_y = [] - - result = {} - - for item in data: - total_num_samples.append(item['sample_id']) - total_num_patients.append(item['sample_id'][:12]) - - if item['x'] != 'NA' and item['y'] != 'NA': - num_samples_w_xy.append(item['sample_id']) - num_patients_w_xy.append(item['sample_id'][:12]) - else: - if item['x'] == 'NA': - num_samples_wo_x.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_x.append(item['sample_id'][:12]) - elif item['y'] == 'NA': - num_samples_wo_y.append(item['sample_id']) - if item['sample_id'][:12] not in num_patients_w_xy: - num_patients_wo_y.append(item['sample_id'][:12]) - - result['total_num_patients'] = len(set(total_num_patients)) - result['total_num_samples'] = len(set(total_num_samples)) - result['num_patients_w_xy'] = len(set(num_patients_w_xy)) - result['num_samples_w_xy'] = len(set(num_samples_w_xy)) - result['num_patients_wo_x'] = len(set(num_patients_wo_x)) - result['num_samples_wo_x'] = len(set(num_samples_wo_x)) - result['num_patients_wo_y'] = len(set(num_patients_wo_y)) - result['num_samples_wo_y'] = len(set(num_samples_wo_y)) - - return result - - # TODO refactor to separate module - @DurationLogged('PAIRWISE', 'GET') - def get_pairwise_result(self, feature_array): - # Format the feature vectors for pairwise - input_vectors = Pairwise.prepare_feature_vector(feature_array) - outputs = None - results = None - - try: - outputs = Pairwise.run_pairwise(input_vectors) - - if outputs is not None: - results = PairwiseResults(result_vectors=[], filter_messages=[]) - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - except Exception as e: - outputs = None - results = None - logger.error(traceback.format_exc()) - - return results - - @DurationLogged('FEATURE', 'VECTOR_MERGE') - def get_merged_dict_timed(self, vms): - return vms.get_merged_dict() - - # TODO refactor missing value logic out of this module - @DurationLogged('FEATURE', 'GET_VECTORS') - def get_merged_feature_vectors(self, x_id, y_id, c_id, cohort_id_array, logTransform, study_id_array): - """ - Fetches and merges data for two or three feature vectors (see parameter documentation below). - The vectors have to be an array of dictionaries, with each dictionary containing a 'value' field - (other fields are ignored): - [ - { - 'value': 0.5 - }, - { - 'value': 1.0 - } - ] - The merged result: - [ - { - 'patient_id': - 'x': - 'y': - 'c': - }, - { - 'patient_id': - 'x': - 'y': - 'c': - } - ... - ] - - :param x_id: Feature identifier for x-axis e.g. 'CLIN:age_at_diagnosis' - :param y_id: Feature identifier for y-axis. If None, values for 'y' in the response will be marked as missing. - :param c_id: Feature identifier for color-by. If None, values for 'c' in the response will be marked as missing. - :param cohort_id_array: Cohort identifier array. - - :return: PlotDataResponse - """ - - async_params = [FeatureIdQueryDescription(x_id, cohort_id_array, study_id_array)] - - c_type, c_vec = ValueType.STRING, [] - y_type, y_vec = ValueType.STRING, [] - - units = get_axis_units(x_id, y_id) - - if c_id is not None: - async_params.append(FeatureIdQueryDescription(c_id, cohort_id_array, study_id_array)) - if y_id is not None: - async_params.append(FeatureIdQueryDescription(y_id, cohort_id_array, study_id_array)) - - async_result = get_feature_vectors_tcga_only(async_params) - - if c_id is not None: - c_type, c_vec = async_result[c_id]['type'], async_result[c_id]['data'] - if y_id is not None: - y_type, y_vec = async_result[y_id]['type'], async_result[y_id]['data'] - if logTransform is not None and logTransform['y'] and y_vec and is_log_transformable(y_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum y-value - ''' - yvals = [] - for yd in y_vec: - if 'value' in yd and yd['value'] is not None and yd['value'] != "NA" and yd['value'] != "None": - yvals.append(float(yd['value'])) - y_min = min(yvals) - ''' - for ydata in y_vec: - if 'value' in ydata and ydata['value'] is not None and ydata['value'] != "NA" and ydata['value'] != "None": - if float(ydata['value']) < 0: - ydata['value'] = "NA" - elif logTransform['yBase'] == 10: - ydata['value'] = str(math.log10((float(ydata['value']) + 1))) - elif logTransform['yBase'] == 'e': - ydata['value'] = str(math.log((float(ydata['value']) + 1))) - elif type(logTransform['yBase']) is int: - ydata['value'] = str(math.log((float(ydata['value']) + 1), logTransform['yBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - x_type, x_vec = async_result[x_id]['type'], async_result[x_id]['data'] - - if logTransform is not None and logTransform['x'] and x_vec and is_log_transformable(x_type): - # If we opt to use a transform that attempts to account for values out of range for log transformation, - # this is the code to get the minimum x-value - ''' - xvals = [] - for xd in x_vec: - if 'value' in xd and xd['value'] is not None and xd['value'] != "NA" and xd['value'] != "None": - xvals.append(float(xd['value'])) - x_min = min(xvals) - ''' - - for xdata in x_vec: - if 'value' in xdata and xdata['value'] is not None and xdata['value'] != "NA" and xdata['value'] != "None": - if float(xdata['value']) < 0: - xdata['value'] = "NA" - elif logTransform['xBase'] == 10: - xdata['value'] = str(math.log10((float(xdata['value']) + 1))) - elif logTransform['xBase'] == 'e': - xdata['value'] = str(math.log((float(xdata['value']) + 1))) - elif type(logTransform['xBase']) is int: - xdata['value'] = str(math.log((float(xdata['value']) + 1), logTransform['xBase'])) - else: - logger.warn( - "[WARNING] No valid log base was supplied - log transformation will not be applied!" - ) - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', ['x', 'y', 'c']) # changed so that it plots per sample not patient - vms.add_dict_array(x_vec, 'x', 'value') - vms.add_dict_array(y_vec, 'y', 'value') - vms.add_dict_array(c_vec, 'c', 'value') - merged = self.get_merged_dict_timed(vms) - - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(id=item['id'], name=item['name'])) - - items = [] - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - - if len(cohort_set) >= DATAPOINT_COHORT_THRESHOLD: - value_bundle['cohort'] = cohort_set - - items.append(PlotDataPoint(**value_bundle)) - - counts = self.get_counts(merged) - count_message = PlotDatapointCount(**counts) - - type_message = PlotDataTypes(x=x_type, y=y_type, c=c_type) - - # TODO assign label for y if y_id is None, as in that case the y-field will be missing from the response - label_message = PlotDataFeatureLabels(x=x_id, y=y_id, c=c_id) - - # TODO Refactor pairwise call to separate function - # Include pairwise results - input_vectors = [PairwiseInputVector(x_id, x_type, x_vec)] - if c_id is not None: - input_vectors.append(PairwiseInputVector(c_id, c_type, c_vec)) - if y_id is not None: - input_vectors.append(PairwiseInputVector(y_id, y_type, y_vec)) - - - pairwise_result = None - - if len(input_vectors) > 1: - pairwise_result = self.get_pairwise_result(input_vectors) - - if pairwise_result is None: - logger.warn("[WARNING] Pairwise results not included in returned object") - - return PlotDataResponse(types=type_message, labels=label_message, items=items, - cohort_set=cohort_info_obj_array, - counts=count_message, pairwise_result=pairwise_result, xUnits=units['x'], yUnits=units['y']) - - def get_feature_id_validity_for_array(self, feature_id_array): - """ - For each feature identifier in an array, check whether or not the identifier is - valid. - - Args: - feature_id_array: - - Returns: - Array of tuples - (feature identifier, ) - """ - result = [] - for feature_id in feature_id_array: - result.append((feature_id, is_valid_feature_identifier(feature_id))) - - return result - - @endpoints_method(PlotDataRequest, PlotDataResponse, - path='feature_data_plot', http_method='GET', name='feature_access.getFeatureDataForPlot') - def data_access_for_plot(self, request): - """ Used by the web application.""" - try: - x_id = request.x_id - y_id = request.y_id - c_id = request.c_id - logTransform = json.loads(request.log_transform) - cohort_id_array = request.cohort_id - - # Check that all requested feature identifiers are valid. Do not check for y_id if it is not - # supplied in the request. - feature_ids_to_check = [x_id] - if c_id is not None: - feature_ids_to_check.append(c_id) - if y_id is not None: - feature_ids_to_check.append(y_id) - - valid_features = self.get_feature_id_validity_for_array(feature_ids_to_check) - - for feature_id, is_valid in valid_features: - logging.info((feature_id, is_valid)) - if not is_valid: - logging.error("Invalid internal feature ID '{}'".format(feature_id)) - raise NotFoundException() - - # Get the project IDs these cohorts' samples come from - cohort_vals = () - cohort_params = "" - - for cohort in cohort_id_array: - cohort_params += "%s," - cohort_vals += (cohort,) - - cohort_params = cohort_params[:-1] - - db = sql_connection() - cursor = db.cursor() - - tcga_studies = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN ("+cohort_params+");",cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_study_ids = [] - unconfirmed_study_ids = [] - - for row in cursor.fetchall(): - if row[0] in tcga_studies: - if row[0] not in confirmed_study_ids: - confirmed_study_ids.append(row[0]) - elif row[0] not in unconfirmed_study_ids: - unconfirmed_study_ids.append(row[0]) - - if len(unconfirmed_study_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_study_ids) - - for project in projects: - if project.get_my_root_and_depth()['root'] in tcga_studies: - confirmed_study_ids.append(project.id) - - return self.get_merged_feature_vectors(x_id, y_id, c_id, cohort_id_array, logTransform, confirmed_study_ids) - except NotFoundException as nfe: - # Pass through NotFoundException so that it is not handled as Exception below. - raise nfe - except Exception as e: - logger.exception(e) - raise InternalServerErrorException() - diff --git a/api_3/feature_access.py b/api_3/feature_access.py deleted file mode 100755 index 1e987916..00000000 --- a/api_3/feature_access.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import BadRequestException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import Message, MessageField, StringField - -from bq_data_access.feature_search.common import BackendException, EmptyQueryException, InvalidFieldException, InvalidDataTypeException -from bq_data_access.gexp_data import GEXP_FEATURE_TYPE -from bq_data_access.clinical_data import CLINICAL_FEATURE_TYPE -from bq_data_access.methylation_data import METH_FEATURE_TYPE -from bq_data_access.copynumber_data import CNVR_FEATURE_TYPE -from bq_data_access.protein_data import RPPA_FEATURE_TYPE -from bq_data_access.mirna_data import MIRN_FEATURE_TYPE -from bq_data_access.gnab_data import GNAB_FEATURE_TYPE -from bq_data_access.feature_search.gexp_searcher import GEXPSearcher -from bq_data_access.feature_search.clinical_searcher import ClinicalSearcher -from bq_data_access.feature_search.methylation_searcher import METHSearcher -from bq_data_access.feature_search.copynumber_search import CNVRSearcher -from bq_data_access.feature_search.protein import RPPASearcher -from bq_data_access.feature_search.microrna_searcher import MIRNSearcher -from bq_data_access.feature_search.gnab_searcher import GNABSearcher - -class ClinicalFeatureType(Message): - feature_type = StringField(1) - gene = StringField(2) - label = StringField(3) - internal_id = StringField(4) - -class FeatureTypesRequest(Message): - keyword = StringField(1, required=True) - -class FeatureDataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = StringField(2, required=True) - -class FeatureTypeSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=False) - gene_name = StringField(3, required=False) - platform = StringField(4, required=False) - center = StringField(5, required=False) - protein_name = StringField(6, required=False) - value_field = StringField(7, required=False) - probe_name = StringField(8, required=False) - relation_to_gene = StringField(9, required=False) - relation_to_island = StringField(10, required=False) - mirna_name = StringField(11, required=False) - -class FeatureSearchResult(Message): - feature_type = StringField(1) - internal_feature_id = StringField(2) - label = StringField(3) - type = StringField(4) - -class FeatureTypeList(Message): - items = MessageField(FeatureSearchResult, 1, repeated=True) - -class FeatureTypeFieldSearchRequest(Message): - datatype = StringField(1, required=True) - keyword = StringField(2, required=True) - field = StringField(3, required=True) - -class FeatureFieldSearchResult(Message): - values = StringField(1, repeated=True) - - -class FeatureDefinitionSearcherFactory(object): - @classmethod - def build_from_datatype(cls, datatype): - if datatype == CLINICAL_FEATURE_TYPE: - return ClinicalSearcher() - elif datatype == GEXP_FEATURE_TYPE: - return GEXPSearcher() - elif datatype == METH_FEATURE_TYPE: - return METHSearcher() - elif datatype == CNVR_FEATURE_TYPE: - return CNVRSearcher() - elif datatype == RPPA_FEATURE_TYPE: - return RPPASearcher() - elif datatype == MIRN_FEATURE_TYPE: - return MIRNSearcher() - elif datatype == GNAB_FEATURE_TYPE: - return GNABSearcher() - #TODO build a full search on all features - #elif datatype == ALL: - # return FullSearcher() - raise InvalidDataTypeException("Invalid datatype '{datatype}'".format(datatype=datatype)) - -FeatureAccessEndpointsAPI = endpoints_api(name='feature_type_api', version='v1', - description='Endpoints used by the web application to return features.') -@FeatureAccessEndpointsAPI.api_class(resource_name='feature_type_endpoints') -class FeatureAccessEndpoints(remote.Service): - @endpoints_method(FeatureTypeSearchRequest, FeatureTypeList, - path='feature_search', http_method='GET', name='feature_access.FeatureSearch') - def feature_search(self, request): - """ Used by the web application.""" - try: - datatype = request.datatype - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - parameters = {} - for message in request.all_fields(): - field_name = message.name - if field_name != 'datatype': - value = request.get_assigned_value(field_name) - if value is not None: - parameters[field_name] = value - - result = searcher.search(parameters) - items = [] - fields = ['label', 'internal_feature_id', 'feature_type'] - for row in result: - obj = {key: row[key] for key in fields} - if obj['feature_type'] == 'CLIN': - obj['type'] = row['type'] - items.append(obj) - - return FeatureTypeList(items=items) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except EmptyQueryException as e: - logging.error("Empty query: %s", str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error("Invalid field: %s", str(e)) - raise BadRequestException(str(e)) - except BackendException: - logging.exception("feature_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - - @endpoints_method(FeatureTypeFieldSearchRequest, FeatureFieldSearchResult, - path='feature_field_search', http_method='GET', name='feature_access.getFeatureFieldSearch') - def feature_field_search(self, request): - """ Used by the web application.""" - try: - datatype, keyword, field = request.datatype, request.keyword, request.field - searcher = FeatureDefinitionSearcherFactory.build_from_datatype(datatype) - result = searcher.field_value_search(keyword, field) - return FeatureFieldSearchResult(values=result) - - except InvalidDataTypeException as e: - logging.error(str(e)) - raise BadRequestException() - except InvalidFieldException as e: - logging.error(str(e)) - raise BadRequestException() - except BackendException: - logging.exception("feature_field_search BackendException") - raise InternalServerErrorException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/isb_cgc_api/__init__.py b/api_3/isb_cgc_api/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api/cohort_file_manifest.py b/api_3/isb_cgc_api/cohort_file_manifest.py deleted file mode 100644 index 4f2244c4..00000000 --- a/api_3/isb_cgc_api/cohort_file_manifest.py +++ /dev/null @@ -1,198 +0,0 @@ -""" - -Copyright 2018, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -import logging -import django -from protorpc import remote, messages - -from django.core.signals import request_finished -from django.contrib.auth.models import User as Django_User -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.conf import settings - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints - -from cohorts.file_helpers import cohort_files -from cohorts.models import Cohort_Perms -from accounts.sa_utils import auth_dataset_whitelists_for_user - -BASE_URL = settings.BASE_URL - -logger = logging.getLogger(__name__) - - -class FileDetail(messages.Message): - program = messages.StringField(1, required=True) - case_barcode = messages.StringField(2, required=True) - file_path = messages.StringField(3, required=True) - file_gdc_uuid = messages.StringField(4) - disease_code = messages.StringField(5, required=True) - experimental_strategy = messages.StringField(6) - platform = messages.StringField(7) - data_category = messages.StringField(8) - data_type = messages.StringField(9) - data_format = messages.StringField(10) - access = messages.StringField(11) - case_gdc_uuid = messages.StringField(12) - project_short_name = messages.StringField(13) - - -class FileManifest(messages.Message): - files = messages.MessageField(FileDetail, 1, repeated=True) - total_file_count = messages.IntegerField(2, variant=messages.Variant.INT32) - files_retrieved = messages.IntegerField(3, variant=messages.Variant.INT32) - - -class FileManifestFilters(messages.Message): - program = messages.StringField(1, repeated=True) - disease_code = messages.StringField(2, repeated=True) - experimental_strategy = messages.StringField(3, repeated=True) - platform = messages.StringField(4, repeated=True) - data_category = messages.StringField(5, repeated=True) - data_type = messages.StringField(6, repeated=True) - data_format = messages.StringField(7, repeated=True) - project_short_name = messages.StringField(8, repeated=True) - - -class CohortFileManifest(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - fetch_count=messages.IntegerField(2), - offset=messages.IntegerField(3), - genomic_build=messages.StringField(4), - do_filter_count=messages.BooleanField(6) - ) - - POST_RESOURCE = endpoints.ResourceContainer( - FileManifestFilters, - cohort_id=messages.IntegerField(1, required=True), - fetch_count=messages.IntegerField(2), - offset=messages.IntegerField(3), - genomic_build=messages.StringField(4), - do_filter_count=messages.BooleanField(6) - ) - - def validate_user(self, cohort_id): - user_email = None - user = None - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.". - format(BASE_URL)) - django.setup() - try: - user = Django_User.objects.get(email=user_email) - Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) - except ObjectDoesNotExist as e: - err_msg = "Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e) - if 'Cohort_Perms' in e.message: - err_msg = "User {} does not have permissions on cohort {}.".format(user_email, cohort_id) - logger.exception(e) - raise endpoints.UnauthorizedException(err_msg) - finally: - request_finished.send(self) - return user - - def file_manifest(self, request): - cohort_id = request.get_assigned_value('cohort_id') - file_manifest = None - - try: - user = self.validate_user(cohort_id) - has_access = auth_dataset_whitelists_for_user(user.id) - - params = {} - - if request.get_assigned_value('offset'): - params['offset'] = request.get_assigned_value('offset') - - if request.get_assigned_value('fetch_count'): - params['limit'] = request.get_assigned_value('fetch_count') - else: - params['limit'] = settings.MAX_FILE_LIST_REQUEST - - if request.get_assigned_value('genomic_build'): - params['build'] = request.get_assigned_value('genomic_build').upper() - else: - params['build'] = 'HG19' - - params['access'] = has_access - - inc_filters = { - filter.name: request.get_assigned_value(filter.name) - for filter in request.all_fields() - if request.get_assigned_value(filter.name) and filter.name not in ['cohort_id','fetch_count','offset','genomic_build'] - } - - response = cohort_files(cohort_id, user=user, inc_filters=inc_filters, **params) - - file_manifest = FileManifest() - files = [] - - for file in response['file_list']: - files.append(FileDetail( - program=file['program'], - case_barcode=file['case'], - case_gdc_uuid=file['case_gdc_id'], - file_path=file['cloudstorage_location'], - file_gdc_uuid=file['file_gdc_id'], - disease_code=file['disease_code'], - project_short_name=file['project_short_name'], - experimental_strategy=file['exp_strat'], - platform=file['platform'], - data_category=file['datacat'], - data_type=file['datatype'], - data_format=file['dataformat'], - access=file['access'] - )) - - file_manifest.files = files - file_manifest.total_file_count = response['total_file_count'] - file_manifest.files_retrieved = len(files) - - except Exception as e: - logger.exception(e) - raise endpoints.InternalServerErrorException("There was an error while processing your request.") - - return file_manifest - - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortFileManifestAPI(CohortFileManifest): - - @endpoints.method(CohortFileManifest.GET_RESOURCE, FileManifest, http_method='GET', path='cohorts/{cohort_id}/file_manifest') - def file_manifest(self, request): - """ - Takes a cohort id as a required parameter and returns a manifest of all files associated - with all the samples in that cohort, as well as their metadata, up to a default limit of - 50,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - return super(CohortFileManifestAPI, self).file_manifest(request) - - @endpoints.method(CohortFileManifest.POST_RESOURCE, FileManifest, http_method='POST', path='cohorts/{cohort_id}/file_manifest') - def file_manifest_filtered(self, request): - """ - Takes a cohort id as a required parameter and a filter set as a required data payload, - and returns a manifest of all files associated with all the samples in that cohort, as - well as their metadata, up to a default limit of 50,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - return super(CohortFileManifestAPI, self).file_manifest(request) diff --git a/api_3/isb_cgc_api/cohort_get_list_helper.py b/api_3/isb_cgc_api/cohort_get_list_helper.py deleted file mode 100755 index 5a0e4f66..00000000 --- a/api_3/isb_cgc_api/cohort_get_list_helper.py +++ /dev/null @@ -1,408 +0,0 @@ -''' -Created on Apr 5, 2017 - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import django -import endpoints -import logging -import MySQLdb -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.cohort_endpoint_helpers import FilterDetails -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - -class CohortDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - case_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - cases = messages.StringField(13, repeated=True) - samples = messages.StringField(14, repeated=True) - -class CohortListDetails(messages.Message): - id = messages.StringField(1) - name = messages.StringField(2) - last_date_saved = messages.StringField(3) - permission = messages.StringField(4) - email = messages.StringField(5) - comments = messages.StringField(6) - source_type = messages.StringField(7) - source_notes = messages.StringField(8) - parent_id = messages.StringField(9, repeated=True) - filters = messages.MessageField(FilterDetails, 10, repeated=True) - case_count = messages.IntegerField(11, variant=messages.Variant.INT32) - sample_count = messages.IntegerField(12, variant=messages.Variant.INT32) - -class CohortDetailsList(messages.Message): - items = messages.MessageField(CohortListDetails, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - -class CohortsGetListQueryBuilder(object): - - def build_cohort_query(self, query_dict): - """ - Builds the query that will select cohort id, name, last_date_saved, - perms, comments, source type, and source notes - :param query_dict: should contain {'cohorts_cohort_perms.user_id': user_id, 'cohorts_cohort.active': unicode('1')} - :return: query_str, query_tuple - """ - query_str = 'SELECT cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'cohorts_cohort_comments.content AS comments, ' \ - 'cohorts_source.type AS source_type, ' \ - 'cohorts_source.notes AS source_notes ' \ - 'FROM cohorts_cohort_perms ' \ - 'JOIN cohorts_cohort ' \ - 'ON cohorts_cohort.id=cohorts_cohort_perms.cohort_id ' \ - 'JOIN auth_user ' \ - 'ON auth_user.id=cohorts_cohort_perms.user_id ' \ - 'LEFT JOIN cohorts_cohort_comments ' \ - 'ON cohorts_cohort_comments.user_id=cohorts_cohort_perms.user_id ' \ - 'AND cohorts_cohort_comments.cohort_id=cohorts_cohort.id ' \ - 'LEFT JOIN cohorts_source ' \ - 'ON cohorts_source.cohort_id=cohorts_cohort_perms.cohort_id ' - - query_tuple = () - if query_dict: - query_str += ' WHERE ' + '=%s and '.join(key for key in query_dict.keys()) + '=%s ' - query_tuple = tuple(value for value in query_dict.values()) - - query_str += 'GROUP BY ' \ - 'cohorts_cohort.id, ' \ - 'cohorts_cohort.name, ' \ - 'cohorts_cohort.last_date_saved, ' \ - 'cohorts_cohort_perms.perm, ' \ - 'auth_user.email, ' \ - 'comments, ' \ - 'source_type, ' \ - 'source_notes ' - - return query_str, query_tuple - - def build_filter_query(self, filter_query_dict): - """ - Builds the query that selects the filter name and value for a particular cohort - :param filter_query_dict: should be {'cohorts_filters.resulting_cohort_id:': id} - :return: filter_query_str, filter_query_tuple - """ - filter_query_str = 'SELECT name, value ' \ - 'FROM cohorts_filters ' - - filter_query_str += ' WHERE ' + '=%s AND '.join(key for key in filter_query_dict.keys()) + '=%s ' - filter_query_tuple = tuple(value for value in filter_query_dict.values()) - - return filter_query_str, filter_query_tuple - - def build_parent_query(self, parent_query_dict): - """ - Builds the query that selects parent_ids for a particular cohort - :param parent_query_dict: should be {'cohort_id': str(row['id'])} - :return: parent_query_str, parent_query_tuple - """ - parent_query_str = 'SELECT parent_id ' \ - 'FROM cohorts_source ' - parent_query_str += ' WHERE ' + '=%s AND '.join(key for key in parent_query_dict.keys()) + '=%s ' - parent_query_tuple = tuple(value for value in parent_query_dict.values()) - - return parent_query_str, parent_query_tuple - - def build_cases_query(self, case_query_dict): - """ - Builds the query that selects the case count for a particular cohort - :param case_query_dict: should be {'cohort_id': str(row['id])} - :return: case_query_str, case_query_tuple - """ - cases_query_str = 'SELECT cohort_id id, case_barcode ' \ - 'FROM cohorts_samples ' - - cases_query_str += ' WHERE ' + '=%s AND '.join(key for key in case_query_dict.keys()) + '=%s ' - case_query_tuple = tuple(value for value in case_query_dict.values()) - - return cases_query_str, case_query_tuple - - def build_samples_query(self, sample_query_dict): - """ - Builds the query that selects the sample count for a particular cohort - :param sample_query_dict: should be {'cohort_id': str(row['id])} - :return: sample_query_str, sample_query_tuple - """ - samples_query_str = 'SELECT sample_barcode, case_barcode ' \ - 'FROM cohorts_samples ' - - samples_query_str += ' WHERE ' + '=%s AND '.join(key for key in sample_query_dict.keys()) + '=%s ' - sample_query_tuple = tuple(value for value in sample_query_dict.values()) - - return samples_query_str, sample_query_tuple - -class CohortsGetListMessageBuilder(object): - def make_filter_details_from_cursor(self, filter_cursor_dict): - """ - Returns list of FilterDetails from a dictionary of results - from a filter query. - """ - filter_data = [] - for filter_row in filter_cursor_dict: - filter_data.append(FilterDetails( - name=str(filter_row['name']), - value=str(filter_row['value']) - )) - - if len(filter_data) == 0: - filter_data.append(FilterDetails( - name="None", - value="None" - )) - - return filter_data - - def make_parent_id_list_from_cursor(self, parent_cursor_dict, row): - """ - Returns list of parent_id's from a dictionary of results - from a parent id query. - """ - parent_id_data = [str(p_row['parent_id']) for p_row in parent_cursor_dict if row.get('parent_id')] - if len(parent_id_data) == 0: - parent_id_data.append("None") - - return parent_id_data - -class CohortsGetListAPI(remote.Service): - def validate_user(self): - user_email = None - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register " - "with the web application.". - format(BASE_URL)) - django.setup() - try: - user_id = Django_User.objects.get(email=user_email).id - except (ObjectDoesNotExist, MultipleObjectsReturned) as e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - return user_id,user_email - - def execute_getlist_query(self, param_map): - query_str, query_tuple = CohortsGetListQueryBuilder().build_cohort_query(param_map) - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - return cursor, db - - def get_cohort_details(self, cursor, row): - # get filters for each cohort - filter_query_str, filter_query_tuple = CohortsGetListQueryBuilder().build_filter_query( - { - 'cohorts_filters.resulting_cohort_id':str(row['id']) - } - ) - cursor.execute(filter_query_str, filter_query_tuple) - filter_data = CohortsGetListMessageBuilder().make_filter_details_from_cursor(cursor.fetchall()) - - # getting the parent_id's for each cohort is a separate query - # since a single cohort may have multiple parent cohorts - parent_query_str, parent_query_tuple = CohortsGetListQueryBuilder().build_parent_query( - { - 'cohort_id':str(row['id']) - } - ) - cursor.execute(parent_query_str, parent_query_tuple) - parent_id_data = CohortsGetListMessageBuilder().make_parent_id_list_from_cursor( - cursor.fetchall(), row) - - # get number of cases for each cohort - case_query_str, case_query_tuple = CohortsGetListQueryBuilder().build_cases_query( - { - 'cohort_id':str(row['id']) - } - ) - cursor.execute(case_query_str, case_query_tuple) - case_list = set() - for row in cursor.fetchall(): - case_list.add(row['case_barcode']) - case_count = len(case_list) - - # get number of samples for each cohort - sample_query_str, sample_query_tuple = CohortsGetListQueryBuilder().build_samples_query( - { - 'cohort_id':str(row['id']) - } - ) - cursor.execute(sample_query_str, sample_query_tuple) - sample_list = [] - for row in cursor.fetchall(): - sample_list.append(row['sample_barcode']) - sample_count = len(sample_list) - - return parent_id_data, filter_data, case_list, case_count, sample_list, sample_count - -class CohortsGetHelper(CohortsGetListAPI): - GET_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - def get(self, request): - """ - Returns information about a specific cohort the user has READER or OWNER permission on - when given a cohort ID. Authentication is required. - """ - user_id, user_email = self.validate_user() - - cursor = None - db = None - cohort_id = request.get_assigned_value('cohort_id') - param_map = { - 'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1'), - 'cohorts_cohort.id': cohort_id - } - try: - cursor, db = self.execute_getlist_query(param_map) - row = cursor.fetchone() - - if row is None: - raise endpoints.NotFoundException( - "Cohort {id} not found. Either it never existed, it was deleted, " - "or {user_email} does not have permission to view it.".format( - id=cohort_id, user_email=user_email)) - - parent_id_data, filter_data, case_list, _, sample_list, _ = self.get_cohort_details(cursor, row) - - temp_list = [] - if len(sample_list) == 0: - sample_list = ["None"] - else: - temp_list = [] - for sample in sample_list: - if sample is not None: - temp_list += [sample] - sample_list = temp_list - - if len(case_list) == 0: - case_list = ["None"] - else: - temp_list = [] - for case in case_list: - if case is not None: - temp_list += [case] - case_list = temp_list - - return CohortDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - case_count=len(case_list), - sample_count=len(sample_list), - cases=case_list, - samples=sample_list - ) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "Cohort {} for user {} not found. {}: {}".format(cohort_id, user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - -class CohortsListHelper(CohortsGetListAPI): - def list(self, unused_request): - """ - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. - """ - user_id, user_email = self.validate_user() - - param_map = { - 'cohorts_cohort_perms.user_id': user_id, - 'cohorts_cohort.active': unicode('1') - } - db = None - cursor = None - try: - cursor, db = self.execute_getlist_query(param_map) - data = [] - for row in cursor.fetchall(): - parent_id_data, filter_data, _, case_count, _, sample_count = self.get_cohort_details(cursor, row) - - data.append( - CohortListDetails( - id=str(row['id']), - name=str(row['name']), - last_date_saved=str(row['last_date_saved']), - permission=str(row['perm']), - email=str(row['email']), - comments=str(row['comments']), - source_type=str(row['source_type']), - source_notes=str(row['source_notes']), - parent_id=parent_id_data, - filters=filter_data, - case_count=case_count, - sample_count=sample_count - ) - ) - - if len(data) == 0: - raise endpoints.NotFoundException("{} has no active cohorts.".format(user_email)) - - return CohortDetailsList(items=data, count=len(data)) - - except (IndexError, TypeError) as e: - raise endpoints.NotFoundException( - "User {}'s cohorts not found. {}: {}".format(user_email, type(e), e)) - - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving cohorts or filters. {}".format(e)) - raise endpoints.BadRequestException("Error retrieving cohorts or filters. {}".format(e)) - - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py b/api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py deleted file mode 100755 index ed47aaf4..00000000 --- a/api_3/isb_cgc_api/cohorts_cloudstoragefilepaths.py +++ /dev/null @@ -1,68 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, CohortsCloudStorageFilePathsHelper - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsCloudStorageFilePathsAPI(CohortsCloudStorageFilePathsHelper): - - def build_program_query(self, final_query_str, query_tuple, program, param_map, build): - query_str = 'SELECT md.file_name_key, md.access '\ - 'FROM {}_metadata_data_{}_r14 md '.format(program, build) - query_str += 'JOIN cohorts_samples cs ON md.sample_barcode=cs.sample_barcode WHERE cs.cohort_id=%s ' - query_tuple += [param_map['cohort_id']] - query_str += 'AND file_name_key != "" AND file_name_key is not null ' - for field, value in param_map.iteritems(): - if field not in ['limit', 'cohort_id', 'sample_barcode', 'genomic_build'] and value: - query_str += ' and md.{}=%s '.format(field) - query_tuple += [value] - - query_str += ' GROUP BY md.file_name_key, md.access ' - if 0 < len(final_query_str): - final_query_str += ' UNION ' - final_query_str += query_str - return final_query_str, query_tuple - - def build_query(self, param_map, program): - builds = self.get_genomic_builds(param_map, '') - final_query_str = '' - query_tuple = [] - for program in ('CCLE', 'TARGET', 'TCGA'): - for build in builds: - if program == 'CCLE' and build != 'HG19': - continue - final_query_str, query_tuple = self.build_program_query(final_query_str, query_tuple, program, param_map, build) - - if 'limit' in param_map and param_map['limit']: - final_query_str += ' LIMIT %s' - query_tuple += [param_map['limit']] - else: - final_query_str += ' LIMIT 10000' - return final_query_str, query_tuple - - @endpoints.method(CohortsCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, http_method='GET', - path='cohorts/{cohort_id}/cloud_storage_file_paths') - def cloud_storage_file_paths(self, request): - """ - Takes a cohort id as a required parameter and returns cloud storage paths to files - associated with all the samples in that cohort, up to a default limit of 10,000 files. - Authentication is required. User must have READER or OWNER permissions on the cohort. - """ - return super(CohortsCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request) diff --git a/api_3/isb_cgc_api/cohorts_delete.py b/api_3/isb_cgc_api/cohorts_delete.py deleted file mode 100755 index 3cc3a2c3..00000000 --- a/api_3/isb_cgc_api/cohorts_delete.py +++ /dev/null @@ -1,85 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import django -import endpoints -import logging - -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints -from cohorts.models import Cohort as Django_Cohort, Cohort_Perms - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - -class ReturnJSON(messages.Message): - message = messages.StringField(1) - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsDeleteAPI(remote.Service): - DELETE_RESOURCE = endpoints.ResourceContainer(cohort_id=messages.IntegerField(1, required=True)) - - @endpoints.method(DELETE_RESOURCE, ReturnJSON, http_method='DELETE', path='cohorts/{cohort_id}') - def delete(self, request): - """ - Deletes a cohort. User must have owner permissions on the cohort. - """ - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - cohort_id = request.get_assigned_value('cohort_id') - - if user_email is None: - raise endpoints.UnauthorizedException( - "Authentication failed. Try signing in to {} to register with the web application.".format(BASE_URL)) - - django.setup() - try: - django_user = Django_User.objects.get(email=user_email) - user_id = django_user.id - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - request_finished.send(self) - raise endpoints.NotFoundException("%s does not have an entry in the user database." % user_email) - try: - cohort_to_deactivate = Django_Cohort.objects.get(id=cohort_id) - if cohort_to_deactivate.active is True: - cohort_perm = Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user_id) - if cohort_perm.perm == 'OWNER': - cohort_to_deactivate.active = False - cohort_to_deactivate.save() - return_message = 'Cohort %d successfully deleted.' % cohort_id - else: - return_message = 'You do not have owner permission on cohort %d.' % cohort_id - else: - return_message = "Cohort %d was already deleted." % cohort_id - return ReturnJSON(message=return_message) - - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Either cohort %d does not have an entry in the database " - "or you do not have owner or reader permissions on this cohort." % cohort_id) - finally: - request_finished.send(self) diff --git a/api_3/isb_cgc_api/cohorts_get.py b/api_3/isb_cgc_api/cohorts_get.py deleted file mode 100755 index 0c3464df..00000000 --- a/api_3/isb_cgc_api/cohorts_get.py +++ /dev/null @@ -1,30 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints -from api_3.isb_cgc_api.cohort_get_list_helper import CohortDetails, CohortsGetHelper - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsGetAPI(CohortsGetHelper): - @endpoints.method(CohortsGetHelper.GET_RESOURCE, CohortDetails, http_method='GET', path='cohorts/{cohort_id}') - def get(self, request): - """ - Returns information about a specific cohort the user has READER or OWNER permission on - when given a cohort ID. Authentication is required. - """ - return super(CohortsGetAPI, self).get(request) diff --git a/api_3/isb_cgc_api/cohorts_list.py b/api_3/isb_cgc_api/cohorts_list.py deleted file mode 100755 index 04078bd2..00000000 --- a/api_3/isb_cgc_api/cohorts_list.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from protorpc import message_types - -from api_3.isb_cgc_api.cohort_get_list_helper import CohortDetailsList, CohortsListHelper -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints - -@ISB_CGC_Endpoints.api_class(resource_name='cohorts') -class CohortsListAPI(CohortsListHelper): - @endpoints.method(message_types.VoidMessage, CohortDetailsList, http_method='GET', path='cohorts') - def list(self, unused_request): - """ - Returns information about cohorts a user has either READER or OWNER permission on. - Authentication is required. - """ - return super(CohortsListAPI, self).list(unused_request) diff --git a/api_3/isb_cgc_api/files_get_file_paths.py b/api_3/isb_cgc_api/files_get_file_paths.py deleted file mode 100755 index f110bbbe..00000000 --- a/api_3/isb_cgc_api/files_get_file_paths.py +++ /dev/null @@ -1,68 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import remote, messages - -from api_3.isb_cgc_api.isb_cgc_api_helpers import ISB_CGC_Endpoints - -from api_3.api_helpers import sql_connection - - -@ISB_CGC_Endpoints.api_class(resource_name='files') -class FilesGetPath(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer(file_uuids=messages.StringField(1, repeated=True)) - - class FilePaths(messages.Message): - paths = messages.StringField(1, repeated=True) - - @endpoints.method(GET_RESOURCE, FilePaths, http_method='GET', path='file_paths') - def get(self, request): - """ - from the list of file gdc UUIDs, returns the google cloud storage file paths for those UUIDs. the returned - filepaths will be in the same order as the passed in UUIDs - """ - uuids = request.get_assigned_value('file_uuids') - in_clause = '' - params = [] - for uuid in uuids: - in_clause += '%s, ' - params += ['{}'.format(uuid)] - in_clause = in_clause[:-2] - - uuid2paths = {} - sql = 'select file_gdc_id, file_name_key from {}_metadata_data_{}_r14 where file_gdc_id in ({}) order by 1, 2' - programs = ['CCLE', 'TARGET', 'TCGA'] - db = sql_connection() - for program in programs: - if 'CCLE' == program: - builds = ['HG19'] - else: - builds = ['HG19', 'HG38'] - - for build in builds: - try: - cursor = db.cursor() - cursor.execute(sql.format(program, build, in_clause), params) - for row in cursor: - paths = uuid2paths.setdefault(row[0], []) - paths += [row[1]] - except Exception as e: - print 'problem executing sql({}):\n\t{}\n\t{}'.format(e, sql, params) - raise - filepaths = FilesGetPath.FilePaths() - filepaths.paths = [item for sublist in uuid2paths.values() for item in sublist] - return filepaths diff --git a/api_3/isb_cgc_api/isb_cgc_api_helpers.py b/api_3/isb_cgc_api/isb_cgc_api_helpers.py deleted file mode 100755 index 6ce7a4af..00000000 --- a/api_3/isb_cgc_api/isb_cgc_api_helpers.py +++ /dev/null @@ -1,27 +0,0 @@ -""" - -Copyright 2018, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_Endpoints = endpoints.api(name='isb_cgc_api', version='v3', - description="Get information about cohorts for ISB-CGC. List, get, delete cohorts, and retrieve or export their file manifests.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC API") diff --git a/api_3/isb_cgc_api_CCLE/__init__.py b/api_3/isb_cgc_api_CCLE/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json b/api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json deleted file mode 100755 index 1c8cef23..00000000 --- a/api_3/isb_cgc_api_CCLE/allowed_values_v3_CCLE.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "Common": { - "disease_code": [ - "BLCA", - "BRCA", - "CESC", - "COAD", - "DLBC", - "ESCA", - "HNSC", - "KIRC", - "LCLL", - "LGG", - "LIHC", - "LUSC", - "MESO", - "MM", - "OV", - "PAAD", - "PRAD", - "SARC", - "SKCM", - "STAD", - "THCA", - "UCEC" - ], - "endpoint_type": [ - "legacy" - ], - "program_name": [ - "CCLE" - ], - "project_short_name": [ - "CCLE-BLCA", - "CCLE-BRCA", - "CCLE-CESC", - "CCLE-COAD", - "CCLE-DLBC", - "CCLE-ESCA", - "CCLE-HNSC", - "CCLE-KIRC", - "CCLE-LCLL", - "CCLE-LGG", - "CCLE-LIHC", - "CCLE-LUSC", - "CCLE-MESO", - "CCLE-MM", - "CCLE-OV", - "CCLE-PAAD", - "CCLE-PRAD", - "CCLE-SARC", - "CCLE-SKCM", - "CCLE-STAD", - "CCLE-THCA", - "CCLE-UCEC" - ] - }, - "Clinical": { - "gender": [ - "Female", - "Male", - "Undefined" - ], - "histology": [ - "carcinoid-endocrine_tumour", - "carcinoma", - "chondrosarcoma", - "Ewings_sarcoma-peripheral_primitive_neuroectodermal_tumour", - "fibrosarcoma", - "giant_cell_tumour", - "glioma", - "haematopoietic_neoplasm", - "leiomyosarcoma", - "lymphoid_neoplasm", - "malignant_fibrous_histiocytoma-pleomorphic_sarcoma", - "malignant_melanoma", - "mesothelioma", - "neuroblastoma", - "osteosarcoma", - "other", - "primitive_neuroectodermal_tumour-medulloblastoma", - "rhabdoid_tumour", - "rhabdomyosarcoma", - "sarcoma", - "sex_cord-stromal_tumour" - ], - "hist_subtype": [ - "acute_lymphoblastic_B_cell_leukaemia", - "acute_lymphoblastic_T_cell_leukaemia", - "acute_myeloid_leukaemia", - "adenocarcinoma", - "adult_T_cell_lymphoma-leukaemia", - "alveolar", - "anaplastic_carcinoma", - "anaplastic_large_cell_lymphoma", - "astrocytoma", - "astrocytoma_Grade_III", - "astrocytoma_Grade_IV", - "blast_phase_chronic_myeloid_leukaemia", - "Brenner_tumour", - "bronchioloalveolar_adenocarcinoma", - "Burkitt_lymphoma", - "B_cell_lymphoma_unspecified", - "carcinosarcoma-malignant_mesodermal_mixed_tumour", - "chronic_lymphocytic_leukaemia-small_lymphocytic_lymphoma", - "chronic_myeloid_leukaemia", - "clear_cell_carcinoma", - "clear_cell_renal_cell_carcinoma", - "dedifferentiated", - "diffuse_adenocarcinoma", - "diffuse_large_B_cell_lymphoma", - "ductal_carcinoma", - "embryonal", - "endometrioid_carcinoma", - "essential_thrombocythaemia", - "follicular_carcinoma", - "giant_cell_tumour", - "gliosarcoma", - "granulosa_cell_tumour", - "hepatoblastoma", - "hepatocellular_carcinoma", - "Hodgkin_lymphoma", - "intestinal_adenocarcinoma", - "large_cell_carcinoma", - "mantle_cell_lymphoma", - "medullary_carcinoma", - "metaplasia", - "metaplastic_carcinoma", - "mixed_adenosquamous_carcinoma", - "mixed_carcinoma", - "mucinous_carcinoma", - "mucoepidermoid_carcinoma", - "mycosis_fungoides-Sezary_syndrome", - "non_small_cell_carcinoma", - "NS", - "oligodendroglioma", - "papillary_carcinoma", - "papilloma", - "peripheral_T_cell_lymphoma_unspecified", - "plasma_cell_myeloma", - "renal_cell_carcinoma", - "serous_carcinoma", - "signet_ring_adenocarcinoma", - "small_cell_adenocarcinoma", - "small_cell_carcinoma", - "squamous_cell_carcinoma", - "transitional_cell_carcinoma", - "tubular_adenocarcinoma", - "undifferentiated_adenocarcinoma", - "undifferentiated_carcinoma" - ], - "site_primary": [ - "autonomic_ganglia", - "biliary_tract", - "bone", - "breast", - "central_nervous_system", - "endometrium", - "haematopoietic_and_lymphoid_tissue", - "kidney", - "large_intestine", - "liver", - "lung", - "oesophagus", - "ovary", - "pancreas", - "pleura", - "prostate", - "salivary_gland", - "skin", - "small_intestine", - "soft_tissue", - "stomach", - "thyroid", - "upper_aerodigestive_tract", - "urinary_tract" - ], - "source": [ - "Academic Lab", - "ATCC", - "DSMZ", - "ECACC", - "HSRRB", - "ICLC", - "KCLB", - "NCI/DCTD", - "RIKEN" - ] - }, - "Biospecimen": { - "sample_type": [ - "13", - "50" - ] - }, - "Data_HG19": { - "access": [ - "open" - ], - "analysis_workflow_type": [ - - ], - "center_code": [ - - ], - "center_name": [ - - ], - "center_type": [ - - ], - "data_category": [ - "Raw sequencing data" - ], - "data_format": [ - "BAM" - ], - "data_type": [ - "Aligned reads" - ], - "experimental_strategy": [ - "RNA-Seq", - "WGS", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "true" - ], - "platform": [ - "Illumina HiSeq" - ], - "sample_type": [ - "13", - "50" - ], - "species": [ - "Homo sapiens" - ] - } -} diff --git a/api_3/isb_cgc_api_CCLE/cohorts_create.py b/api_3/isb_cgc_api_CCLE/cohorts_create.py deleted file mode 100755 index 2d284f50..00000000 --- a/api_3/isb_cgc_api_CCLE/cohorts_create.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsCreateHelper, CreatedCohort -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.isb_cgc_api_CCLE.message_classes import MetadataRangesItem - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='cohorts') -class CCLECohortsCreateAPI(CohortsCreateHelper): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='ccle/cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - self.program = 'CCLE' - return super(CCLECohortsCreateAPI, self).create(request) diff --git a/api_3/isb_cgc_api_CCLE/cohorts_preview.py b/api_3/isb_cgc_api_CCLE/cohorts_preview.py deleted file mode 100755 index 3166782f..00000000 --- a/api_3/isb_cgc_api_CCLE/cohorts_preview.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from api_3.cohort_create_preview_helper import CohortsPreviewHelper -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from message_classes import MetadataRangesItem - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='cohorts') -class CCLECohortsPreviewAPI(CohortsPreviewHelper): - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem) - - @endpoints.method(POST_RESOURCE, CohortsPreviewHelper.CohortCasesSamplesList, path='tcga/cohorts/preview', http_method='POST') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - self.program = 'CCLE' - return super(CCLECohortsPreviewAPI, self).preview(request) diff --git a/api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py b/api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py deleted file mode 100755 index 57e67f8c..00000000 --- a/api_3/isb_cgc_api_CCLE/isb_cgc_api_helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_CCLE_Endpoints = endpoints.api(name='isb_cgc_ccle_api', version='v3', - description="Get information about cohorts, cases, and samples for CCLE. Create cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - package_path='ccle', - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC CCLE API") diff --git a/api_3/isb_cgc_api_CCLE/message_classes.py b/api_3/isb_cgc_api_CCLE/message_classes.py deleted file mode 100755 index c17da327..00000000 --- a/api_3/isb_cgc_api_CCLE/message_classes.py +++ /dev/null @@ -1,94 +0,0 @@ -from protorpc import messages - -class CommonMetadataRangesItem(messages.Message): - disease_code = messages.StringField(1, repeated=True) - endpoint_type = messages.StringField(2, repeated=True) - program_name = messages.StringField(3, repeated=True) - project_short_name = messages.StringField(4, repeated=True) - -class CommonMetadataItem(messages.Message): - disease_code = messages.StringField(1) - endpoint_type = messages.StringField(2) - program_name = messages.StringField(3) - project_short_name = messages.StringField(4) - -class ClinicalMetadataRangesItem(messages.Message): - case_barcode = messages.StringField(1, repeated=True) - case_gdc_id = messages.StringField(2, repeated=True) - gender = messages.StringField(3, repeated=True) - histology = messages.StringField(4, repeated=True) - hist_subtype = messages.StringField(5, repeated=True) - site_primary = messages.StringField(6, repeated=True) - source = messages.StringField(7, repeated=True) - summary_file_count = messages.IntegerField(8, repeated=True, variant=messages.Variant.INT32) - summary_file_count_lte = messages.IntegerField(9, variant=messages.Variant.INT32) - summary_file_count_gte = messages.IntegerField(10, variant=messages.Variant.INT32) - - -class ClinicalMetadataItem(messages.Message): - case_barcode = messages.StringField(1) - case_gdc_id = messages.StringField(2) - gender = messages.StringField(3) - histology = messages.StringField(4) - hist_subtype = messages.StringField(5) - site_primary = messages.StringField(6) - source = messages.StringField(7) - summary_file_count = messages.IntegerField(8, variant=messages.Variant.INT32) - -class BiospecimenMetadataRangesItem(messages.Message): - case_barcode = messages.StringField(1, repeated=True) - case_gdc_id = messages.StringField(2, repeated=True) - sample_barcode = messages.StringField(3, repeated=True) - sample_gdc_id = messages.StringField(4, repeated=True) - sample_type = messages.StringField(5, repeated=True) - -class BiospecimenMetadataItem(messages.Message): - case_barcode = messages.StringField(1) - case_gdc_id = messages.StringField(2) - sample_barcode = messages.StringField(3) - sample_gdc_id = messages.StringField(4) - sample_type = messages.StringField(5) - -class Data_HG19MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - case_barcode = messages.StringField(5, repeated=True) - case_gdc_id = messages.StringField(6, repeated=True) - data_category = messages.StringField(10, repeated=True) - data_format = messages.StringField(11, repeated=True) - data_type = messages.StringField(12, repeated=True) - experimental_strategy = messages.StringField(13, repeated=True) - file_name_key = messages.StringField(14, repeated=True) - index_file_name_key = messages.StringField(17, repeated=True) - platform = messages.StringField(18, repeated=True) - sample_barcode = messages.StringField(19, repeated=True) - sample_gdc_id = messages.StringField(20, repeated=True) - -class Data_HG19MetadataItem(messages.Message): - access = messages.StringField(1) - case_barcode = messages.StringField(5) - case_gdc_id = messages.StringField(6) - center_code = messages.StringField(7) - center_name = messages.StringField(8) - center_type = messages.StringField(9) - data_category = messages.StringField(10) - data_format = messages.StringField(11) - data_type = messages.StringField(12) - experimental_strategy = messages.StringField(13) - file_name_key = messages.StringField(14) - index_file_name_key = messages.StringField(17) - platform = messages.StringField(18) - sample_barcode = messages.StringField(19) - sample_gdc_id = messages.StringField(20) - -class MetadataRangesItem(messages.Message): - Common = messages.MessageField(CommonMetadataRangesItem, 1) - Clinical = messages.MessageField(ClinicalMetadataRangesItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataRangesItem, 3) - data_HG19_r14 = messages.MessageField(Data_HG19MetadataRangesItem, 4) - -class MetadataItem(messages.Message): - Common = messages.MessageField(CommonMetadataItem, 1) - Clinical = messages.MessageField(ClinicalMetadataItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataItem, 3) - data_HG19_r14 = messages.MessageField(Data_HG19MetadataItem, 4) - diff --git a/api_3/isb_cgc_api_CCLE/patients_get.py b/api_3/isb_cgc_api_CCLE/patients_get.py deleted file mode 100755 index 84d8aea1..00000000 --- a/api_3/isb_cgc_api_CCLE/patients_get.py +++ /dev/null @@ -1,55 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.patients_get_helper import CasesGetHelper -from api_3.isb_cgc_api_CCLE.message_classes import ClinicalMetadataItem as MetadataItem -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints - -class CaseDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4) - -class CaseSetDetails(messages.Message): - cases = messages.MessageField(CaseDetails, 1, repeated=True) - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='cases') -class CCLECasesGetAPI(CasesGetHelper): - @endpoints.method(CasesGetHelper.GET_RESOURCE, CaseDetails, path='ccle/cases/{case_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case barcode (*eg* ACC-MESO-1) as a required parameter. - User does not need to be authenticated. - """ - return super(CCLECasesGetAPI, self).get(request, CaseDetails, MetadataItem, 'CCLE') - - - @endpoints.method(CasesGetHelper.POST_RESOURCE, CaseSetDetails, path='ccle/cases', http_method='POST') - def get_list(self, request): - """ - Given a list of case barcodes (*eg* ACC-MESO-1), returns information about them, including a - list of samples and aliquots derived from this case. - Takes a list of case barcodes (*eg* ACC-MESO-1) as a required data payload. - User does not need to be authenticated. - """ - return super(CCLECasesGetAPI, self).get_list(request, CaseSetDetails, CaseDetails, MetadataItem, 'CCLE') diff --git a/api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py b/api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py deleted file mode 100755 index 62b2afd8..00000000 --- a/api_3/isb_cgc_api_CCLE/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, SamplesCloudStorageFilePathsHelper - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='samples') -class CCLESamplesCloudStorageFilePathsAPI(SamplesCloudStorageFilePathsHelper): - - @endpoints.method(SamplesCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, - path='ccle/samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - return super(CCLESamplesCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request, 'CCLE') diff --git a/api_3/isb_cgc_api_CCLE/samples_get.py b/api_3/isb_cgc_api_CCLE/samples_get.py deleted file mode 100755 index bc59dd43..00000000 --- a/api_3/isb_cgc_api_CCLE/samples_get.py +++ /dev/null @@ -1,59 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.isb_cgc_api_CCLE.message_classes import MetadataItem -from api_3.samples_get_helper import SamplesGetAPI, DataDetails - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(MetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - case_barcode = messages.StringField(3) - case_gdc_id = messages.StringField(4) - data_details = messages.MessageField(DataDetails, 5, repeated=True) - data_details_count = messages.IntegerField(6, variant=messages.Variant.INT32) - sample_barcode = messages.StringField(7) - - -class SampleSetDetails(messages.Message): - samples = messages.MessageField(SampleDetails, 1, repeated=True) - - -@ISB_CGC_CCLE_Endpoints.api_class(resource_name='samples') -class CCLESamplesGetAPI(SamplesGetAPI): - @endpoints.method(SamplesGetAPI.GET_RESOURCE, SampleDetails, path='ccle/samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (*eg* CCLE-ACC-MESO-1), this endpoint returns - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(CCLESamplesGetAPI, self).get(request, 'CCLE', SampleDetails, MetadataItem) - - - @endpoints.method(SamplesGetAPI.POST_RESOURCE, SampleSetDetails, path='ccle/samples', http_method='POST') - def get_list(self, request): - """ - Given a list of sample barcodes (of length 16, *eg* CCLE-ACC-MESO-1), this endpoint returns - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(CCLESamplesGetAPI, self).get_list(request, 'CCLE', SampleSetDetails, SampleDetails, MetadataItem) diff --git a/api_3/isb_cgc_api_CCLE/samples_googlegenomics.py b/api_3/isb_cgc_api_CCLE/samples_googlegenomics.py deleted file mode 100755 index 2b6e1921..00000000 --- a/api_3/isb_cgc_api_CCLE/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from api_3.isb_cgc_api_CCLE.isb_cgc_api_helpers import ISB_CGC_CCLE_Endpoints -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_CCLE_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api_3/isb_cgc_api_TARGET/__init__.py b/api_3/isb_cgc_api_TARGET/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json b/api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json deleted file mode 100755 index 0f61d013..00000000 --- a/api_3/isb_cgc_api_TARGET/allowed_values_v3_TARGET.json +++ /dev/null @@ -1,243 +0,0 @@ -{ - "Common": { - "disease_code": [ - "ALL", - "AML", - "CCSK", - "NBL", - "OS", - "RT", - "WT" - ], - "endpoint_type": [ - "current", - "legacy" - ], - "program_name": [ - "TARGET" - ], - "project_short_name": [ - "TARGET-ALL-P1", - "TARGET-ALL-P2", - "TARGET-AML", - "TARGET-CCSK", - "TARGET-NBL", - "TARGET-OS", - "TARGET-RT", - "TARGET-WT" - ] - }, - "Clinical": { - "ethnicity": [ - "Hispanic or Latino", - "Not Hispanic or Latino" - ], - "first_event": [ - "Censored", - "Death", - "Death without remission", - "Event", - "Induction failure", - "Progression", - "Relapse", - "Second Malignant Neoplasm" - ], - "gender": [ - "Female", - "Male" - ], - "race": [ - "American Indian or Alaska Native", - "Asian", - "Black or African American", - "Native Hawaiian or other Pacific Islander", - "Other", - "White" - ], - "vital_status": [ - "alive", - "dead" - ] - }, - "Biospecimen": { - "sample_type": [ - "01", - "02", - "03", - "04", - "06", - "08", - "09", - "10", - "11", - "13", - "14", - "15", - "20", - "40", - "41", - "42", - "50", - "60" - ], - "tumor_code": [ - "00", - "10", - "20", - "21", - "30", - "40", - "50", - "51", - "52" - ] - }, - "Data_HG19": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - - ], - "center_code": [ - "01", - "12", - "13", - "37" - ], - "center_name": [ - "Baylor College of Medicine", - "Broad Institute of MIT and Harvard", - "Canada's Michael Smith Genome Sciences Centre", - "Complete Genomics Inc." - ], - "center_type": [ - "CGCC", - "COM" - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw sequencing data" - ], - "data_format": [ - "BAM", - "XLSX" - ], - "data_type": [ - "Aligned reads", - "Biospecimen Supplement", - "Clinical Supplement" - ], - "experimental_strategy": [ - "Bisulfite-Seq", - "miRNA-Seq", - "RNA-Seq", - "VALIDATION", - "WGS", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "false", - "true" - ], - "platform": [ - "Complete Genomics", - "Illumina GA", - "Illumina HiSeq", - "Ion Torrent PGM" - ], - "sample_type": [ - "01", - "02", - "03", - "04", - "06", - "09", - "10", - "11", - "14", - "15", - "20", - "40", - "41", - "42", - "50" - ], - "species": [ - "Homo sapiens" - ] - }, - "Data_HG38": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - "BWA with Mark Duplicates and Cocleaning", - "BWA-aln", - "STAR 2-Pass" - ], - "center_code": [ - - ], - "center_name": [ - - ], - "center_type": [ - - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw Sequencing Data" - ], - "data_format": [ - "BAM", - "XLSX" - ], - "data_type": [ - "Aligned Reads", - "Biospecimen Supplement", - "Clinical Supplement" - ], - "experimental_strategy": [ - "miRNA-Seq", - "RNA-Seq", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "false", - "true" - ], - "platform": [ - "Illumina" - ], - "sample_type": [ - "01", - "02", - "03", - "04", - "06", - "09", - "10", - "11", - "14", - "20", - "40", - "41", - "42", - "50" - ], - "species": [ - "Homo sapiens" - ] - } -} diff --git a/api_3/isb_cgc_api_TARGET/cohorts_create.py b/api_3/isb_cgc_api_TARGET/cohorts_create.py deleted file mode 100755 index 486654f1..00000000 --- a/api_3/isb_cgc_api_TARGET/cohorts_create.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsCreateHelper, CreatedCohort -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.isb_cgc_api_TARGET.message_classes import MetadataRangesItem - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='cohorts') -class TARGETCohortsCreateAPI(CohortsCreateHelper): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='target/cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - self.program = 'TARGET' - return super(TARGETCohortsCreateAPI, self).create(request) diff --git a/api_3/isb_cgc_api_TARGET/cohorts_preview.py b/api_3/isb_cgc_api_TARGET/cohorts_preview.py deleted file mode 100755 index 09cb1e74..00000000 --- a/api_3/isb_cgc_api_TARGET/cohorts_preview.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints - -from api_3.cohort_create_preview_helper import CohortsPreviewHelper -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from message_classes import MetadataRangesItem - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='cohorts') -class TARGETCohortsPreviewAPI(CohortsPreviewHelper): - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem) - - @endpoints.method(POST_RESOURCE, CohortsPreviewHelper.CohortCasesSamplesList, path='tcga/cohorts/preview', http_method='POST') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - self.program = 'TARGET' - return super(TARGETCohortsPreviewAPI, self).preview(request) diff --git a/api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py b/api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py deleted file mode 100755 index cda91463..00000000 --- a/api_3/isb_cgc_api_TARGET/isb_cgc_api_helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_TARGET_Endpoints = endpoints.api(name='isb_cgc_target_api', version='v3', - description="Get information about cohorts, cases, and samples for TARGET. Create cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - package_path='target', - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC TARGET API") diff --git a/api_3/isb_cgc_api_TARGET/message_classes.py b/api_3/isb_cgc_api_TARGET/message_classes.py deleted file mode 100755 index 627398b1..00000000 --- a/api_3/isb_cgc_api_TARGET/message_classes.py +++ /dev/null @@ -1,170 +0,0 @@ -from protorpc import messages - -class CommonMetadataRangesItem(messages.Message): - disease_code = messages.StringField(1, repeated=True) - endpoint_type = messages.StringField(2, repeated=True) - program_name = messages.StringField(3, repeated=True) - project_short_name = messages.StringField(4, repeated=True) - -class CommonMetadataItem(messages.Message): - disease_code = messages.StringField(1) - endpoint_type = messages.StringField(2) - program_name = messages.StringField(3) - project_short_name = messages.StringField(4) - -class ClinicalMetadataRangesItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, repeated=True, variant=messages.Variant.INT32) - age_at_diagnosis_lte = messages.IntegerField(2, variant=messages.Variant.INT32) - age_at_diagnosis_gte = messages.IntegerField(3, variant=messages.Variant.INT32) - - case_barcode = messages.StringField(4, repeated=True) - case_gdc_id = messages.StringField(5, repeated=True) - days_to_birth = messages.IntegerField(6, repeated=True, variant=messages.Variant.INT32) - days_to_birth_lte = messages.IntegerField(7, variant=messages.Variant.INT32) - days_to_birth_gte = messages.IntegerField(8, variant=messages.Variant.INT32) - - days_to_death = messages.IntegerField(9, repeated=True, variant=messages.Variant.INT32) - days_to_death_lte = messages.IntegerField(10, variant=messages.Variant.INT32) - days_to_death_gte = messages.IntegerField(11, variant=messages.Variant.INT32) - - days_to_last_followup = messages.IntegerField(12, repeated=True, variant=messages.Variant.INT32) - days_to_last_followup_lte = messages.IntegerField(13, variant=messages.Variant.INT32) - days_to_last_followup_gte = messages.IntegerField(14, variant=messages.Variant.INT32) - - days_to_last_known_alive = messages.IntegerField(15, repeated=True, variant=messages.Variant.INT32) - days_to_last_known_alive_lte = messages.IntegerField(16, variant=messages.Variant.INT32) - days_to_last_known_alive_gte = messages.IntegerField(17, variant=messages.Variant.INT32) - - ethnicity = messages.StringField(18, repeated=True) - event_free_survival = messages.IntegerField(19, repeated=True, variant=messages.Variant.INT32) - event_free_survival_lte = messages.IntegerField(20, variant=messages.Variant.INT32) - event_free_survival_gte = messages.IntegerField(21, variant=messages.Variant.INT32) - - first_event = messages.StringField(22, repeated=True) - gender = messages.StringField(23, repeated=True) - protocol = messages.StringField(24, repeated=True) - race = messages.StringField(25, repeated=True) - summary_file_count = messages.IntegerField(26, repeated=True, variant=messages.Variant.INT32) - summary_file_count_lte = messages.IntegerField(27, variant=messages.Variant.INT32) - summary_file_count_gte = messages.IntegerField(28, variant=messages.Variant.INT32) - - vital_status = messages.StringField(29, repeated=True) - wbc_at_diagnosis = messages.FloatField(30, repeated=True) - wbc_at_diagnosis_lte = messages.FloatField(31) - wbc_at_diagnosis_gte = messages.FloatField(32) - - year_of_diagnosis = messages.IntegerField(33, repeated=True, variant=messages.Variant.INT32) - year_of_diagnosis_lte = messages.IntegerField(34, variant=messages.Variant.INT32) - year_of_diagnosis_gte = messages.IntegerField(35, variant=messages.Variant.INT32) - - year_of_last_follow_up = messages.IntegerField(36, repeated=True, variant=messages.Variant.INT32) - year_of_last_follow_up_lte = messages.IntegerField(37, variant=messages.Variant.INT32) - year_of_last_follow_up_gte = messages.IntegerField(38, variant=messages.Variant.INT32) - - -class ClinicalMetadataItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, variant=messages.Variant.INT32) - case_barcode = messages.StringField(2) - case_gdc_id = messages.StringField(3) - days_to_birth = messages.IntegerField(4, variant=messages.Variant.INT32) - days_to_death = messages.IntegerField(5, variant=messages.Variant.INT32) - days_to_last_followup = messages.IntegerField(6, variant=messages.Variant.INT32) - days_to_last_known_alive = messages.IntegerField(7, variant=messages.Variant.INT32) - ethnicity = messages.StringField(8) - event_free_survival = messages.IntegerField(9, variant=messages.Variant.INT32) - first_event = messages.StringField(10) - gender = messages.StringField(11) - protocol = messages.StringField(12) - race = messages.StringField(13) - summary_file_count = messages.IntegerField(14, variant=messages.Variant.INT32) - vital_status = messages.StringField(15) - wbc_at_diagnosis = messages.FloatField(16) - year_of_diagnosis = messages.IntegerField(17, variant=messages.Variant.INT32) - year_of_last_follow_up = messages.IntegerField(18, variant=messages.Variant.INT32) - -class BiospecimenMetadataRangesItem(messages.Message): - case_barcode = messages.StringField(1, repeated=True) - case_gdc_id = messages.StringField(2, repeated=True) - sample_barcode = messages.StringField(3, repeated=True) - sample_gdc_id = messages.StringField(4, repeated=True) - sample_type = messages.StringField(5, repeated=True) - tumor_code = messages.StringField(6, repeated=True) - -class BiospecimenMetadataItem(messages.Message): - case_barcode = messages.StringField(1) - case_gdc_id = messages.StringField(2) - sample_barcode = messages.StringField(3) - sample_gdc_id = messages.StringField(4) - sample_type = messages.StringField(5) - tumor_code = messages.StringField(6) - -class Data_HG19MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - case_barcode = messages.StringField(5, repeated=True) - case_gdc_id = messages.StringField(6, repeated=True) - data_category = messages.StringField(10, repeated=True) - data_format = messages.StringField(11, repeated=True) - data_type = messages.StringField(12, repeated=True) - experimental_strategy = messages.StringField(13, repeated=True) - file_name_key = messages.StringField(14, repeated=True) - index_file_name_key = messages.StringField(17, repeated=True) - platform = messages.StringField(18, repeated=True) - sample_barcode = messages.StringField(19, repeated=True) - sample_gdc_id = messages.StringField(20, repeated=True) - -class Data_HG19MetadataItem(messages.Message): - access = messages.StringField(1) - case_barcode = messages.StringField(5) - case_gdc_id = messages.StringField(6) - data_category = messages.StringField(10) - data_format = messages.StringField(11) - data_type = messages.StringField(12) - experimental_strategy = messages.StringField(13) - file_name_key = messages.StringField(14) - index_file_name_key = messages.StringField(17) - platform = messages.StringField(18) - sample_barcode = messages.StringField(19) - sample_gdc_id = messages.StringField(20) - -class Data_HG38MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - case_barcode = messages.StringField(5, repeated=True) - case_gdc_id = messages.StringField(6, repeated=True) - data_category = messages.StringField(10, repeated=True) - data_format = messages.StringField(11, repeated=True) - data_type = messages.StringField(12, repeated=True) - experimental_strategy = messages.StringField(13, repeated=True) - file_name_key = messages.StringField(14, repeated=True) - index_file_name_key = messages.StringField(17, repeated=True) - platform = messages.StringField(18, repeated=True) - sample_barcode = messages.StringField(19, repeated=True) - sample_gdc_id = messages.StringField(20, repeated=True) - -class Data_HG38MetadataItem(messages.Message): - access = messages.StringField(1) - case_barcode = messages.StringField(5) - case_gdc_id = messages.StringField(6) - data_category = messages.StringField(10) - data_format = messages.StringField(11) - data_type = messages.StringField(12) - experimental_strategy = messages.StringField(13) - file_name_key = messages.StringField(14) - index_file_name_key = messages.StringField(17) - platform = messages.StringField(18) - sample_barcode = messages.StringField(19) - sample_gdc_id = messages.StringField(20) - -class MetadataRangesItem(messages.Message): - Common = messages.MessageField(CommonMetadataRangesItem, 1) - Clinical = messages.MessageField(ClinicalMetadataRangesItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataRangesItem, 3) - data_HG19_r14 = messages.MessageField(Data_HG19MetadataRangesItem, 4) - data_HG38_r14 = messages.MessageField(Data_HG38MetadataRangesItem, 5) - -class MetadataItem(messages.Message): - Common = messages.MessageField(CommonMetadataItem, 1) - Clinical = messages.MessageField(ClinicalMetadataItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataItem, 3) - data_HG19_r14 = messages.MessageField(Data_HG19MetadataItem, 4) - data_HG38_r14 = messages.MessageField(Data_HG38MetadataItem, 5) - diff --git a/api_3/isb_cgc_api_TARGET/patients_get.py b/api_3/isb_cgc_api_TARGET/patients_get.py deleted file mode 100755 index 7e40dbfb..00000000 --- a/api_3/isb_cgc_api_TARGET/patients_get.py +++ /dev/null @@ -1,55 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.patients_get_helper import CasesGetHelper -from api_3.isb_cgc_api_TARGET.message_classes import ClinicalMetadataItem as MetadataItem -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints - -class CaseDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4) - -class CaseSetDetails(messages.Message): - cases = messages.MessageField(CaseDetails, 1, repeated=True) - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='cases') -class TARGETCasesGetAPI(CasesGetHelper): - @endpoints.method(CasesGetHelper.GET_RESOURCE, CaseDetails, path='target/cases/{case_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case barcode (of length 16, *eg* TARGET-51-PALFYG) as a required parameter. - User does not need to be authenticated. - """ - return super(TARGETCasesGetAPI, self).get(request, CaseDetails, MetadataItem, 'TARGET') - - @endpoints.method(CasesGetHelper.POST_RESOURCE, CaseSetDetails, path='target/cases', http_method='POST') - def get_list(self, request): - """ - Given a list of case barcodes (of length 11, *eg* TARGET-51-PALFYG), this endpoint returns - all available "biospecimen" information about them, including a list of samples and aliquots - derived from them. - Takes a list of case barcodes (of length 12, *eg* TARGET-51-PALFYG) as a required data payload. - User does not need to be authenticated. - """ - return super(TARGETCasesGetAPI, self).get_list(request, CaseSetDetails, CaseDetails, MetadataItem, 'TARGET') diff --git a/api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py b/api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py deleted file mode 100755 index 5612988e..00000000 --- a/api_3/isb_cgc_api_TARGET/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, SamplesCloudStorageFilePathsHelper - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='samples') -class TARGETSamplesCloudStorageFilePathsAPI(SamplesCloudStorageFilePathsHelper): - - @endpoints.method(SamplesCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, - path='target/samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - return super(TARGETSamplesCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request, 'TARGET') diff --git a/api_3/isb_cgc_api_TARGET/samples_get.py b/api_3/isb_cgc_api_TARGET/samples_get.py deleted file mode 100755 index d31b2fa2..00000000 --- a/api_3/isb_cgc_api_TARGET/samples_get.py +++ /dev/null @@ -1,60 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.isb_cgc_api_TARGET.message_classes import BiospecimenMetadataItem -from api_3.samples_get_helper import SamplesGetAPI, DataDetails - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(BiospecimenMetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - case_barcode = messages.StringField(3) - case_gdc_id = messages.StringField(4) - data_details = messages.MessageField(DataDetails, 5, repeated=True) - data_details_count = messages.IntegerField(6, variant=messages.Variant.INT32) - sample_barcode = messages.StringField(7) - - -class SampleSetDetails(messages.Message): - samples = messages.MessageField(SampleDetails, 1, repeated=True) - - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='samples') -class TARGETSamplesGetAPI(SamplesGetAPI): - @endpoints.method(SamplesGetAPI.GET_RESOURCE, SampleDetails, path='target/samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (of length 20-22, *eg* TARGET-51-PALFYG-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TARGETSamplesGetAPI, self).get(request, 'TARGET', SampleDetails, BiospecimenMetadataItem) - - @endpoints.method(SamplesGetAPI.POST_RESOURCE, SampleSetDetails, path='target/samples', http_method='POST') - def get_list(self, request): - """ - Given a list of sample barcodes (of length 16, *eg* TARGET-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TARGETSamplesGetAPI, self).get_list(request, 'TARGET', SampleSetDetails, SampleDetails, BiospecimenMetadataItem) diff --git a/api_3/isb_cgc_api_TARGET/samples_googlegenomics.py b/api_3/isb_cgc_api_TARGET/samples_googlegenomics.py deleted file mode 100755 index b31fcc8d..00000000 --- a/api_3/isb_cgc_api_TARGET/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_TARGET_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api_3/isb_cgc_api_TARGET/users_get.py b/api_3/isb_cgc_api_TARGET/users_get.py deleted file mode 100755 index 767066e1..00000000 --- a/api_3/isb_cgc_api_TARGET/users_get.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import message_types - -from api_3.users_get_common import UserGetAPICommon, UserGetAPIReturnJSON -from api_3.isb_cgc_api_TARGET.isb_cgc_api_helpers import ISB_CGC_TARGET_Endpoints - -@ISB_CGC_TARGET_Endpoints.api_class(resource_name='users') -class TARGETUserGetAPI(UserGetAPICommon): - - @endpoints.method(message_types.VoidMessage, UserGetAPIReturnJSON, http_method='GET', path='target/users') - def get(self, _): - ''' - Returns the dbGaP authorization status of the user. - ''' - return super(TARGETUserGetAPI, self).get('TARGET') diff --git a/api_3/isb_cgc_api_TCGA/__init__.py b/api_3/isb_cgc_api_TCGA/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json b/api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json deleted file mode 100755 index 39bb1364..00000000 --- a/api_3/isb_cgc_api_TCGA/allowed_values_v3_TCGA.json +++ /dev/null @@ -1,780 +0,0 @@ -{ - "Common": { - "disease_code": [ - "ACC", - "BLCA", - "BRCA", - "CESC", - "CHOL", - "COAD", - "DLBC", - "ESCA", - "GBM", - "HNSC", - "KICH", - "KIRC", - "KIRP", - "LAML", - "LGG", - "LIHC", - "LUAD", - "LUSC", - "MESO", - "OV", - "PAAD", - "PCPG", - "PRAD", - "READ", - "SARC", - "SKCM", - "STAD", - "TGCT", - "THCA", - "THYM", - "UCEC", - "UCS", - "UVM" - ], - "endpoint_type": [ - "current", - "legacy" - ], - "program_name": [ - "TCGA" - ], - "project_short_name": [ - "TCGA-ACC", - "TCGA-BLCA", - "TCGA-BRCA", - "TCGA-CESC", - "TCGA-CHOL", - "TCGA-COAD", - "TCGA-DLBC", - "TCGA-ESCA", - "TCGA-GBM", - "TCGA-HNSC", - "TCGA-KICH", - "TCGA-KIRC", - "TCGA-KIRP", - "TCGA-LAML", - "TCGA-LGG", - "TCGA-LIHC", - "TCGA-LUAD", - "TCGA-LUSC", - "TCGA-MESO", - "TCGA-OV", - "TCGA-PAAD", - "TCGA-PCPG", - "TCGA-PRAD", - "TCGA-READ", - "TCGA-SARC", - "TCGA-SKCM", - "TCGA-STAD", - "TCGA-TGCT", - "TCGA-THCA", - "TCGA-THYM", - "TCGA-UCEC", - "TCGA-UCS", - "TCGA-UVM" - ] - }, - "Clinical": { - "anatomic_neoplasm_subdivision": [ - "Alveolar Ridge", - "Antrum/Distal", - "Ascending Colon", - "Base of tongue", - "Bilateral", - "Bladder - NOS", - "Body of Pancreas", - "Bronchial", - "Buccal Mucosa", - "Cardia/Proximal", - "Cecum", - "Descending Colon", - "Dome", - "Endometrium", - "Floor of mouth", - "Fundus uteri", - "Fundus/Body", - "Gastroesophageal Junction", - "Hard Palate", - "Head of Pancreas", - "Hepatic Flexure", - "Hypopharynx", - "L-Lower", - "L-Upper", - "Larynx", - "Left", - "Left Lower Inner Quadrant", - "Left Lower Outer Quadrant", - "Left Upper Inner Quadrant", - "Left Upper Outer Quadrant", - "Lip", - "Lower uterine segment/Isthmus uteri", - "Myometrium", - "Neck", - "Oral Cavity", - "Oral Tongue", - "Oropharynx", - "Other (please specify)", - "R-Lower", - "R-Middle", - "R-Upper", - "Rectosigmoid Junction", - "Rectum", - "Right", - "Right Lower Inner Quadrant", - "Right Lower Outer Quadrant", - "Right Upper Inner Quadrant", - "Right Upper Outer Quadrant", - "Sigmoid Colon", - "Splenic Flexure", - "Stomach (NOS)", - "Tail of Pancreas", - "Tonsil", - "Transverse Colon", - "Trigone", - "Unknown - Uterus NOS", - "Wall Anterior", - "Wall Lateral", - "Wall NOS", - "Wall Posterior" - ], - "bcr": [ - "Nationwide Children's Hospital", - "Washington University" - ], - "clinical_M": [ - "M0", - "M1", - "M1a", - "M1b", - "M1c", - "MX" - ], - "clinical_N": [ - "N0", - "N1", - "N2", - "N2a", - "N2b", - "N2c", - "N3", - "NX" - ], - "clinical_stage": [ - "Stage I", - "Stage IA", - "Stage IA1", - "Stage IA2", - "Stage IB", - "Stage IB1", - "Stage IB2", - "Stage IC", - "Stage II", - "Stage IIA", - "Stage IIA1", - "Stage IIA2", - "Stage IIB", - "Stage IIC", - "Stage III", - "Stage IIIA", - "Stage IIIB", - "Stage IIIC", - "Stage IIIC1", - "Stage IIIC2", - "Stage IS", - "Stage IV", - "Stage IVA", - "Stage IVB", - "Stage IVC" - ], - "clinical_T": [ - "T1", - "T1a", - "T1b", - "T1c", - "T2", - "T2a", - "T2b", - "T2c", - "T3", - "T3a", - "T3b", - "T4", - "T4a", - "T4b", - "T4c", - "T4d", - "T4e", - "TX" - ], - "colorectal_cancer": [ - "NO", - "YES" - ], - "country": [ - "Afghanistan", - "Algeria", - "American Samoa", - "Australia", - "Brazil", - "Bulgaria", - "Canada", - "Croatia", - "Czech Republic", - "France", - "Georgia", - "Germany", - "Hamburg/Germany", - "Israel", - "Italy", - "Korea", - "Korea South", - "Moldova", - "Netherlands", - "Nigeria", - "Ontario Canada", - "Ontario/Canada", - "Pakistan", - "Poland", - "Puerto Rico", - "Republic of Moldova", - "Romania", - "Russia", - "Sao Paulo", - "Singapore", - "Spain", - "Switzerland", - "Ukraine", - "United Kingdom", - "United States", - "Vietnam", - "Yemen" - ], - "ethnicity": [ - "HISPANIC OR LATINO", - "NOT HISPANIC OR LATINO" - ], - "gender": [ - "FEMALE", - "MALE" - ], - "history_of_colon_polyps": [ - "NO", - "YES" - ], - "history_of_neoadjuvant_treatment": [ - "No", - "Yes", - "Yes, Pharmaceutical Treatment Prior to Resection", - "Yes, Radiation Prior to Resection" - ], - "hpv_calls": [ - "HPV16", - "HPV16;HPV18", - "HPV16;HPV18;HPV58", - "HPV16;HPV31", - "HPV16;HPV33", - "HPV16;HPV35", - "HPV16;HPV39", - "HPV16;HPV52", - "HPV16;HPV66", - "HPV18", - "HPV18;HPV31", - "HPV31", - "HPV33", - "HPV35", - "HPV39", - "HPV45", - "HPV51", - "HPV52", - "HPV56", - "HPV58", - "HPV59", - "HPV68", - "HPV73" - ], - "hpv_status": [ - "Indeterminate", - "Negative", - "Positive" - ], - "h_pylori_infection": [ - "Current", - "Never", - "No", - "Yes" - ], - "lymphatic_invasion": [ - "NO", - "YES" - ], - "lymphnodes_examined": [ - "NO", - "YES" - ], - "lymphovascular_invasion_present": [ - "NO", - "YES" - ], - "menopause_status": [ - "Indeterminate (neither Pre or Postmenopausal)", - "Peri (6-12 months since last menstrual period)", - "Post (prior bilateral ovariectomy OR >12 mo since LMP with no prior hysterectomy)", - "Pre (<6 months since LMP AND no prior bilateral ovariectomy AND not on estrogen replacement)" - ], - "mononucleotide_and_dinucleotide_marker_panel_analysis_status": [ - "Indeterminate", - "MSI-H", - "MSI-L", - "MSS", - "Not Tested" - ], - "neoplasm_histologic_grade": [ - "G1", - "G2", - "G3", - "G4", - "GB", - "GX", - "High Grade", - "Low Grade" - ], - "new_tumor_event_after_initial_treatment": [ - "NO", - "YES" - ], - "other_dx": [ - "Both History of Synchronous/ Bilateral and Prior Malignancy", - "No", - "Yes, History of Prior Malignancy", - "Yes, History of Synchronous/Bilateral Malignancy" - ], - "other_malignancy_histological_type": [ - "Adenocarcinoma, Not Otherwise Specified", - "Adenocarcinoma, Not Otherwise Specified, Adenocarcinoma, Not Otherwise Specified", - "Adenocarcinoma, Not Otherwise Specified, Colon Adenocarcinoma", - "Adenocarcinoma, Not Otherwise Specified, Kidney Clear Cell Renal Carcinoma", - "Adenocarcinoma, Not Otherwise Specified, Lung Acinar Adenocarcinoma", - "Adenocarcinoma, Not Otherwise Specified, Other, specify", - "Adenocarcinoma, Not Otherwise Specified, Other, specify, Other, specify", - "Adenocarcinoma, Not Otherwise Specified, Squamous Cell Carcinoma, Not Otherwise Specified", - "Adenosquamous", - "Astrocytoma", - "Basaloid Squamous Cell", - "Basaloid Squamous Cell, Adenocarcinoma, Not Otherwise Specified", - "Clear Cell Adenocarcinoma", - "Clear Cell Squamous Cell", - "Colon Adenocarcinoma", - "Colon Adenocarcinoma, Colon Adenocarcinoma", - "Colon Mucinous Adenocarcinoma", - "Endometrioid endometrial adenocarcinoma (Grade 1 or 2)", - "Endometrioid endometrial adenocarcinoma (Grade 3)", - "Head & Neck Squamous Cell Carcinoma", - "Hepatocellular Carcinoma", - "Kidney Clear Cell Renal Carcinoma", - "Kidney Clear Cell Renal Carcinoma, Kidney Clear Cell Renal Carcinoma", - "Kidney Clear Cell Renal Carcinoma, Kidney Clear Cell Renal Carcinoma, Other, specify", - "Kidney Clear Cell Renal Carcinoma, Kidney Papillary Renal Cell Carcinoma", - "Kidney Clear Cell Renal Carcinoma, Other, specify", - "Kidney Papillary Renal Cell Carcinoma", - "Kidney Papillary Renal Cell Carcinoma, Kidney Papillary Renal Cell Carcinoma", - "Kidney Papillary Renal Cell Carcinoma, Kidney Papillary Renal Cell Carcinoma, Adenocarcinoma, Not Otherwise Specified", - "Lung Adenocarcinoma Mixed Subtype", - "Lung Adenocarcinoma- Not Otherwise Specified (NOS)", - "Lung Adenocarcinoma- Not Otherwise Specified (NOS), Adenocarcinoma, Not Otherwise Specified", - "Lung Bronchioloalveolar Carcinoma Nonmucinous", - "Lung Clear Cell Squamous Cell Carcinoma", - "Lung Clear Cell Squamous Cell Carcinoma, Other, specify", - "Lung Papillary Adenocarcinoma", - "Lung Small Cell Squamous Cell Carcinoma", - "Other, specify", - "Other, specify, Adenocarcinoma, Not Otherwise Specified", - "Other, specify, Adenocarcinoma, Not Otherwise Specified, Other, specify", - "Other, specify, Basaloid Squamous Cell", - "Other, specify, Clear Cell Adenocarcinoma", - "Other, specify, Kidney Papillary Renal Cell Carcinoma", - "Other, specify, Kidney Papillary Renal Cell Carcinoma, Kidney Papillary Renal Cell Carcinoma", - "Other, specify, Lung Mucinous Adenocarcinoma", - "Other, specify, Other, specify", - "Other, specify, Other, specify, Kidney Papillary Renal Cell Carcinoma", - "Other, specify, Other, specify, Other, specify", - "Other, specify, Other, specify, Other, specify, Other, specify", - "Other, specify, Other, specify, Squamous Cell Carcinoma, Not Otherwise Specified", - "Other, specify, Squamous Cell Carcinoma, Not Otherwise Specified", - "Papillary Squamous Cell", - "Rectal Adenocarcinoma", - "Small Cell Squamous Cell", - "Squamous Cell Carcinoma, Not Otherwise Specified", - "Squamous Cell Carcinoma, Not Otherwise Specified, Basaloid Squamous Cell", - "Squamous Cell Carcinoma, Not Otherwise Specified, Kidney Papillary Renal Cell Carcinoma", - "Squamous Cell Carcinoma, Not Otherwise Specified, Lung Adenocarcinoma Mixed Subtype", - "Squamous Cell Carcinoma, Not Otherwise Specified, Other, specify", - "Squamous Cell Carcinoma, Not Otherwise Specified, Other, specify, Other, specify", - "Squamous Cell Carcinoma, Not Otherwise Specified, Squamous Cell Carcinoma, Not Otherwise Specified", - "Squamous Cell Carcinoma, Not Otherwise Specified, Squamous Cell Carcinoma, Not Otherwise Specified, Basaloid Squamous Cell", - "Thyroid Papillary Carcinoma - Classical/usual", - "Thyroid Papillary Carcinoma - Classical/usual, Adenocarcinoma, Not Otherwise Specified", - "Thyroid Papillary Carcinoma - Follicular (>= 99% follicular patterned)", - "Thyroid Papillary Carcinoma - Other, specify", - "Thyroid Papillary Carcinoma - Other, specify, Thyroid Papillary Carcinoma - Other, specify, Other, specify", - "Uterine serous endometrial adenocarcinoma" - ], - "other_malignancy_type": [ - "Prior Malignancy", - "Prior Malignancy, Prior Malignancy", - "Prior Malignancy, Prior Malignancy, Prior Malignancy", - "Prior Malignancy, Prior Malignancy, Prior Malignancy, Synchronous Malignancy", - "Prior Malignancy, Prior Malignancy, Synchronous Malignancy", - "Prior Malignancy, Synchronous Malignancy", - "Prior Malignancy, Synchronous Malignancy, Prior Malignancy", - "Synchronous Malignancy", - "Synchronous Malignancy, Prior Malignancy", - "Synchronous Malignancy, Prior Malignancy, Prior Malignancy, Prior Malignancy", - "Synchronous Malignancy, Prior Malignancy, Synchronous Malignancy", - "Synchronous Malignancy, Synchronous Malignancy", - "Synchronous Malignancy, Synchronous Malignancy, Prior Malignancy" - ], - "pathologic_M": [ - "cM0 (i+)", - "M0", - "M1", - "M1a", - "M1b", - "M1c", - "MX" - ], - "pathologic_N": [ - "N0", - "N0 (i+)", - "N0 (i-)", - "N0 (mol+)", - "N1", - "N1a", - "N1b", - "N1c", - "N1mi", - "N2", - "N2a", - "N2b", - "N2c", - "N3", - "N3a", - "N3b", - "N3c", - "NX" - ], - "pathologic_stage": [ - "I/II NOS", - "IS", - "Stage 0", - "Stage I", - "Stage IA", - "Stage IB", - "Stage II", - "Stage IIA", - "Stage IIB", - "Stage IIC", - "Stage III", - "Stage IIIA", - "Stage IIIB", - "Stage IIIC", - "Stage IV", - "Stage IVA", - "Stage IVB", - "Stage IVC", - "Stage X" - ], - "pathologic_T": [ - "T0", - "T1", - "T1a", - "T1a1", - "T1b", - "T1b1", - "T1b2", - "T1c", - "T2", - "T2a", - "T2a1", - "T2a2", - "T2b", - "T2c", - "T3", - "T3a", - "T3b", - "T3c", - "T4", - "T4a", - "T4b", - "T4c", - "T4d", - "T4e", - "Tis", - "TX" - ], - "person_neoplasm_cancer_status": [ - "TUMOR FREE", - "WITH TUMOR" - ], - "pregnancies": [ - "0", - "1", - "2", - "3", - "4+" - ], - "primary_neoplasm_melanoma_dx": [ - "NO", - "YES" - ], - "primary_therapy_outcome_success": [ - "Complete Remission/Response", - "No Measureable Tumor or Tumor Markers", - "Normalization of Tumor Markers, but Residual Tumor Mass", - "Partial Remission/Response", - "Persistent Disease", - "Progressive Disease", - "Stable Disease" - ], - "race": [ - "AMERICAN INDIAN OR ALASKA NATIVE", - "ASIAN", - "BLACK OR AFRICAN AMERICAN", - "NATIVE HAWAIIAN OR OTHER PACIFIC ISLANDER", - "WHITE" - ], - "residual_tumor": [ - "R0", - "R1", - "R2", - "RX" - ], - "tobacco_smoking_history": [ - "1", - "2", - "3", - "4", - "5" - ], - "tumor_type": [ - "Primary", - "Type 1", - "Type 2" - ], - "venous_invasion": [ - "NO", - "YES" - ], - "vital_status": [ - "Alive", - "Dead" - ] - }, - "Biospecimen": { - "bcr": [ - "Nationwide Children's Hospital", - "Washington University" - ], - "preservation_method": [ - "FFPE", - "frozen" - ], - "sample_type": [ - "01", - "02", - "03", - "05", - "06", - "07", - "10", - "11", - "12", - "14" - ] - }, - "Data_HG19": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - - ], - "center_code": [ - "01", - "02", - "05", - "06", - "07", - "08", - "09", - "10", - "12", - "13", - "21", - "25", - "26", - "31", - "32", - "34", - "36" - ], - "center_name": [ - "Baylor College of Medicine", - "Broad Institute of MIT and Harvard", - "Canada's Michael Smith Genome Sciences Centre", - "Harvard Medical School", - "HudsonAlpha Institute for Biotechnology", - "Johns Hopkins / University of Southern California", - "MD Anderson - Institute for Applied Cancer Science", - "Nationwide Children's Hospital BCR", - "University of California, Santa Cruz", - "University of North Carolina", - "Washington University School of Medicine", - "Wellcome Trust Sanger Institute" - ], - "center_type": [ - "BCR", - "CGCC", - "GSC" - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw sequencing data", - "Simple nucleotide variation" - ], - "data_format": [ - "BAM", - "BCR XML", - "dat", - "SVS", - "TXT", - "VCF", - "ZIP" - ], - "data_type": [ - "Aligned reads", - "Biospecimen Supplement", - "Clinical Supplement", - "Diagnostic image", - "Genotypes", - "Radiology image", - "Simple nucleotide variation", - "Tissue slide image" - ], - "experimental_strategy": [ - "AMPLICON", - "Bisulfite-Seq", - "DNA-Seq", - "Genotyping array", - "miRNA-Seq", - "RNA-Seq", - "VALIDATION", - "WGS", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "false", - "true" - ], - "platform": [ - "ABI SOLiD", - "Affymetrix SNP Array 6.0", - "Clinical", - "HiSeq X Ten", - "Illumina GA", - "Illumina HiSeq", - "Illumina Human 1M Duo", - "Illumina HumanHap550", - "Illumina MiSeq", - "Ion Torrent PGM", - "LS 454" - ], - "sample_type": [ - "01", - "02", - "03", - "05", - "06", - "07", - "10", - "11", - "12", - "14" - ], - "species": [ - "Homo sapien", - "Homo sapiens" - ] - }, - "Data_HG38": { - "access": [ - "controlled", - "open" - ], - "analysis_workflow_type": [ - "BWA with Mark Duplicates and Cocleaning", - "BWA-aln", - "STAR 2-Pass" - ], - "center_code": [ - - ], - "center_name": [ - - ], - "center_type": [ - - ], - "data_category": [ - "Biospecimen", - "Clinical", - "Raw Sequencing Data" - ], - "data_format": [ - "BAM", - "BCR XML" - ], - "data_type": [ - "Aligned Reads", - "Biospecimen Supplement", - "Clinical Supplement" - ], - "experimental_strategy": [ - "miRNA-Seq", - "RNA-Seq", - "WXS" - ], - "file_state": [ - "submitted" - ], - "file_uploaded": [ - "true" - ], - "platform": [ - "Illumina" - ], - "sample_type": [ - "01", - "02", - "03", - "05", - "06", - "07", - "10", - "11", - "12", - "14" - ], - "species": [ - "Homo sapiens" - ] - } -} diff --git a/api_3/isb_cgc_api_TCGA/cohorts_create.py b/api_3/isb_cgc_api_TCGA/cohorts_create.py deleted file mode 100644 index 3b8071dd..00000000 --- a/api_3/isb_cgc_api_TCGA/cohorts_create.py +++ /dev/null @@ -1,37 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsCreateHelper, CreatedCohort -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.isb_cgc_api_TCGA.message_classes import MetadataRangesItem - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cohorts') -class TCGACohortsCreateAPI(CohortsCreateHelper): - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, name=messages.StringField(2, required=True)) - - @endpoints.method(POST_RESOURCE, CreatedCohort, path='tcga/cohorts/create', http_method='POST') - def create(self, request): - """ - Creates and saves a cohort. Takes a JSON object in the request body to use as the cohort's filters. - Authentication is required. - Returns information about the saved cohort, including the number of cases and the number - of samples in that cohort. - """ - self.program = 'TCGA' - return super(TCGACohortsCreateAPI, self).create(request) diff --git a/api_3/isb_cgc_api_TCGA/cohorts_preview.py b/api_3/isb_cgc_api_TCGA/cohorts_preview.py deleted file mode 100644 index c1a936df..00000000 --- a/api_3/isb_cgc_api_TCGA/cohorts_preview.py +++ /dev/null @@ -1,38 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from protorpc import messages - -from api_3.cohort_create_preview_helper import CohortsPreviewHelper -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from message_classes import MetadataRangesItem - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cohorts') -class TCGACohortsPreviewAPI(CohortsPreviewHelper): - - POST_RESOURCE = endpoints.ResourceContainer(MetadataRangesItem, fields=messages.StringField(3)) - - @endpoints.method(POST_RESOURCE, CohortsPreviewHelper.CohortCasesSamplesList, path='tcga/cohorts/preview', http_method='POST') - def preview(self, request): - """ - Takes a JSON object of filters in the request body and returns a "preview" of the cohort that would - result from passing a similar request to the cohort **save** endpoint. This preview consists of - two lists: the lists of case barcodes, and the list of sample barcodes. - Authentication is not required. - """ - self.program = 'TCGA' - return super(TCGACohortsPreviewAPI, self).preview(request) diff --git a/api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py b/api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py deleted file mode 100644 index c9fc093e..00000000 --- a/api_3/isb_cgc_api_TCGA/isb_cgc_api_helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2017, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -""" -import endpoints -from django.conf import settings - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID - -ISB_CGC_TCGA_Endpoints = endpoints.api(name='isb_cgc_tcga_api', version='v3', - description="Get information about cohorts, cases, and samples for TCGA. Create cohorts.", - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID, - settings.WEB_CLIENT_ID], - package_path='tcga', - documentation='http://isb-cancer-genomics-cloud.readthedocs.io/en/latest/sections/progapi/Programmatic-API.html#isb-cgc-api-v3', - title="ISB-CGC TCGA API") diff --git a/api_3/isb_cgc_api_TCGA/message_classes.py b/api_3/isb_cgc_api_TCGA/message_classes.py deleted file mode 100644 index 258fee81..00000000 --- a/api_3/isb_cgc_api_TCGA/message_classes.py +++ /dev/null @@ -1,453 +0,0 @@ -from protorpc import messages - -class CommonMetadataRangesItem(messages.Message): - disease_code = messages.StringField(1, repeated=True) - endpoint_type = messages.StringField(2, repeated=True) - program_name = messages.StringField(3, repeated=True) - project_short_name = messages.StringField(4, repeated=True) - -class CommonMetadataItem(messages.Message): - disease_code = messages.StringField(1) - endpoint_type = messages.StringField(2) - program_name = messages.StringField(3) - project_short_name = messages.StringField(4) - -class ClinicalMetadataRangesItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, repeated=True, variant=messages.Variant.INT32) - age_at_diagnosis_lte = messages.IntegerField(2, variant=messages.Variant.INT32) - age_at_diagnosis_gte = messages.IntegerField(3, variant=messages.Variant.INT32) - - age_began_smoking_in_years = messages.IntegerField(4, repeated=True, variant=messages.Variant.INT32) - age_began_smoking_in_years_lte = messages.IntegerField(5, variant=messages.Variant.INT32) - age_began_smoking_in_years_gte = messages.IntegerField(6, variant=messages.Variant.INT32) - - anatomic_neoplasm_subdivision = messages.StringField(7, repeated=True) - batch_number = messages.IntegerField(8, repeated=True, variant=messages.Variant.INT32) - batch_number_lte = messages.IntegerField(9, variant=messages.Variant.INT32) - batch_number_gte = messages.IntegerField(10, variant=messages.Variant.INT32) - - bcr = messages.StringField(11, repeated=True) - bmi = messages.FloatField(12, repeated=True) - bmi_lte = messages.FloatField(13) - bmi_gte = messages.FloatField(14) - - case_barcode = messages.StringField(15, repeated=True) - case_gdc_id = messages.StringField(16, repeated=True) - clinical_M = messages.StringField(17, repeated=True) - clinical_N = messages.StringField(18, repeated=True) - clinical_stage = messages.StringField(19, repeated=True) - clinical_T = messages.StringField(20, repeated=True) - colorectal_cancer = messages.StringField(21, repeated=True) - country = messages.StringField(22, repeated=True) - days_to_birth = messages.IntegerField(23, repeated=True, variant=messages.Variant.INT32) - days_to_birth_lte = messages.IntegerField(24, variant=messages.Variant.INT32) - days_to_birth_gte = messages.IntegerField(25, variant=messages.Variant.INT32) - - days_to_death = messages.IntegerField(26, repeated=True, variant=messages.Variant.INT32) - days_to_death_lte = messages.IntegerField(27, variant=messages.Variant.INT32) - days_to_death_gte = messages.IntegerField(28, variant=messages.Variant.INT32) - - days_to_initial_pathologic_diagnosis = messages.IntegerField(29, repeated=True, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_lte = messages.IntegerField(30, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis_gte = messages.IntegerField(31, variant=messages.Variant.INT32) - - days_to_last_followup = messages.IntegerField(32, repeated=True, variant=messages.Variant.INT32) - days_to_last_followup_lte = messages.IntegerField(33, variant=messages.Variant.INT32) - days_to_last_followup_gte = messages.IntegerField(34, variant=messages.Variant.INT32) - - days_to_last_known_alive = messages.IntegerField(35, repeated=True, variant=messages.Variant.INT32) - days_to_last_known_alive_lte = messages.IntegerField(36, variant=messages.Variant.INT32) - days_to_last_known_alive_gte = messages.IntegerField(37, variant=messages.Variant.INT32) - - days_to_submitted_specimen_dx = messages.IntegerField(38, repeated=True, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_lte = messages.IntegerField(39, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx_gte = messages.IntegerField(40, variant=messages.Variant.INT32) - - ethnicity = messages.StringField(41, repeated=True) - gender = messages.StringField(42, repeated=True) - gleason_score_combined = messages.IntegerField(43, repeated=True, variant=messages.Variant.INT32) - gleason_score_combined_lte = messages.IntegerField(44, variant=messages.Variant.INT32) - gleason_score_combined_gte = messages.IntegerField(45, variant=messages.Variant.INT32) - - height = messages.IntegerField(46, repeated=True, variant=messages.Variant.INT32) - height_lte = messages.IntegerField(47, variant=messages.Variant.INT32) - height_gte = messages.IntegerField(48, variant=messages.Variant.INT32) - - histological_type = messages.StringField(49, repeated=True) - history_of_colon_polyps = messages.StringField(50, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(51, repeated=True) - hpv_calls = messages.StringField(52, repeated=True) - hpv_status = messages.StringField(53, repeated=True) - h_pylori_infection = messages.StringField(54, repeated=True) - icd_10 = messages.StringField(55, repeated=True) - icd_o_3_histology = messages.StringField(56, repeated=True) - icd_o_3_site = messages.StringField(57, repeated=True) - lymphatic_invasion = messages.StringField(58, repeated=True) - lymphnodes_examined = messages.StringField(59, repeated=True) - lymphovascular_invasion_present = messages.StringField(60, repeated=True) - menopause_status = messages.StringField(61, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(62, repeated=True) - neoplasm_histologic_grade = messages.StringField(63, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(64, repeated=True) - number_of_lymphnodes_examined = messages.IntegerField(65, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_lte = messages.IntegerField(66, variant=messages.Variant.INT32) - number_of_lymphnodes_examined_gte = messages.IntegerField(67, variant=messages.Variant.INT32) - - number_of_lymphnodes_positive_by_he = messages.IntegerField(68, repeated=True, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_lte = messages.IntegerField(69, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he_gte = messages.IntegerField(70, variant=messages.Variant.INT32) - - number_pack_years_smoked = messages.IntegerField(71, repeated=True, variant=messages.Variant.INT32) - number_pack_years_smoked_lte = messages.IntegerField(72, variant=messages.Variant.INT32) - number_pack_years_smoked_gte = messages.IntegerField(73, variant=messages.Variant.INT32) - - other_dx = messages.StringField(74, repeated=True) - other_malignancy_anatomic_site = messages.StringField(75, repeated=True) - other_malignancy_histological_type = messages.StringField(76, repeated=True) - other_malignancy_type = messages.StringField(77, repeated=True) - pathologic_M = messages.StringField(78, repeated=True) - pathologic_N = messages.StringField(79, repeated=True) - pathologic_stage = messages.StringField(80, repeated=True) - pathologic_T = messages.StringField(81, repeated=True) - person_neoplasm_cancer_status = messages.StringField(82, repeated=True) - pregnancies = messages.StringField(83, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(84, repeated=True) - primary_therapy_outcome_success = messages.StringField(85, repeated=True) - psa_value = messages.FloatField(86, repeated=True) - psa_value_lte = messages.FloatField(87) - psa_value_gte = messages.FloatField(88) - - race = messages.StringField(89, repeated=True) - residual_tumor = messages.StringField(90, repeated=True) - stopped_smoking_year = messages.IntegerField(91, repeated=True, variant=messages.Variant.INT32) - stopped_smoking_year_lte = messages.IntegerField(92, variant=messages.Variant.INT32) - stopped_smoking_year_gte = messages.IntegerField(93, variant=messages.Variant.INT32) - - summary_file_count = messages.IntegerField(94, repeated=True, variant=messages.Variant.INT32) - summary_file_count_lte = messages.IntegerField(95, variant=messages.Variant.INT32) - summary_file_count_gte = messages.IntegerField(96, variant=messages.Variant.INT32) - - tobacco_smoking_history = messages.StringField(97, repeated=True) - tss_code = messages.StringField(98, repeated=True) - tumor_tissue_site = messages.StringField(99, repeated=True) - tumor_type = messages.StringField(100, repeated=True) - venous_invasion = messages.StringField(101, repeated=True) - vital_status = messages.StringField(102, repeated=True) - weight = messages.IntegerField(103, repeated=True, variant=messages.Variant.INT32) - weight_lte = messages.IntegerField(104, variant=messages.Variant.INT32) - weight_gte = messages.IntegerField(105, variant=messages.Variant.INT32) - - year_of_diagnosis = messages.IntegerField(106, repeated=True, variant=messages.Variant.INT32) - year_of_diagnosis_lte = messages.IntegerField(107, variant=messages.Variant.INT32) - year_of_diagnosis_gte = messages.IntegerField(108, variant=messages.Variant.INT32) - - year_of_tobacco_smoking_onset = messages.IntegerField(109, repeated=True, variant=messages.Variant.INT32) - year_of_tobacco_smoking_onset_lte = messages.IntegerField(110, variant=messages.Variant.INT32) - year_of_tobacco_smoking_onset_gte = messages.IntegerField(111, variant=messages.Variant.INT32) - - -class ClinicalMetadataItem(messages.Message): - age_at_diagnosis = messages.IntegerField(1, variant=messages.Variant.INT32) - age_began_smoking_in_years = messages.IntegerField(2, variant=messages.Variant.INT32) - anatomic_neoplasm_subdivision = messages.StringField(3) - batch_number = messages.IntegerField(4, variant=messages.Variant.INT32) - bcr = messages.StringField(5) - bmi = messages.FloatField(6) - case_barcode = messages.StringField(7) - case_gdc_id = messages.StringField(8) - clinical_M = messages.StringField(9) - clinical_N = messages.StringField(10) - clinical_stage = messages.StringField(11) - clinical_T = messages.StringField(12) - colorectal_cancer = messages.StringField(13) - country = messages.StringField(14) - days_to_birth = messages.IntegerField(15, variant=messages.Variant.INT32) - days_to_death = messages.IntegerField(16, variant=messages.Variant.INT32) - days_to_initial_pathologic_diagnosis = messages.IntegerField(17, variant=messages.Variant.INT32) - days_to_last_followup = messages.IntegerField(18, variant=messages.Variant.INT32) - days_to_last_known_alive = messages.IntegerField(19, variant=messages.Variant.INT32) - days_to_submitted_specimen_dx = messages.IntegerField(20, variant=messages.Variant.INT32) - ethnicity = messages.StringField(21) - gender = messages.StringField(22) - gleason_score_combined = messages.IntegerField(23, variant=messages.Variant.INT32) - height = messages.IntegerField(24, variant=messages.Variant.INT32) - histological_type = messages.StringField(25) - history_of_colon_polyps = messages.StringField(26) - history_of_neoadjuvant_treatment = messages.StringField(27) - hpv_calls = messages.StringField(28) - hpv_status = messages.StringField(29) - h_pylori_infection = messages.StringField(30) - icd_10 = messages.StringField(31) - icd_o_3_histology = messages.StringField(32) - icd_o_3_site = messages.StringField(33) - lymphatic_invasion = messages.StringField(34) - lymphnodes_examined = messages.StringField(35) - lymphovascular_invasion_present = messages.StringField(36) - menopause_status = messages.StringField(37) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(38) - neoplasm_histologic_grade = messages.StringField(39) - new_tumor_event_after_initial_treatment = messages.StringField(40) - number_of_lymphnodes_examined = messages.IntegerField(41, variant=messages.Variant.INT32) - number_of_lymphnodes_positive_by_he = messages.IntegerField(42, variant=messages.Variant.INT32) - number_pack_years_smoked = messages.IntegerField(43, variant=messages.Variant.INT32) - other_dx = messages.StringField(44) - other_malignancy_anatomic_site = messages.StringField(45) - other_malignancy_histological_type = messages.StringField(46) - other_malignancy_type = messages.StringField(47) - pathologic_M = messages.StringField(48) - pathologic_N = messages.StringField(49) - pathologic_stage = messages.StringField(50) - pathologic_T = messages.StringField(51) - person_neoplasm_cancer_status = messages.StringField(52) - pregnancies = messages.StringField(53) - primary_neoplasm_melanoma_dx = messages.StringField(54) - primary_therapy_outcome_success = messages.StringField(55) - psa_value = messages.FloatField(56) - race = messages.StringField(57) - residual_tumor = messages.StringField(58) - stopped_smoking_year = messages.IntegerField(59, variant=messages.Variant.INT32) - summary_file_count = messages.IntegerField(60, variant=messages.Variant.INT32) - tobacco_smoking_history = messages.StringField(61) - tss_code = messages.StringField(62) - tumor_tissue_site = messages.StringField(63) - tumor_type = messages.StringField(64) - venous_invasion = messages.StringField(65) - vital_status = messages.StringField(66) - weight = messages.IntegerField(67, variant=messages.Variant.INT32) - year_of_diagnosis = messages.IntegerField(68, variant=messages.Variant.INT32) - year_of_tobacco_smoking_onset = messages.IntegerField(69, variant=messages.Variant.INT32) - -class BiospecimenMetadataRangesItem(messages.Message): - avg_percent_lymphocyte_infiltration = messages.FloatField(1, repeated=True) - avg_percent_lymphocyte_infiltration_lte = messages.FloatField(2) - avg_percent_lymphocyte_infiltration_gte = messages.FloatField(3) - - avg_percent_monocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_monocyte_infiltration_lte = messages.FloatField(5) - avg_percent_monocyte_infiltration_gte = messages.FloatField(6) - - avg_percent_necrosis = messages.FloatField(7, repeated=True) - avg_percent_necrosis_lte = messages.FloatField(8) - avg_percent_necrosis_gte = messages.FloatField(9) - - avg_percent_neutrophil_infiltration = messages.FloatField(10, repeated=True) - avg_percent_neutrophil_infiltration_lte = messages.FloatField(11) - avg_percent_neutrophil_infiltration_gte = messages.FloatField(12) - - avg_percent_normal_cells = messages.FloatField(13, repeated=True) - avg_percent_normal_cells_lte = messages.FloatField(14) - avg_percent_normal_cells_gte = messages.FloatField(15) - - avg_percent_stromal_cells = messages.FloatField(16, repeated=True) - avg_percent_stromal_cells_lte = messages.FloatField(17) - avg_percent_stromal_cells_gte = messages.FloatField(18) - - avg_percent_tumor_cells = messages.FloatField(19, repeated=True) - avg_percent_tumor_cells_lte = messages.FloatField(20) - avg_percent_tumor_cells_gte = messages.FloatField(21) - - avg_percent_tumor_nuclei = messages.FloatField(22, repeated=True) - avg_percent_tumor_nuclei_lte = messages.FloatField(23) - avg_percent_tumor_nuclei_gte = messages.FloatField(24) - - batch_number = messages.IntegerField(25, repeated=True, variant=messages.Variant.INT32) - batch_number_lte = messages.IntegerField(26, variant=messages.Variant.INT32) - batch_number_gte = messages.IntegerField(27, variant=messages.Variant.INT32) - - bcr = messages.StringField(28, repeated=True) - case_barcode = messages.StringField(29, repeated=True) - case_gdc_id = messages.StringField(30, repeated=True) - days_to_collection = messages.IntegerField(31, repeated=True, variant=messages.Variant.INT32) - days_to_collection_lte = messages.IntegerField(32, variant=messages.Variant.INT32) - days_to_collection_gte = messages.IntegerField(33, variant=messages.Variant.INT32) - - days_to_sample_procurement = messages.IntegerField(34, repeated=True, variant=messages.Variant.INT32) - days_to_sample_procurement_lte = messages.IntegerField(35, variant=messages.Variant.INT32) - days_to_sample_procurement_gte = messages.IntegerField(36, variant=messages.Variant.INT32) - - max_percent_lymphocyte_infiltration = messages.FloatField(37, repeated=True) - max_percent_lymphocyte_infiltration_lte = messages.FloatField(38) - max_percent_lymphocyte_infiltration_gte = messages.FloatField(39) - - max_percent_monocyte_infiltration = messages.FloatField(40, repeated=True) - max_percent_monocyte_infiltration_lte = messages.FloatField(41) - max_percent_monocyte_infiltration_gte = messages.FloatField(42) - - max_percent_necrosis = messages.FloatField(43, repeated=True) - max_percent_necrosis_lte = messages.FloatField(44) - max_percent_necrosis_gte = messages.FloatField(45) - - max_percent_neutrophil_infiltration = messages.FloatField(46, repeated=True) - max_percent_neutrophil_infiltration_lte = messages.FloatField(47) - max_percent_neutrophil_infiltration_gte = messages.FloatField(48) - - max_percent_normal_cells = messages.FloatField(49, repeated=True) - max_percent_normal_cells_lte = messages.FloatField(50) - max_percent_normal_cells_gte = messages.FloatField(51) - - max_percent_stromal_cells = messages.FloatField(52, repeated=True) - max_percent_stromal_cells_lte = messages.FloatField(53) - max_percent_stromal_cells_gte = messages.FloatField(54) - - max_percent_tumor_cells = messages.FloatField(55, repeated=True) - max_percent_tumor_cells_lte = messages.FloatField(56) - max_percent_tumor_cells_gte = messages.FloatField(57) - - max_percent_tumor_nuclei = messages.FloatField(58, repeated=True) - max_percent_tumor_nuclei_lte = messages.FloatField(59) - max_percent_tumor_nuclei_gte = messages.FloatField(60) - - min_percent_lymphocyte_infiltration = messages.FloatField(61, repeated=True) - min_percent_lymphocyte_infiltration_lte = messages.FloatField(62) - min_percent_lymphocyte_infiltration_gte = messages.FloatField(63) - - min_percent_monocyte_infiltration = messages.FloatField(64, repeated=True) - min_percent_monocyte_infiltration_lte = messages.FloatField(65) - min_percent_monocyte_infiltration_gte = messages.FloatField(66) - - min_percent_necrosis = messages.FloatField(67, repeated=True) - min_percent_necrosis_lte = messages.FloatField(68) - min_percent_necrosis_gte = messages.FloatField(69) - - min_percent_neutrophil_infiltration = messages.FloatField(70, repeated=True) - min_percent_neutrophil_infiltration_lte = messages.FloatField(71) - min_percent_neutrophil_infiltration_gte = messages.FloatField(72) - - min_percent_normal_cells = messages.FloatField(73, repeated=True) - min_percent_normal_cells_lte = messages.FloatField(74) - min_percent_normal_cells_gte = messages.FloatField(75) - - min_percent_stromal_cells = messages.FloatField(76, repeated=True) - min_percent_stromal_cells_lte = messages.FloatField(77) - min_percent_stromal_cells_gte = messages.FloatField(78) - - min_percent_tumor_cells = messages.FloatField(79, repeated=True) - min_percent_tumor_cells_lte = messages.FloatField(80) - min_percent_tumor_cells_gte = messages.FloatField(81) - - min_percent_tumor_nuclei = messages.FloatField(82, repeated=True) - min_percent_tumor_nuclei_lte = messages.FloatField(83) - min_percent_tumor_nuclei_gte = messages.FloatField(84) - - num_portions = messages.IntegerField(85, repeated=True, variant=messages.Variant.INT32) - num_portions_lte = messages.IntegerField(86, variant=messages.Variant.INT32) - num_portions_gte = messages.IntegerField(87, variant=messages.Variant.INT32) - - num_slides = messages.IntegerField(88, repeated=True, variant=messages.Variant.INT32) - num_slides_lte = messages.IntegerField(89, variant=messages.Variant.INT32) - num_slides_gte = messages.IntegerField(90, variant=messages.Variant.INT32) - - pathology_report_uuid = messages.StringField(91, repeated=True) - preservation_method = messages.StringField(92, repeated=True) - sample_barcode = messages.StringField(93, repeated=True) - sample_gdc_id = messages.StringField(94, repeated=True) - sample_type = messages.StringField(95, repeated=True) - -class BiospecimenMetadataItem(messages.Message): - avg_percent_lymphocyte_infiltration = messages.FloatField(1) - avg_percent_monocyte_infiltration = messages.FloatField(2) - avg_percent_necrosis = messages.FloatField(3) - avg_percent_neutrophil_infiltration = messages.FloatField(4) - avg_percent_normal_cells = messages.FloatField(5) - avg_percent_stromal_cells = messages.FloatField(6) - avg_percent_tumor_cells = messages.FloatField(7) - avg_percent_tumor_nuclei = messages.FloatField(8) - batch_number = messages.IntegerField(9, variant=messages.Variant.INT32) - bcr = messages.StringField(10) - case_barcode = messages.StringField(11) - case_gdc_id = messages.StringField(12) - days_to_collection = messages.IntegerField(13, variant=messages.Variant.INT32) - days_to_sample_procurement = messages.IntegerField(14, variant=messages.Variant.INT32) - max_percent_lymphocyte_infiltration = messages.FloatField(15) - max_percent_monocyte_infiltration = messages.FloatField(16) - max_percent_necrosis = messages.FloatField(17) - max_percent_neutrophil_infiltration = messages.FloatField(18) - max_percent_normal_cells = messages.FloatField(19) - max_percent_stromal_cells = messages.FloatField(20) - max_percent_tumor_cells = messages.FloatField(21) - max_percent_tumor_nuclei = messages.FloatField(22) - min_percent_lymphocyte_infiltration = messages.FloatField(23) - min_percent_monocyte_infiltration = messages.FloatField(24) - min_percent_necrosis = messages.FloatField(25) - min_percent_neutrophil_infiltration = messages.FloatField(26) - min_percent_normal_cells = messages.FloatField(27) - min_percent_stromal_cells = messages.FloatField(28) - min_percent_tumor_cells = messages.FloatField(29) - min_percent_tumor_nuclei = messages.FloatField(30) - num_portions = messages.IntegerField(31, variant=messages.Variant.INT32) - num_slides = messages.IntegerField(32, variant=messages.Variant.INT32) - pathology_report_uuid = messages.StringField(33) - preservation_method = messages.StringField(34) - sample_barcode = messages.StringField(35) - sample_gdc_id = messages.StringField(36) - sample_type = messages.StringField(37) - -class Data_HG19MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - case_barcode = messages.StringField(7, repeated=True) - case_gdc_id = messages.StringField(8, repeated=True) - data_category = messages.StringField(12, repeated=True) - data_format = messages.StringField(13, repeated=True) - data_type = messages.StringField(14, repeated=True) - experimental_strategy = messages.StringField(15, repeated=True) - index_file_name_key = messages.StringField(19, repeated=True) - file_name_key = messages.StringField(18, repeated=True) - platform = messages.StringField(20, repeated=True) - sample_barcode = messages.StringField(21, repeated=True) - sample_gdc_id = messages.StringField(22, repeated=True) - -class Data_HG19MetadataItem(messages.Message): - access = messages.StringField(1) - case_barcode = messages.StringField(7) - case_gdc_id = messages.StringField(8) - data_category = messages.StringField(12) - data_format = messages.StringField(13) - data_type = messages.StringField(14) - experimental_strategy = messages.StringField(15) - index_file_name_key = messages.StringField(19) - file_name_key = messages.StringField(18) - platform = messages.StringField(20) - sample_barcode = messages.StringField(21) - sample_gdc_id = messages.StringField(22) - -class Data_HG38MetadataRangesItem(messages.Message): - access = messages.StringField(1, repeated=True) - case_barcode = messages.StringField(7, repeated=True) - case_gdc_id = messages.StringField(8, repeated=True) - data_category = messages.StringField(12, repeated=True) - data_format = messages.StringField(13, repeated=True) - data_type = messages.StringField(14, repeated=True) - experimental_strategy = messages.StringField(15, repeated=True) - index_file_name_key = messages.StringField(19, repeated=True) - file_name_key = messages.StringField(18, repeated=True) - platform = messages.StringField(20, repeated=True) - sample_barcode = messages.StringField(21, repeated=True) - sample_gdc_id = messages.StringField(22, repeated=True) - -class Data_HG38MetadataItem(messages.Message): - access = messages.StringField(1) - case_barcode = messages.StringField(7) - case_gdc_id = messages.StringField(8) - data_category = messages.StringField(12) - data_format = messages.StringField(13) - data_type = messages.StringField(14) - experimental_strategy = messages.StringField(15) - index_file_name_key = messages.StringField(19) - file_name_key = messages.StringField(18) - platform = messages.StringField(20) - sample_barcode = messages.StringField(21) - sample_gdc_id = messages.StringField(22) - -class MetadataRangesItem(messages.Message): - Common = messages.MessageField(CommonMetadataRangesItem, 1) - Clinical = messages.MessageField(ClinicalMetadataRangesItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataRangesItem, 3) - data_HG19_r14 = messages.MessageField(Data_HG19MetadataRangesItem, 4) - data_HG38_r14 = messages.MessageField(Data_HG38MetadataRangesItem, 5) - -class MetadataItem(messages.Message): - Common = messages.MessageField(CommonMetadataItem, 1) - Clinical = messages.MessageField(ClinicalMetadataItem, 2) - Biospecimen = messages.MessageField(BiospecimenMetadataItem, 3) - data_HG19_r14 = messages.MessageField(Data_HG19MetadataItem, 4) - data_HG38_r14 = messages.MessageField(Data_HG38MetadataItem, 5) diff --git a/api_3/isb_cgc_api_TCGA/patients_get.py b/api_3/isb_cgc_api_TCGA/patients_get.py deleted file mode 100644 index 1694378c..00000000 --- a/api_3/isb_cgc_api_TCGA/patients_get.py +++ /dev/null @@ -1,55 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.patients_get_helper import CasesGetHelper -from api_3.isb_cgc_api_TCGA.message_classes import ClinicalMetadataItem as MetadataItem -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -class CaseDetails(messages.Message): - clinical_data = messages.MessageField(MetadataItem, 1) - samples = messages.StringField(2, repeated=True) - aliquots = messages.StringField(3, repeated=True) - case_barcode = messages.StringField(4) - -class CaseSetDetails(messages.Message): - cases = messages.MessageField(CaseDetails, 1, repeated=True) - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='cases') -class TCGACasesGetAPI(CasesGetHelper): - @endpoints.method(CasesGetHelper.GET_RESOURCE, CaseDetails, path='tcga/cases/{case_barcode}', http_method='GET') - def get(self, request): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case barcode (of length 12, *eg* TCGA-B9-7268) as a required parameter. - User does not need to be authenticated. - """ - return super(TCGACasesGetAPI, self).get(request, CaseDetails, MetadataItem, 'TCGA') - - @endpoints.method(CasesGetHelper.POST_RESOURCE, CaseSetDetails, path='tcga/cases', http_method='POST') - def get_list(self, request): - """ - Given a list of case barcodes (of length 11, *eg* TCGA-B9-7268), this endpoint returns - all available "biospecimen" information about the cases, including a list of samples and aliquots - derived from them. - Takes a list of case barcodes (of length 12, *eg* TCGA-B9-7268) as a required data payload. - User does not need to be authenticated. - """ - return super(TCGACasesGetAPI, self).get_list(request, CaseSetDetails, CaseDetails, MetadataItem, 'TCGA') diff --git a/api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py b/api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py deleted file mode 100644 index 6d8b4f3f..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_cloudstoragefilepaths.py +++ /dev/null @@ -1,33 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.cloudstoragefilepaths_helper import GCSFilePathList, SamplesCloudStorageFilePathsHelper - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class TCGASamplesCloudStorageFilePathsAPI(SamplesCloudStorageFilePathsHelper): - - @endpoints.method(SamplesCloudStorageFilePathsHelper.GET_RESOURCE, GCSFilePathList, - path='tcga/samples/{sample_barcode}/cloud_storage_file_paths', http_method='GET') - def cloud_storage_file_paths(self, request): - """ - Takes a sample barcode as a required parameter and - returns cloud storage paths to files associated with that sample. - """ - return super(TCGASamplesCloudStorageFilePathsAPI, self).cloud_storage_file_paths(request, 'TCGA') diff --git a/api_3/isb_cgc_api_TCGA/samples_get.py b/api_3/isb_cgc_api_TCGA/samples_get.py deleted file mode 100644 index fa44d0a6..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_get.py +++ /dev/null @@ -1,60 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import messages - -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.samples_get_helper import SamplesGetAPI, DataDetails -from api_3.isb_cgc_api_TCGA.message_classes import BiospecimenMetadataItem - - -class SampleDetails(messages.Message): - biospecimen_data = messages.MessageField(BiospecimenMetadataItem, 1) - aliquots = messages.StringField(2, repeated=True) - case_barcode = messages.StringField(3) - case_gdc_id = messages.StringField(4) - data_details = messages.MessageField(DataDetails, 5, repeated=True) - data_details_count = messages.IntegerField(6, variant=messages.Variant.INT32) - sample_barcode = messages.StringField(7) - - -class SampleSetDetails(messages.Message): - samples = messages.MessageField(SampleDetails, 1, repeated=True) - - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class TCGASamplesGetAPI(SamplesGetAPI): - @endpoints.method(SamplesGetAPI.GET_RESOURCE, SampleDetails, path='tcga/samples/{sample_barcode}', http_method='GET') - def get(self, request): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TCGASamplesGetAPI, self).get(request, 'TCGA', SampleDetails, BiospecimenMetadataItem) - - @endpoints.method(SamplesGetAPI.POST_RESOURCE, SampleSetDetails, path='tcga/samples', http_method='POST') - def get_list(self, request): - """ - Given a list of sample barcodes (of length 16, *eg* TCGA-B9-7268-01A), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - return super(TCGASamplesGetAPI, self).get_list(request, 'TCGA', SampleSetDetails, SampleDetails, BiospecimenMetadataItem) diff --git a/api_3/isb_cgc_api_TCGA/samples_googlegenomics.py b/api_3/isb_cgc_api_TCGA/samples_googlegenomics.py deleted file mode 100644 index 27a1bfc3..00000000 --- a/api_3/isb_cgc_api_TCGA/samples_googlegenomics.py +++ /dev/null @@ -1,96 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import logging -import MySQLdb - -from django.conf import settings -from protorpc import remote, messages - -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - -BASE_URL = settings.BASE_URL - - -class GoogleGenomics(messages.Message): - SampleBarcode = messages.StringField(1) - GG_dataset_id = messages.StringField(2) - GG_readgroupset_id = messages.StringField(3) - - -class GoogleGenomicsList(messages.Message): - items = messages.MessageField(GoogleGenomics, 1, repeated=True) - count = messages.IntegerField(2, variant=messages.Variant.INT32) - - -# @ISB_CGC_TCGA_Endpoints.api_class(resource_name='samples') -class SamplesGoogleGenomicsAPI(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(sample_barcode=messages.StringField(1, required=True)) - - # @endpoints.method(GET_RESOURCE, GoogleGenomicsList, http_method='GET', - # path='samples/{sample_barcode}/googlegenomics') - def googlegenomics(self, request): - """ - Takes a sample barcode as a required parameter and returns the Google Genomics dataset id - and readgroupset id associated with the sample, if any. - """ - - cursor = None - db = None - sample_barcode = request.get_assigned_value('sample_barcode') - - query_str = 'SELECT SampleBarcode, GG_dataset_id, GG_readgroupset_id ' \ - 'FROM metadata_data ' \ - 'WHERE SampleBarcode=%s ' \ - 'AND GG_dataset_id !="" AND GG_readgroupset_id !="" ' \ - 'GROUP BY SampleBarcode, GG_dataset_id, GG_readgroupset_id;' - - query_tuple = (sample_barcode,) - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, query_tuple) - - google_genomics_items = [ - GoogleGenomics( - SampleBarcode=row['SampleBarcode'], - GG_dataset_id=row['GG_dataset_id'], - GG_readgroupset_id=row['GG_readgroupset_id'] - ) - for row in cursor.fetchall() - ] - return GoogleGenomicsList(items=google_genomics_items, count=len(google_genomics_items)) - - except (IndexError, TypeError), e: - logger.warn(e) - raise endpoints.NotFoundException( - "Google Genomics dataset and readgroupset id's for sample {} not found." - .format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - msg = '{}:\n\tquery: {} {}' \ - .format(e, query_str, query_tuple) - logger.warn(msg) - raise endpoints.BadRequestException("Error retrieving genomics data for sample. {}".format(msg)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() \ No newline at end of file diff --git a/api_3/isb_cgc_api_TCGA/users_get.py b/api_3/isb_cgc_api_TCGA/users_get.py deleted file mode 100644 index a1ee020a..00000000 --- a/api_3/isb_cgc_api_TCGA/users_get.py +++ /dev/null @@ -1,32 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints -from protorpc import message_types - -from api_3.users_get_common import UserGetAPICommon, UserGetAPIReturnJSON -from api_3.isb_cgc_api_TCGA.isb_cgc_api_helpers import ISB_CGC_TCGA_Endpoints - -@ISB_CGC_TCGA_Endpoints.api_class(resource_name='users') -class TCGAUserGetAPI(UserGetAPICommon): - - @endpoints.method(message_types.VoidMessage, UserGetAPIReturnJSON, http_method='GET', path='tcga/users') - def get(self, _): - ''' - Returns the dbGaP authorization status of the user. - ''' - return super(TCGAUserGetAPI, self).get('TCGA') diff --git a/api_3/maf.py b/api_3/maf.py deleted file mode 100755 index 19e61845..00000000 --- a/api_3/maf.py +++ /dev/null @@ -1,104 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -from protorpc import remote -from protorpc.messages import Message, MessageField, IntegerField, StringField, Variant -from MySQLdb.cursors import DictCursor - -from api_helpers import * - - -# Generated code - in scripts/maf_import in Python do: -# -# > import import_maf -# > repr(import_maf.FIELD_NAMES_AND_TYPES) -# -FIELD_NAMES_AND_TYPES = [ - {'type': 'varchar(128)', 'name': 'hugo_symbol'}, - {'type': 'int', 'name': 'amino_acid_position'}, - {'type': 'varchar(128)', 'name': 'tumor_type'}, - {'type': 'varchar(12)', 'name': 'uniprot_id'}, - {'type': 'varchar(128)', 'name': 'variant_classification'}, - {'type': 'varchar(128)', 'name': 'c_position'}, - {'type': 'varchar(128)', 'name': 'tumor_sample_barcode'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'tumor_seq_allele2'}, - {'type': 'varchar(128)', 'name': 'match_norm_seq_allele1'}, - {'type': 'varchar(128)', 'name': 'reference_allele'}, - {'type': 'varchar(128)', 'name': 'variant_type'}, - {'type': 'varchar(128)', 'name': 'ucsc_cons'} -] - - -class MAFRecord(Message): - hugo_symbol = StringField(1) - tumor_type = StringField(3) - amino_acid_position = IntegerField(2, variant=Variant.INT32) - uniprot_id = StringField(4) - variant_classification = StringField(5) - c_position = StringField(6) - tumor_sample_barcode = StringField(7) - match_norm_seq_allele2 = StringField(8) - tumor_seq_allele1 = StringField(9) - tumor_seq_allele2 = StringField(10) - match_norm_seq_allele1 = StringField(11) - reference_allele = StringField(12) - variant_type = StringField(13) - ucsc_cons = StringField(14) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - - -class MAFRequest(Message): - gene = StringField(1, required=True) - tumor = StringField(2, repeated=True) - -MAFEndpointsAPI = endpoints.api(name='maf_api', version='v3') - - -@MAFEndpointsAPI .api_class(resource_name='maf_endpoints') -class MAFEndpointsAPI(remote.Service): - @endpoints.method(MAFRequest, MAFRecordList, - path='maf_search', http_method='GET', name='maf.getMAF') - def maf_search(self, request): - gene = request.gene - tumor_type_list = request.tumor - tumor_set_template = ', '.join(['%s' for x in range(len(tumor_type_list))]) - query = 'SELECT * FROM maf WHERE hugo_symbol=%s AND tumor_type IN ({0})'.format(tumor_set_template) - - values = [gene] - values.extend(tumor_type_list) - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - data_list = [] - for row in cursor.fetchall(): - data_list.append(MAFRecord(**row)) - - cursor.close() - db.close() - return MAFRecordList(items=data_list) - - except: - raise endpoints.NotFoundException('MAF query error') \ No newline at end of file diff --git a/api_3/message_generator.py b/api_3/message_generator.py deleted file mode 100755 index 77cf470a..00000000 --- a/api_3/message_generator.py +++ /dev/null @@ -1,241 +0,0 @@ -''' -copyright 2017, Institute for Systems Biology. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -''' -from argparse import ArgumentParser -from datetime import datetime -import MySQLdb - -def get_sql_connection(args): - try: - connect_options = { - 'host': args.host, - 'db': args.database, - 'user': args.user, - 'passwd': args.password, - 'ssl': { - 'ca': args.ssl_ca, - 'key': args.ssl_key, - 'cert': args.ssl_cert - } - } - db = MySQLdb.connect(**connect_options) - except: - print 'failed' - raise - return db - -FIELD_TYPES ={ - 'varchar': 'StringField', - 'float': 'FloatField', - 'tinyint': 'BooleanField', - 'int': 'IntegerField' -} - -seen_rows = set() -def write_allowed_values(cursor, rows, path, program, metadata_type, column_filter, append_file=True, complete_file=False, write_file=False): - if not write_file: - return - - MAX_VALUES = 70 - try: - mode = 'a+' if append_file else 'w' - with open(path, mode) as av: - if not append_file: - av.write('{\n') - av.write(' "{}": {{\n'.format(metadata_type)) - table_name = 'Clinical' if 'Common' == metadata_type else metadata_type - query = 'select {} from %s_metadata_%s group by 1 order by 1' % (program, table_name[:1].lower() + table_name[1:]) - body = '' - for row in rows: -# if row['DATA_TYPE'] != 'varchar' or row['COLUMN_NAME'] in seen_rows: - if row['DATA_TYPE'] != 'varchar': - continue - if ('Common' in metadata_type and row['COLUMN_NAME'] not in column_filter) or ('Common' not in metadata_type and row['COLUMN_NAME'] in column_filter): - continue - - seen_rows.add(row['COLUMN_NAME']) - cursor.execute(query.format(row['COLUMN_NAME'])) - currows = cursor.fetchall() - if MAX_VALUES < len(currows): - continue - if 1 == len(currows) and 'None' == currows[0][row['COLUMN_NAME']]: - continue - - body += ' "{}": [\n {}\n ],\n'.format(row['COLUMN_NAME'], ' '.join('"{}",\n'.format(currow[row['COLUMN_NAME']]) for currow in currows if not currow[row['COLUMN_NAME']] is None)[:-2]) - av.write(body[:-2]) - av.write('\n }') - - if complete_file: - av.write('\n}\n') - else: - av.write(',\n') - av.flush() - return - except: - cursor.close() - raise - -def create_nesting_class(table_list, path, write_file): - if not write_file: - return - - with open(path, 'a') as f: - ranges_body = 'class MetadataRangesItem(messages.Message):\n' - items_body = 'class MetadataItem(messages.Message):\n' - for index, table in enumerate(table_list): - ranges_body += ' {0} = messages.MessageField({0}MetadataRangesItem, {1})\n'.format(table, index+1) - items_body += ' {0} = messages.MessageField({0}MetadataItem, {1})\n'.format(table, index+1) - f.write('{}\n'.format(ranges_body)) - f.write('{}\n'.format(items_body)) - -def write_metadata_file(class_name, rows, path, append_file=True, write_file=False): - - ranges_text = 'class {}RangesItem(messages.Message):\n '.format(class_name) - - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if not field_type: - continue - ranges_text += '%-65s = messages.%s(%d, repeated=True' % (row['COLUMN_NAME'], field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if field_type == 'IntegerField' or field_type == 'FloatField': - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_lte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME']+'_gte', field_type, i) - ranges_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - ranges_text += '\n ' - - item_text = '\nclass {}Item(messages.Message):\n '.format(class_name) - i = 1 - for row in rows: - field_type = FIELD_TYPES.get(row['DATA_TYPE']) - if not field_type: - continue - item_text += '%-65s = messages.%s(%d' % (row['COLUMN_NAME'], field_type, i) - item_text += ')\n ' if field_type is not 'IntegerField' else ', variant=messages.Variant.INT32)\n ' - i += 1 - - if write_file is True: - mode = 'a' if append_file else 'w' - with open(path, mode) as f: - if not append_file: - f.write('from protorpc import messages\n\n') - f.write(ranges_text) - f.write(item_text + '\n') - else: - print path + '\n' - print ranges_text - print '\n\n' - print item_text - -def get_table_column_info(cursor, program, metadata_type, column_filter = []): - key_part = metadata_type.split('_')[0].lower() - table_name = 'Clinical' if 'Common' == metadata_type else metadata_type - query_str = 'SELECT COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'FROM INFORMATION_SCHEMA.COLUMNS ' \ - 'WHERE table_name = "{}_metadata_{}" ' \ - 'AND COLUMN_NAME != "metadata_{}_id" ' + 'group by COLUMN_NAME, DATA_TYPE, COLUMN_TYPE ' \ - 'ORDER BY column_name' - cursor.execute(query_str.format(program, table_name[:1].lower() + table_name[1:], key_part)) - rows = cursor.fetchall() - retrows = [] - seen = set() - for row in rows: - if ('Common' in metadata_type and row['COLUMN_NAME'] not in column_filter) or ('Common' not in metadata_type and row['COLUMN_NAME'] in column_filter): - continue - if row['COLUMN_NAME'] in seen: - continue - seen.add(row['COLUMN_NAME']) - retrows += [row] - return retrows - -def create_metadata_file(cursor, program, metadata_type, path, column_filter, append_file=True, complete_file=False, write_file=True): - rows = get_table_column_info(cursor, program, metadata_type, column_filter) - write_metadata_file(metadata_type + 'Metadata', rows, path, append_file, write_file) - path = 'api_3/isb_cgc_api_{0}/allowed_values_v3_{0}.json'.format(program) - write_allowed_values(cursor, rows, path, program, metadata_type, column_filter, append_file, complete_file, write_file) - return rows - -def main(args): - db = get_sql_connection(args) - cursor = None - try: - path_template = 'api_3/isb_cgc_api_%s/message_classes.py' - programs = [ - ('CCLE', False), - ('TARGET', False), - ('TCGA', True), - ] - for program in programs: - print datetime.now(), 'starting program {}'.format(program) - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - write_file = not bool(args.dry_run) - # for the columns that are common to all tables and have the same values from the subsequent tables, get them here and any filters - column_filter = ["disease_code", "endpoint_type", "program_name", "project_short_name"] - create_metadata_file(cursor, program[0], 'Common', path_template % (program[0]), column_filter, False, False, write_file) - column_filter += [ - 'acl', - 'analysis_gdc_id', - 'analysis_workflow_link', - 'archive_gdc_id', - 'archive_revision', - 'archive_revision_lte', - 'archive_revision_gte', - 'file_gdc_id', - 'file_name_key', - 'index_file_id', - 'md5sum', - 'type' - ] - create_metadata_file(cursor, program[0], 'Clinical', path_template % (program[0]), column_filter, True, False, write_file) - column_filter += ["disease_code", "endpoint_type", "program_name", "project_short_name"] - create_metadata_file(cursor, program[0], 'Biospecimen', path_template % (program[0]), column_filter, True, False, write_file) - table_list = ['Common', 'Clinical', 'Biospecimen', 'data_HG19_r14'] - create_metadata_file(cursor, program[0], 'Data_HG19', path_template % (program[0]), column_filter, True, True if 'CCLE' == program[0] else False, write_file) - if 'CCLE' != program[0]: - create_metadata_file(cursor, program[0], 'Data_HG38', path_template % (program[0]), column_filter, True, False if program[1] else True, write_file) - table_list += ['data_HG38_r14'] - create_nesting_class(table_list, path_template % (program[0]), write_file) - print datetime.now(), 'finished program {}'.format(program) - finally: - if cursor: - cursor.close() - db.close() - -if __name__ == '__main__': - parser = ArgumentParser() - parser.add_argument('--host', help='database host') - parser.add_argument('--database', '-d', help='database name') - parser.add_argument('--user', '-u', help='database username') - parser.add_argument('--password', '-p', help='database password') - parser.add_argument('--ssl_key') - parser.add_argument('--ssl_cert') - parser.add_argument('--ssl_ca') - parser.add_argument('--dry_run', dest='dry_run', action='store_true') - parser.add_argument('--write_file', dest='dry_run', action='store_false') - parser.set_defaults(feature=True) - args = parser.parse_args() - - print datetime.now(), 'starting...' - main(args) - print datetime.now(), 'finished' - - diff --git a/api_3/metadata.py b/api_3/metadata.py deleted file mode 100755 index 880203a8..00000000 --- a/api_3/metadata.py +++ /dev/null @@ -1,1915 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import endpoints -import time -from protorpc import messages -from protorpc import message_types -from protorpc import remote -from django.conf import settings -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.contrib.auth.models import User as Django_User -from allauth.socialaccount.models import SocialToken, SocialAccount -from accounts.models import NIH_User -from cohorts.models import Cohort_Perms, Cohort as Django_Cohort, Samples, Filters -from projects.models import Project, User_Feature_Definitions, User_Feature_Counts, User_Data_Tables -from django.core.signals import request_finished -from time import sleep -import django -import sys -import logging -import re -import json -import traceback -import copy -from uuid import uuid4 - -from api_3.api_helpers import * - -logger = logging.getLogger(__name__) - -debug = settings.DEBUG - -INSTALLED_APP_CLIENT_ID = settings.INSTALLED_APP_CLIENT_ID -OPEN_DATA_BUCKET = settings.OPEN_DATA_BUCKET -#CONTROLLED_DATA_BUCKET = settings.CONTROLLED_DATA_BUCKET - -METADATA_SHORTLIST = [ - # 'adenocarcinoma_invasion', - 'age_at_diagnosis', - # 'anatomic_neoplasm_subdivision', - # 'avg_percent_lymphocyte_infiltration', - # 'avg_percent_monocyte_infiltration', - # 'avg_percent_necrosis', - # 'avg_percent_neutrophil_infiltration', - # 'avg_percent_normal_cells', - # 'avg_percent_stromal_cells', - # 'avg_percent_tumor_cells', - # 'avg_percent_tumor_nuclei', - # 'batch_number', - # 'bcr', - 'BMI', - # 'clinical_M', - # 'clinical_N', - # 'clinical_stage', - # 'clinical_T', - # 'colorectal_cancer', - # 'country', - # 'country_of_procurement', - # 'days_to_birth', - # 'days_to_collection', - # 'days_to_death', - # 'days_to_initial_pathologic_diagnosis', - # 'days_to_last_followup', - # 'days_to_submitted_specimen_dx', - 'Study', - # 'ethnicity', - # 'frozen_specimen_anatomic_site', - 'gender', - # 'gleason_score_combined', - 'has_27k', - 'has_450k', - 'has_BCGSC_GA_RNASeq', - 'has_BCGSC_HiSeq_RNASeq', - 'has_GA_miRNASeq', - 'has_HiSeq_miRnaSeq', - 'has_Illumina_DNASeq', - 'has_RPPA', - 'has_SNP6', - 'has_UNC_GA_RNASeq', - 'has_UNC_HiSeq_RNASeq', - # 'height', - 'histological_type', - # 'history_of_colon_polyps', - # 'history_of_neoadjuvant_treatment', - # 'history_of_prior_malignancy', - # 'hpv_calls', - 'hpv_status', - 'icd_10', - 'icd_o_3_histology', - 'icd_o_3_site', - # 'lymph_node_examined_count', - # 'lymphatic_invasion', - # 'lymphnodes_examined', - # 'lymphovascular_invasion_present', - # 'max_percent_lymphocyte_infiltration', - # 'max_percent_monocyte_infiltration', - # 'max_percent_necrosis', - # 'max_percent_neutrophil_infiltration', - # 'max_percent_normal_cells', - # 'max_percent_stromal_cells', - # 'max_percent_tumor_cells', - # 'max_percent_tumor_nuclei', - # 'menopause_status', - # 'min_percent_lymphocyte_infiltration', - # 'min_percent_monocyte_infiltration', - # 'min_percent_necrosis', - # 'min_percent_neutrophil_infiltration', - # 'min_percent_normal_cells', - # 'min_percent_stromal_cells', - # 'min_percent_tumor_cells', - # 'min_percent_tumor_nuclei', - # 'mononucleotide_and_dinucleotide_marker_panel_analysis_status', - # 'mononucleotide_marker_panel_analysis_status', - 'neoplasm_histologic_grade', - 'new_tumor_event_after_initial_treatment', - # 'number_of_lymphnodes_examined', - # 'number_of_lymphnodes_positive_by_he', - # 'number_pack_years_smoked', - # 'other_dx', - # 'case_barcode', - # 'pathologic_M', - # 'pathologic_N', - 'pathologic_stage', - # 'pathologic_T', - 'person_neoplasm_cancer_status', - # 'pregnancies', - # 'preservation_method', - # 'primary_neoplasm_melanoma_dx', - # 'primary_therapy_outcome_success', - 'Project', - # 'psa_value', - # 'race', - 'residual_tumor', - # 'sample_barcode', - 'SampleTypeCode', - # 'Study', - 'tobacco_smoking_history', - # 'total_number_of_pregnancies', - # 'tumor_pathology', - 'tumor_tissue_site', - 'tumor_type', - # 'weiss_venous_invasion', - 'vital_status' - # 'weight', - # 'year_of_initial_pathologic_diagnosis', -] - -metadata_dict = { - 'age_at_diagnosis': 'INTEGER', - 'anatomic_neoplasm_subdivision': 'VARCHAR(63)', - 'avg_percent_lymphocyte_infiltration': 'FLOAT', - 'avg_percent_monocyte_infiltration': 'FLOAT', - 'avg_percent_necrosis': 'FLOAT', - 'avg_percent_neutrophil_infiltration': 'FLOAT', - 'avg_percent_normal_cells': 'FLOAT', - 'avg_percent_stromal_cells': 'FLOAT', - 'avg_percent_tumor_cells': 'FLOAT', - 'avg_percent_tumor_nuclei': 'FLOAT', - 'batch_number': 'INTEGER', - 'bcr': 'VARCHAR(63)', - 'BMI': 'FLOAT', - 'clinical_M': 'VARCHAR(12)', - 'clinical_N': 'VARCHAR(12)', - 'clinical_T': 'VARCHAR(12)', - 'clinical_stage': 'VARCHAR(12)', - 'colorectal_cancer': 'VARCHAR(10)', - 'country': 'VARCHAR(63)', - 'days_to_birth': 'INTEGER', - 'days_to_collection': 'INTEGER', - 'days_to_death': 'INTEGER', - 'days_to_initial_pathologic_diagnosis': 'INTEGER', - 'days_to_last_followup': 'INTEGER', - 'days_to_submitted_specimen_dx': 'INTEGER', - 'Study': 'VARCHAR(4)', - 'ethnicity': 'VARCHAR(20)', - 'frozen_specimen_anatomic_site': 'VARCHAR(63)', - 'gender': 'VARCHAR(15)', - 'gleason_score_combined': 'INTEGER', - 'height': 'INTEGER', - 'histological_type': 'VARCHAR(63)', - 'history_of_colon_polyps': 'VARCHAR(8)', - 'history_of_neoadjuvant_treatment': 'VARCHAR(63)', - 'history_of_prior_malignancy': 'VARCHAR(25)', - 'hpv_calls': 'VARCHAR(20)', - 'hpv_status': 'VARCHAR(20)', - 'icd_10': 'VARCHAR(8)', - 'icd_o_3_histology': 'VARCHAR(10)', - 'icd_o_3_site': 'VARCHAR(8)', - 'lymphatic_invasion': 'VARCHAR(8)', - 'lymphnodes_examined': 'VARCHAR(8)', - 'lymphovascular_invasion_present': 'VARCHAR(63)', - 'max_percent_lymphocyte_infiltration': 'INTEGER', - 'max_percent_monocyte_infiltration': 'INTEGER', - 'max_percent_necrosis': 'INTEGER', - 'max_percent_neutrophil_infiltration': 'INTEGER', - 'max_percent_normal_cells': 'INTEGER', - 'max_percent_stromal_cells': 'INTEGER', - 'max_percent_tumor_cells': 'INTEGER', - 'max_percent_tumor_nuclei': 'INTEGER', - 'menopause_status': 'VARCHAR(30)', - 'min_percent_lymphocyte_infiltration': 'INTEGER', - 'min_percent_monocyte_infiltration': 'INTEGER', - 'min_percent_necrosis': 'INTEGER', - 'min_percent_neutrophil_infiltration': 'INTEGER', - 'min_percent_normal_cells': 'INTEGER', - 'min_percent_stromal_cells': 'INTEGER', - 'min_percent_tumor_cells': 'INTEGER', - 'min_percent_tumor_nuclei': 'INTEGER', - 'mononucleotide_and_dinucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'mononucleotide_marker_panel_analysis_status': 'VARCHAR(20)', - 'neoplasm_histologic_grade': 'VARCHAR(15)', - 'new_tumor_event_after_initial_treatment': 'VARCHAR(8)', - 'number_of_lymphnodes_examined': 'INTEGER', - 'number_of_lymphnodes_positive_by_he': 'INTEGER', - 'number_pack_years_smoked': 'INTEGER', - 'case_barcode': 'VARCHAR(12)', - 'pathologic_M': 'VARCHAR(5)', - 'pathologic_N': 'VARCHAR(5)', - 'pathologic_T': 'VARCHAR(5)', - 'pathologic_stage': 'VARCHAR(10)', - 'person_neoplasm_cancer_status': 'VARCHAR(15)', - 'pregnancies': 'VARCHAR(35)', - 'primary_neoplasm_melanoma_dx': 'VARCHAR(10)', - 'primary_therapy_outcome_success': 'VARCHAR(35)', - 'prior_dx': 'VARCHAR(50)', - 'Project': 'VARCHAR(4)', - 'psa_value': 'FLOAT', - 'race': 'VARCHAR(30)', - 'residual_tumor': 'VARCHAR(5)', - 'sample_barcode': 'VARCHAR(16)', - 'Study': 'VARCHAR(4)', - 'tobacco_smoking_history': 'VARCHAR(30)', - 'tumor_tissue_site': 'VARCHAR(20)', - 'tumor_type': 'VARCHAR(4)', - 'weiss_venous_invasion': 'VARCHAR(63)', - 'vital_status': 'VARCHAR(63)', - 'weight': 'VARCHAR(63)', - 'year_of_initialPY_pathologic_diagnosis': 'VARCHAR(63)', - 'SampleTypeCode': 'VARCHAR(3)', - 'has_Illumina_DNASeq': 'TINYINT', - 'has_BCGSC_HiSeq_RNASeq': 'TINYINT', - 'has_UNC_HiSeq_RNASeq': 'TINYINT', - 'has_BCGSC_GA_RNASeq': 'TINYINT', - 'has_UNC_GA_RNASeq': 'TINYINT', - 'has_HiSeq_miRnaSeq': 'TINYINT', - 'has_GA_miRNASeq': 'TINYINT', - 'has_RPPA': 'TINYINT', - 'has_SNP6': 'TINYINT', - 'has_27k': 'TINYINT', - 'has_450k': 'TINYINT', -} - - -class MetaValueListCount(messages.Message): - value = messages.StringField(1) # note: this means converting booleans to strings - count = messages.IntegerField(2) - - -class MetaAttrValuesList(messages.Message): - adenocarcinoma_invasion = messages.MessageField(MetaValueListCount, 1, repeated=True) - age_at_diagnosis = messages.MessageField(MetaValueListCount, 2, repeated=True) - anatomic_neoplasm_subdivision = messages.MessageField(MetaValueListCount, 3, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(5, repeated=True) - avg_percent_necrosis = messages.FloatField(6, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(7, repeated=True) - avg_percent_normal_cells = messages.FloatField(8, repeated=True) - avg_percent_stromal_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_cells = messages.FloatField(10, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(11, repeated=True) - batch_number = messages.MessageField(MetaValueListCount, 12, repeated=True) - bcr = messages.MessageField(MetaValueListCount, 13, repeated=True) - clinical_M = messages.MessageField(MetaValueListCount, 14, repeated=True) - clinical_N = messages.MessageField(MetaValueListCount, 15, repeated=True) - clinical_stage = messages.MessageField(MetaValueListCount, 16, repeated=True) - clinical_T = messages.MessageField(MetaValueListCount, 17, repeated=True) - colorectal_cancer = messages.MessageField(MetaValueListCount, 18, repeated=True) - country = messages.MessageField(MetaValueListCount, 19, repeated=True) - country_of_procurement = messages.MessageField(MetaValueListCount, 20, repeated=True) - days_to_birth = messages.MessageField(MetaValueListCount, 21, repeated=True) - days_to_collection = messages.MessageField(MetaValueListCount, 22, repeated=True) - days_to_death = messages.MessageField(MetaValueListCount, 23, repeated=True) - days_to_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 24, repeated=True) - days_to_last_followup = messages.MessageField(MetaValueListCount, 25, repeated=True) - days_to_submitted_specimen_dx = messages.MessageField(MetaValueListCount, 26, repeated=True) - Study = messages.MessageField(MetaValueListCount, 27, repeated=True) - ethnicity = messages.MessageField(MetaValueListCount, 28, repeated=True) - frozen_specimen_anatomic_site = messages.MessageField(MetaValueListCount, 29, repeated=True) - gender = messages.MessageField(MetaValueListCount, 30, repeated=True) - height = messages.MessageField(MetaValueListCount, 31, repeated=True) - histological_type = messages.MessageField(MetaValueListCount, 32, repeated=True) - history_of_colon_polyps = messages.MessageField(MetaValueListCount, 33, repeated=True) - history_of_neoadjuvant_treatment = messages.MessageField(MetaValueListCount, 34, repeated=True) - history_of_prior_malignancy = messages.MessageField(MetaValueListCount, 35, repeated=True) - hpv_calls = messages.MessageField(MetaValueListCount, 36, repeated=True) - hpv_status = messages.MessageField(MetaValueListCount, 37, repeated=True) - icd_10 = messages.MessageField(MetaValueListCount, 38, repeated=True) - icd_o_3_histology = messages.MessageField(MetaValueListCount, 39, repeated=True) - icd_o_3_site = messages.MessageField(MetaValueListCount, 40, repeated=True) - lymph_node_examined_count = messages.MessageField(MetaValueListCount, 41, repeated=True) - lymphatic_invasion = messages.MessageField(MetaValueListCount, 42, repeated=True) - lymphnodes_examined = messages.MessageField(MetaValueListCount, 43, repeated=True) - lymphovascular_invasion_present = messages.MessageField(MetaValueListCount, 44, repeated=True) - max_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 45, repeated=True) - max_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 46, repeated=True) - max_percent_necrosis = messages.MessageField(MetaValueListCount, 47, repeated=True) - max_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 48, repeated=True) - max_percent_normal_cells = messages.MessageField(MetaValueListCount, 49, repeated=True) - max_percent_stromal_cells = messages.MessageField(MetaValueListCount, 50, repeated=True) - max_percent_tumor_cells = messages.MessageField(MetaValueListCount, 51, repeated=True) - max_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 52, repeated=True) - menopause_status = messages.MessageField(MetaValueListCount, 53, repeated=True) - min_percent_lymphocyte_infiltration = messages.MessageField(MetaValueListCount, 54, repeated=True) - min_percent_monocyte_infiltration = messages.MessageField(MetaValueListCount, 55, repeated=True) - min_percent_necrosis = messages.MessageField(MetaValueListCount, 56, repeated=True) - min_percent_neutrophil_infiltration = messages.MessageField(MetaValueListCount, 57, repeated=True) - min_percent_normal_cells = messages.MessageField(MetaValueListCount, 58, repeated=True) - min_percent_stromal_cells = messages.MessageField(MetaValueListCount, 59, repeated=True) - min_percent_tumor_cells = messages.MessageField(MetaValueListCount, 60, repeated=True) - min_percent_tumor_nuclei = messages.MessageField(MetaValueListCount, 61, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 62, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.MessageField(MetaValueListCount, 63, repeated=True) - neoplasm_histologic_grade = messages.MessageField(MetaValueListCount, 64, repeated=True) - new_tumor_event_after_initial_treatment = messages.MessageField(MetaValueListCount, 65, repeated=True) - number_of_lymphnodes_examined = messages.MessageField(MetaValueListCount, 66, repeated=True) - number_of_lymphnodes_positive_by_he = messages.MessageField(MetaValueListCount, 67, repeated=True) - case_barcode = messages.MessageField(MetaValueListCount, 68, repeated=True) - pathologic_M = messages.MessageField(MetaValueListCount, 69, repeated=True) - pathologic_N = messages.MessageField(MetaValueListCount, 70, repeated=True) - pathologic_stage = messages.MessageField(MetaValueListCount, 71, repeated=True) - pathologic_T = messages.MessageField(MetaValueListCount, 72, repeated=True) - person_neoplasm_cancer_status = messages.MessageField(MetaValueListCount, 73, repeated=True) - pregnancies = messages.MessageField(MetaValueListCount, 74, repeated=True) - preservation_method = messages.MessageField(MetaValueListCount, 75, repeated=True) - primary_neoplasm_melanoma_dx = messages.MessageField(MetaValueListCount, 76, repeated=True) - primary_therapy_outcome_success = messages.MessageField(MetaValueListCount, 77, repeated=True) - prior_dx = messages.MessageField(MetaValueListCount, 78, repeated=True) - Project = messages.MessageField(MetaValueListCount, 79, repeated=True) - psa_value = messages.FloatField(80, repeated=True) - race = messages.MessageField(MetaValueListCount, 81, repeated=True) - residual_tumor = messages.MessageField(MetaValueListCount, 82, repeated=True) - sample_barcode = messages.MessageField(MetaValueListCount, 83, repeated=True) - tobacco_smoking_history = messages.MessageField(MetaValueListCount, 86, repeated=True) - total_number_of_pregnancies = messages.MessageField(MetaValueListCount, 87, repeated=True) - tumor_tissue_site = messages.MessageField(MetaValueListCount, 88, repeated=True) - tumor_pathology = messages.MessageField(MetaValueListCount, 89, repeated=True) - tumor_type = messages.MessageField(MetaValueListCount, 90, repeated=True) - weiss_venous_invasion = messages.MessageField(MetaValueListCount, 91, repeated=True) - vital_status = messages.MessageField(MetaValueListCount, 92, repeated=True) - weight = messages.MessageField(MetaValueListCount, 93, repeated=True) - year_of_initial_pathologic_diagnosis = messages.MessageField(MetaValueListCount, 94, repeated=True) - SampleTypeCode = messages.MessageField(MetaValueListCount, 95, repeated=True) - has_Illumina_DNASeq = messages.MessageField(MetaValueListCount, 96, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 97, repeated=True) - has_UNC_HiSeq_RNASeq = messages.MessageField(MetaValueListCount, 98, repeated=True) - has_BCGSC_GA_RNASeq = messages.MessageField(MetaValueListCount, 99, repeated=True) - has_UNC_GA_RNASeq = messages.MessageField(MetaValueListCount, 100, repeated=True) - has_HiSeq_miRnaSeq = messages.MessageField(MetaValueListCount, 101, repeated=True) - has_GA_miRNASeq = messages.MessageField(MetaValueListCount, 102, repeated=True) - has_RPPA = messages.MessageField(MetaValueListCount, 103, repeated=True) - has_SNP6 = messages.MessageField(MetaValueListCount, 104, repeated=True) - has_27k = messages.MessageField(MetaValueListCount, 105, repeated=True) - has_450k = messages.MessageField(MetaValueListCount, 106, repeated=True) - BMI = messages.FloatField(107, repeated=True) - - -class MetadataItem(messages.Message): - adenocarcinoma_invasion = messages.StringField(1) - age_at_diagnosis = messages.IntegerField(2) - anatomic_neoplasm_subdivision = messages.StringField(3) - avg_percent_lymphocyte_infiltration = messages.FloatField(4) - avg_percent_monocyte_infiltration = messages.FloatField(5) - avg_percent_necrosis = messages.FloatField(6) - avg_percent_neutrophil_infiltration = messages.FloatField(7) - avg_percent_normal_cells = messages.FloatField(8) - avg_percent_stromal_cells = messages.FloatField(9) - avg_percent_tumor_cells = messages.FloatField(10) - avg_percent_tumor_nuclei = messages.FloatField(11) - batch_number = messages.IntegerField(12) - bcr = messages.StringField(13) - clinical_M = messages.StringField(14) - clinical_N = messages.StringField(15) - clinical_stage = messages.StringField(16) - clinical_T = messages.StringField(17) - colorectal_cancer = messages.StringField(18) - country = messages.StringField(19) - country_of_procurement = messages.StringField(20) - days_to_birth = messages.IntegerField(21) - days_to_collection = messages.IntegerField(22) - days_to_death = messages.IntegerField(23) - days_to_initial_pathologic_diagnosis = messages.IntegerField(24) - days_to_last_followup = messages.IntegerField(25) - days_to_submitted_specimen_dx = messages.IntegerField(26) - Study = messages.StringField(27) - ethnicity = messages.StringField(28) - frozen_specimen_anatomic_site = messages.StringField(29) - gender = messages.StringField(30) - height = messages.IntegerField(31) - histological_type = messages.StringField(32) - history_of_colon_polyps = messages.StringField(33) - history_of_neoadjuvant_treatment = messages.StringField(34) - history_of_prior_malignancy = messages.StringField(35) - hpv_calls = messages.StringField(36) - hpv_status = messages.StringField(37) - icd_10 = messages.StringField(38) - icd_o_3_histology = messages.StringField(39) - icd_o_3_site = messages.StringField(40) - lymph_node_examined_count = messages.IntegerField(41) - lymphatic_invasion = messages.StringField(42) - lymphnodes_examined = messages.StringField(43) - lymphovascular_invasion_present = messages.StringField(44) - max_percent_lymphocyte_infiltration = messages.IntegerField(45) - max_percent_monocyte_infiltration = messages.IntegerField(46) - max_percent_necrosis = messages.IntegerField(47) - max_percent_neutrophil_infiltration = messages.IntegerField(48) - max_percent_normal_cells = messages.IntegerField(49) - max_percent_stromal_cells = messages.IntegerField(50) - max_percent_tumor_cells = messages.IntegerField(51) - max_percent_tumor_nuclei = messages.IntegerField(52) - menopause_status = messages.StringField(53) - min_percent_lymphocyte_infiltration = messages.IntegerField(54) - min_percent_monocyte_infiltration = messages.IntegerField(55) - min_percent_necrosis = messages.IntegerField(56) - min_percent_neutrophil_infiltration = messages.IntegerField(57) - min_percent_normal_cells = messages.IntegerField(58) - min_percent_stromal_cells = messages.IntegerField(59) - min_percent_tumor_cells = messages.IntegerField(60) - min_percent_tumor_nuclei = messages.IntegerField(61) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(62) - mononucleotide_marker_panel_analysis_status = messages.StringField(63) - neoplasm_histologic_grade = messages.StringField(64) - new_tumor_event_after_initial_treatment = messages.StringField(65) - number_of_lymphnodes_examined = messages.IntegerField(66) - number_of_lymphnodes_positive_by_he = messages.IntegerField(67) - case_barcode = messages.StringField(68) - pathologic_M = messages.StringField(69) - pathologic_N = messages.StringField(70) - pathologic_stage = messages.StringField(71) - pathologic_T = messages.StringField(72) - person_neoplasm_cancer_status = messages.StringField(73) - pregnancies = messages.StringField(74) - preservation_method = messages.StringField(75) - primary_neoplasm_melanoma_dx = messages.StringField(76) - primary_therapy_outcome_success = messages.StringField(77) - prior_dx = messages.StringField(78) - Project = messages.StringField(79) - psa_value = messages.FloatField(80) - race = messages.StringField(81) - residual_tumor = messages.StringField(82) - sample_barcode = messages.StringField(83) - tobacco_smoking_history = messages.StringField(86) - total_number_of_pregnancies = messages.IntegerField(87) - tumor_tissue_site = messages.StringField(88) - tumor_pathology = messages.StringField(89) - tumor_type = messages.StringField(90) - weiss_venous_invasion = messages.StringField(91) - vital_status = messages.StringField(92) - weight = messages.IntegerField(93) - year_of_initial_pathologic_diagnosis = messages.StringField(94) - SampleTypeCode = messages.StringField(95) - has_Illumina_DNASeq = messages.StringField(96) - has_BCGSC_HiSeq_RNASeq = messages.StringField(97) - has_UNC_HiSeq_RNASeq = messages.StringField(98) - has_BCGSC_GA_RNASeq = messages.StringField(99) - has_UNC_GA_RNASeq = messages.StringField(100) - has_HiSeq_miRnaSeq = messages.StringField(101) - has_GA_miRNASeq = messages.StringField(102) - has_RPPA = messages.StringField(103) - has_SNP6 = messages.StringField(104) - has_27k = messages.StringField(105) - has_450k = messages.StringField(106) - BMI = messages.FloatField(107) - -''' -Incoming object needs to use age and BMI that's a string (eg. 10_to_39) -''' -class IncomingMetadataItem(messages.Message): - age_at_diagnosis = messages.StringField(1, repeated=True) - anatomic_neoplasm_subdivision = messages.StringField(2, repeated=True) - avg_percent_lymphocyte_infiltration = messages.FloatField(3, repeated=True) - avg_percent_monocyte_infiltration = messages.FloatField(4, repeated=True) - avg_percent_necrosis = messages.FloatField(5, repeated=True) - avg_percent_neutrophil_infiltration = messages.FloatField(6, repeated=True) - avg_percent_normal_cells = messages.FloatField(7, repeated=True) - avg_percent_stromal_cells = messages.FloatField(8, repeated=True) - avg_percent_tumor_cells = messages.FloatField(9, repeated=True) - avg_percent_tumor_nuclei = messages.FloatField(10, repeated=True) - batch_number = messages.IntegerField(11, repeated=True) - bcr = messages.StringField(12, repeated=True) - clinical_M = messages.StringField(13, repeated=True) - clinical_N = messages.StringField(14, repeated=True) - clinical_stage = messages.StringField(15, repeated=True) - clinical_T = messages.StringField(16, repeated=True) - colorectal_cancer = messages.StringField(17, repeated=True) - country = messages.StringField(18, repeated=True) - days_to_birth = messages.IntegerField(19, repeated=True) - days_to_collection = messages.IntegerField(20, repeated=True) - days_to_death = messages.IntegerField(21, repeated=True) - days_to_initial_pathologic_diagnosis = messages.IntegerField(22, repeated=True) - days_to_last_followup = messages.IntegerField(23, repeated=True) - days_to_submitted_specimen_dx = messages.IntegerField(24, repeated=True) - Study = messages.StringField(25, repeated=True) - ethnicity = messages.StringField(26, repeated=True) - frozen_specimen_anatomic_site = messages.StringField(27, repeated=True) - gender = messages.StringField(28, repeated=True) - height = messages.IntegerField(29, repeated=True) - histological_type = messages.StringField(30, repeated=True) - history_of_colon_polyps = messages.StringField(31, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(32, repeated=True) - history_of_prior_malignancy = messages.StringField(33, repeated=True) - hpv_calls = messages.StringField(34, repeated=True) - hpv_status = messages.StringField(35, repeated=True) - icd_10 = messages.StringField(36, repeated=True) - icd_o_3_histology = messages.StringField(37, repeated=True) - icd_o_3_site = messages.StringField(38, repeated=True) - lymphatic_invasion = messages.StringField(39, repeated=True) - lymphnodes_examined = messages.StringField(40, repeated=True) - lymphovascular_invasion_present = messages.StringField(41, repeated=True) - max_percent_lymphocyte_infiltration = messages.IntegerField(42, repeated=True) - max_percent_monocyte_infiltration = messages.IntegerField(43, repeated=True) - max_percent_necrosis = messages.IntegerField(44, repeated=True) - max_percent_neutrophil_infiltration = messages.IntegerField(45, repeated=True) - max_percent_normal_cells = messages.IntegerField(46, repeated=True) - max_percent_stromal_cells = messages.IntegerField(47, repeated=True) - max_percent_tumor_cells = messages.IntegerField(48, repeated=True) - max_percent_tumor_nuclei = messages.IntegerField(49, repeated=True) - menopause_status = messages.StringField(50, repeated=True) - min_percent_lymphocyte_infiltration = messages.IntegerField(51, repeated=True) - min_percent_monocyte_infiltration = messages.IntegerField(52, repeated=True) - min_percent_necrosis = messages.IntegerField(53, repeated=True) - min_percent_neutrophil_infiltration = messages.IntegerField(54, repeated=True) - min_percent_normal_cells = messages.IntegerField(55, repeated=True) - min_percent_stromal_cells = messages.IntegerField(56, repeated=True) - min_percent_tumor_cells = messages.IntegerField(57, repeated=True) - min_percent_tumor_nuclei = messages.IntegerField(58, repeated=True) - mononucleotide_and_dinucleotide_marker_panel_analysis_status = messages.StringField(59, repeated=True) - mononucleotide_marker_panel_analysis_status = messages.StringField(60, repeated=True) - neoplasm_histologic_grade = messages.StringField(61, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(62, repeated=True) - number_of_lymphnodes_examined = messages.IntegerField(63, repeated=True) - number_of_lymphnodes_positive_by_he = messages.IntegerField(64, repeated=True) - case_barcode = messages.StringField(65, repeated=True) - pathologic_M = messages.StringField(66, repeated=True) - pathologic_N = messages.StringField(67, repeated=True) - pathologic_stage = messages.StringField(68, repeated=True) - pathologic_T = messages.StringField(69, repeated=True) - person_neoplasm_cancer_status = messages.StringField(70, repeated=True) - pregnancies = messages.StringField(71, repeated=True) - primary_neoplasm_melanoma_dx = messages.StringField(72, repeated=True) - primary_therapy_outcome_success = messages.StringField(73, repeated=True) - prior_dx = messages.StringField(74, repeated=True) - Project = messages.StringField(75, repeated=True) - psa_value = messages.FloatField(76, repeated=True) - race = messages.StringField(77, repeated=True) - residual_tumor = messages.StringField(78, repeated=True) - sample_barcode = messages.StringField(79, repeated=True) - tobacco_smoking_history = messages.StringField(80, repeated=True) - tumor_tissue_site = messages.StringField(81, repeated=True) - tumor_type = messages.StringField(82, repeated=True) - weiss_venous_invasion = messages.StringField(83, repeated=True) - vital_status = messages.StringField(84, repeated=True) - weight = messages.IntegerField(85, repeated=True) - year_of_initial_pathologic_diagnosis = messages.StringField(86, repeated=True) - SampleTypeCode = messages.StringField(87, repeated=True) - has_Illumina_DNASeq = messages.StringField(88, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(89, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(90, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(91, repeated=True) - has_UNC_GA_RNASeq = messages.StringField(92, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(93, repeated=True) - has_GA_miRNASeq = messages.StringField(94, repeated=True) - has_RPPA = messages.StringField(95, repeated=True) - has_SNP6 = messages.StringField(96, repeated=True) - has_27k = messages.StringField(97, repeated=True) - has_450k = messages.StringField(98, repeated=True) - BMI = messages.StringField(99, repeated=True) - -class MetadataAttributeValues(messages.Message): - name = messages.StringField(1) - id = messages.StringField(2) - values = messages.MessageField(MetaValueListCount, 3, repeated=True) - total = messages.IntegerField(4) - -class MetadataItemList(messages.Message): - items = messages.MessageField(MetadataItem, 1, repeated=True) - count = messages.MessageField(MetaAttrValuesList, 2) - total = messages.IntegerField(3) - -class MetadataCountsItem(messages.Message): - count = messages.MessageField(MetadataAttributeValues, 1, repeated=True) - total = messages.IntegerField(2) - -class MetaDomainsList(messages.Message): - gender = messages.StringField(1, repeated=True) - history_of_neoadjuvant_treatment = messages.StringField(2, repeated=True) - country = messages.StringField(3, repeated=True) - Study = messages.StringField(4, repeated=True) - ethnicity = messages.StringField(5, repeated=True) - histological_type = messages.StringField(6, repeated=True) - icd_10 = messages.StringField(7, repeated=True) - icd_o_3_histology = messages.StringField(8, repeated=True) - icd_o_3_site = messages.StringField(9, repeated=True) - new_tumor_event_after_initial_treatment = messages.StringField(10, repeated=True) - neoplasm_histologic_grade = messages.StringField(11, repeated=True) - pathologic_N = messages.StringField(12, repeated=True) - pathologic_T = messages.StringField(13, repeated=True) - pathologic_stage = messages.StringField(14, repeated=True) - person_neoplasm_cancer_status = messages.StringField(15, repeated=True) - prior_dx = messages.StringField(16, repeated=True) - Project = messages.StringField(17, repeated=True) - race = messages.StringField(18, repeated=True) - residual_tumor = messages.StringField(19, repeated=True) - SampleTypeCode = messages.StringField(20, repeated=True) - tumor_tissue_site = messages.StringField(21, repeated=True) - tumor_type = messages.StringField(22, repeated=True) - vital_status = messages.StringField(23, repeated=True) - has_Illumina_DNASeq = messages.StringField(24, repeated=True) - has_BCGSC_HiSeq_RNASeq = messages.StringField(25, repeated=True) - has_UNC_HiSeq_RNASeq = messages.StringField(26, repeated=True) - has_BCGSC_GA_RNASeq = messages.StringField(27, repeated=True) - has_HiSeq_miRnaSeq = messages.StringField(28, repeated=True) - has_GA_miRNASeq = messages.StringField(29, repeated=True) - has_RPPA = messages.StringField(30, repeated=True) - has_SNP6 = messages.StringField(31, repeated=True) - has_27k = messages.StringField(32, repeated=True) - has_450k = messages.StringField(33, repeated=True) - - -class SampleBarcodeItem(messages.Message): - sample_barcode = messages.StringField(1) - study_id = messages.IntegerField(2) - -class MetadataAttr(messages.Message): - attribute = messages.StringField(1) - code = messages.StringField(2) - spec = messages.StringField(3) - key = messages.StringField(4) - - -class MetadataAttrList(messages.Message): - items = messages.MessageField(MetadataAttr, 1, repeated=True) - count = messages.IntegerField(2) - - -class SampleBarcodeList(messages.Message): - items = messages.MessageField(SampleBarcodeItem, 1, repeated=True) - count = messages.IntegerField(2) - - -class MetadataPlatformItem(messages.Message): - DNAseq_data = messages.StringField(1) - cnvrPlatform = messages.StringField(2) - gexpPlatform = messages.StringField(3) - methPlatform = messages.StringField(4) - mirnPlatform = messages.StringField(5) - rppaPlatform = messages.StringField(6) - - -class MetadataPlatformItemList(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - - -class MetadataCountsPlatformItem(messages.Message): - items = messages.MessageField(MetadataPlatformItem, 1, repeated=True) - count = messages.MessageField(MetadataAttributeValues, 2, repeated=True) - participants = messages.IntegerField(3) - total = messages.IntegerField(4) - - -def createDataItem(data, selectors): - if len(selectors): - item = MetadataItem() - for attr in selectors: - attr = attr.encode('utf-8') - if data[attr] is not None: - if type(data[attr]) is not long and type(data[attr]) is not int: - item.__setattr__(attr, data[attr].encode('utf-8')) - if attr.startswith('has_'): - item.__setattr__(attr, str(bool(data[attr]))) - else: - item.__setattr__(attr, data[attr]) - else: - item.__setattr__(attr, None) - return item - - -class MetadataDomainItem(messages.Message): - attribute = messages.StringField(1) - domains = messages.StringField(2, repeated=True) - - -def submit_bigquery_job(bq_service, project_id, query_body, batch=False): - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid4()) - }, - 'configuration': { - 'query': { - 'query': query_body, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - - return bq_service.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=5) - - -def is_bigquery_job_finished(bq_service, project_id, job_id): - job = bq_service.jobs().get(projectId=project_id, - jobId=job_id).execute() - - return job['status']['state'] == 'DONE' - - -def get_bq_job_results(bq_service, job_reference): - result = [] - page_token = None - - while True: - page = bq_service.jobs().getQueryResults( - pageToken=page_token, - **job_reference).execute(num_retries=2) - - if int(page['totalRows']) == 0: - break - - rows = page['rows'] - result.extend(rows) - - page_token = page.get('pageToken') - if not page_token: - break - - return result - - -def generateSQLQuery(request): - db = sql_connection() - query_dict = {} - select = '*' - sample_ids = () - selector_list = [] - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s AND project_id IS NULL;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - cursor.close() - - except (TypeError, IndexError) as e: - print e - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - if request.__getattribute__('selectors') is not None and len(request.__getattribute__('selectors')): - select = ','.join(request.selectors) - selector_list = select.split(',') # request.selectors - - # Get the list of valid parameters from request - for key, value in MetadataItem.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) is not None: - if key.startswith('has_'): - query_dict[key] = 1 if request.__getattribute__(key) == 'True' else 0 - else: - query_dict[key] = request.__getattribute__(key).replace('_', ' ') # values coming in with _ replaced with spaces - - query_str = 'SELECT %s FROM metadata_samples' % select - value_tuple = () - if len(query_dict) > 0: - where_clause = build_where_clause(query_dict) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - if request.__getattribute__('limit') is not None: - query_str += ' LIMIT %s' % request.__getattribute__('limit') - - query_str += ';' - db.close() - - return query_str, value_tuple, selector_list - - -class MetadataDomainList(messages.Message): - items = messages.MessageField(MetadataDomainItem, 1, repeated=True) - - -def normalize_metadata_ages(ages): - result = [] - new_age_list = {'10 to 39': 0, '40 to 49': 0, '50 to 59': 0, '60 to 69': 0, '70 to 79': 0, 'Over 80': 0, 'None': 0} - for age in ages: - if type(age) != dict: - - if age.value != 'None': - int_age = float(age.value) - if int_age < 40: - new_age_list['10 to 39'] += int(age.count) - elif int_age < 50: - new_age_list['40 to 49'] += int(age.count) - elif int_age < 60: - new_age_list['50 to 59'] += int(age.count) - elif int_age < 70: - new_age_list['60 to 69'] += int(age.count) - elif int_age < 80: - new_age_list['70 to 79'] += int(age.count) - else: - new_age_list['Over 80'] += int(age.count) - else: - new_age_list['None'] += int(age.count) - - for key, value in new_age_list.items(): - result.append({'count': value, 'value': key}) - return result - - -class PlatformCount(messages.Message): - platform = messages.StringField(1) - count = messages.IntegerField(2) - - -class FileDetails(messages.Message): - sample = messages.StringField(1) - filename = messages.StringField(2) - pipeline = messages.StringField(3) - platform = messages.StringField(4) - datalevel = messages.StringField(5) - datatype = messages.StringField(6) - gg_readgroupset_id = messages.StringField(7) - cloudstorage_location = messages.StringField(8) - access = messages.StringField(9) - - -class SampleFiles(messages.Message): - total_file_count = messages.IntegerField(1) - page = messages.IntegerField(2) - platform_count_list = messages.MessageField(PlatformCount, 3, repeated=True) - file_list = messages.MessageField(FileDetails, 4, repeated=True) - - -class SampleFileCount(messages.Message): - sample_id = messages.StringField(1) - count = messages.IntegerField(2) - - -class CohortFileCountSampleList(messages.Message): - sample_list = messages.MessageField(SampleFileCount, 1, repeated=True) - - -class IncomingPlatformSelection(messages.Message): - ABSOLiD_DNASeq = messages.StringField(1) - Genome_Wide_SNP_6 = messages.StringField(2) - HumanMethylation27 = messages.StringField(3) - HumanMethylation450 = messages.StringField(4) - IlluminaGA_DNASeq = messages.StringField(5) - IlluminaGA_DNASeq_automated = messages.StringField(6) - IlluminaGA_DNASeq_Cont_automated = messages.StringField(7) - IlluminaGA_DNASeq_curated = messages.StringField(8) - IlluminaGA_miRNASeq = messages.StringField(9) - IlluminaGA_None = messages.StringField(10) - IlluminaGA_RNASeq = messages.StringField(11) - IlluminaGA_RNASeqV2 = messages.StringField(12) - IlluminaHiSeq_DNASeq = messages.StringField(13) - IlluminaHiSeq_DNASeq_automated = messages.StringField(14) - IlluminaHiSeq_DNASeq_Cont_automated = messages.StringField(15) - IlluminaHiSeq_miRNASeq = messages.StringField(16) - IlluminaHiSeq_None = messages.StringField(17) - IlluminaHiSeq_RNASeq = messages.StringField(18) - IlluminaHiSeq_RNASeqV2 = messages.StringField(19) - IlluminaHiSeq_TotalRNASeqV2 = messages.StringField(20) - IlluminaMiSeq_DNASeq = messages.StringField(21) - IlluminaMiSeq_None = messages.StringField(22) - LifeIonTorrentPGM_None = messages.StringField(23) - LifeIonTorrentProton_None = messages.StringField(24) - MDA_RPPA_Core = messages.StringField(25) - microsat_i = messages.StringField(26) - Mixed_DNASeq_Cont = messages.StringField(27) - Mixed_DNASeq_Cont_curated = messages.StringField(28) - Mixed_DNASeq_curated = messages.StringField(29) - RocheGSFLX_DNASeq = messages.StringField(30) - - -class IncomingMetadataCount(messages.Message): - filters = messages.StringField(1) - cohort_id = messages.IntegerField(2) - token = messages.StringField(3) - - -def get_current_user(request): - """ - Returns a Django_User object from either endpoints.get_current_user() or an access token. - Anytime this is used in an endpoint, request_finished.send(self) must be used - """ - user_email = None - access_token = request.__getattribute__('token') - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - django.setup() - try: - if access_token is not None and user_email is None: - social_account_id = SocialToken.objects.get(token=access_token).account_id - user_id = SocialAccount.objects.get(id=social_account_id).user_id - return Django_User.objects.get(id=user_id) - elif user_email is not None: - return Django_User.objects.get(email=user_email) - else: - return None - except (ObjectDoesNotExist, MultipleObjectsReturned), e: - logger.warn(e) - return None - - -# TODO: needs to be refactored to use other samples tables -def get_participant_list(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - return results - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_list') - -# TODO: needs to be refactored to use other samples tables -def get_participant_count(sample_ids): - - db = sql_connection() - cursor = None - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - participant_query = 'SELECT COUNT(DISTINCT case_barcode) AS ParticipantCount FROM metadata_samples WHERE sample_barcode IN (' - first = True - samples = () - for barcode in sample_ids: - samples += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - count = 0; - cursor.execute(participant_query, samples) - for row in cursor.fetchall(): - count = row['ParticipantCount'] - - return count - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in get_participant_count') - - -def count_metadata(user, cohort_id=None, sample_ids=None, filters=None): - counts_and_total = {} - sample_tables = {} - valid_attrs = {} - project_ids = () - table_key_map = {} - - if filters is None: - filters = {} - - if sample_ids is None: - sample_ids = {} - - for key in sample_ids: - samples_by_project = sample_ids[key] - sample_ids[key] = { - 'sample_barcode': build_where_clause({'sample_barcode': samples_by_project}), - } - - db = sql_connection() - django.setup() - - try: - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'sample_ids': None} - if sample_ids and None in sample_ids: - sample_tables['metadata_samples']['sample_ids'] = sample_ids[None] - - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - if row['attribute'] in METADATA_SHORTLIST: - valid_attrs[row['spec'] + ':' + row['attribute']] = { - 'name': row['attribute'], - 'tables': ('metadata_samples',), - 'sample_ids': None - } - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = {'sample_ids': None} - if sample_ids and project.id in sample_ids: - sample_tables[tables.metadata_samples_table]['sample_ids'] = sample_ids[project.id] - - features = User_Feature_Definitions.objects.filter(project__in=project_ids) - for feature in features: - if ' ' in feature.feature_name: - # It is not a column name and comes from molecular data, ignore it - continue - - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name, 'tables': (), 'sample_ids': None} - - for tables in User_Data_Tables.objects.filter(project_id=feature.project_id): - valid_attrs[key]['tables'] += (tables.metadata_samples_table,) - - if tables.metadata_samples_table not in table_key_map: - table_key_map[tables.metadata_samples_table] = {} - table_key_map[tables.metadata_samples_table][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - - if sample_ids and feature.project_id in sample_ids: - valid_attrs[key]['sample_ids'] = sample_ids[feature.project_id] - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - # For filters with no tables at this point, assume its the TCGA metadata_samples table - for key, obj in filters.items(): - if not obj['tables']: - filters[key]['tables'].append('metadata_samples') - - resulting_samples = {} - - # Loop through the features - for key, feature in valid_attrs.items(): - # Get a count for each feature - table_values = {} - feature['total'] = 0 - for table in feature['tables']: - # Check if the filters make this table 0 anyway - # We do this to avoid SQL errors for columns that don't exist - should_be_queried = True - if cohort_id and sample_tables[table]['sample_ids'] is None: - should_be_queried = False - - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - # Build Filter Where Clause - filter_copy = copy.deepcopy(filters) - key_map = table_key_map[table] if table in table_key_map else False - where_clause = build_where_clause(filter_copy, alt_key_map=key_map) - col_name = feature['name'] - if key_map and key in key_map: - col_name = key_map[key] - - cursor = db.cursor() - if should_be_queried: - # Query the table for counts and values - query = ('SELECT DISTINCT %s, COUNT(1) as count FROM %s') % (col_name, table) - sample_query = ('SELECT DISTINCT %s AS sample_id FROM %s') % ('sample_barcode', table) - query_clause = '' - if where_clause['query_str']: - query_clause = ' WHERE ' + where_clause['query_str'] - if sample_tables[table]['sample_ids']: - barcode_key = 'sample_barcode' - addt_cond = sample_tables[table]['sample_ids'][barcode_key]['query_str'] - if addt_cond and where_clause['query_str']: - query_clause += ' AND ' + addt_cond - elif addt_cond: - query_clause = ' WHERE ' + addt_cond - where_clause['value_tuple'] += sample_tables[table]['sample_ids'][barcode_key]['value_tuple'] - query += query_clause + (' GROUP BY %s ' % col_name) - sample_query += query_clause - - logger.debug("In api count_metadata, executing query "+query) - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - table_values[row[0]] += int(row[1]) - feature['total'] += int(row[1]) - - cursor.execute(sample_query, where_clause['value_tuple']) - for row in cursor.fetchall(): - resulting_samples[row[0]] = 1 - else: - # Just get the values so we can have them be 0 - cursor.execute(('SELECT DISTINCT %s FROM %s') % (col_name, table)) - for row in cursor.fetchall(): - if not row[0] in table_values: - table_values[row[0]] = 0 - - cursor.close() - - feature['values'] = table_values - - sample_set = () - for sample in resulting_samples: - sample_set += (sample,) - - counts_and_total['participants'] = get_participant_count(sample_set) if sample_set.__len__() > 0 else 0 - counts_and_total['counts'] = [] - counts_and_total['total'] = 0 - for key, feature in valid_attrs.items(): - value_list = [] - - # Special case for age ranges and BMI - if key == 'CLIN:age_at_diagnosis': - feature['values'] = normalize_ages(feature['values']) - elif key == 'CLIN:BMI': - feature['values'] = normalize_BMI(feature['values']) - - - for value, count in feature['values'].items(): - if feature['name'].startswith('has_'): - value = 'True' if value else 'False' - - value_list.append(MetaValueListCount(value=str(value), count=count)) - - counts_and_total['counts'].append( - MetadataAttributeValues(name=feature['name'], values=value_list, id=key, total=feature['total'])) - if feature['total'] > counts_and_total['total']: - counts_and_total['total'] = feature['total'] - - db.close() - - return counts_and_total - - except (Exception) as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in count_metadata.') - - -def query_samples_and_studies(parameter, bucket_by=None): - - query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - if bucket_by is not None and bucket_by not in query_str: - logging.error("Cannot group barcodes: column '" + bucket_by + - "' not found in query string '"+query_str+"'. Barcodes will not be grouped.") - bucket_by = None - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, (parameter,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs for cohort '" + parameter + "': " + (stop-start).__str__()) - - samples = () - - if bucket_by is not None: - samples = {} - - for row in cursor.fetchall(): - if bucket_by is not None: - if row[bucket_by] not in samples: - samples[row[bucket_by]] = [] - samples[row[bucket_by]].append(row['sample_barcode']) - else: - samples += ({"sample_id": row['sample_barcode'], "project_id": row['project_id']},) - cursor.close() - db.close() - - return samples - - except (TypeError, IndexError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - -""" -Metadata Endpoints v3 - -Includes User Uploaded Data -""" -Meta_Endpoints_v3 = endpoints.api(name='meta_api', version='v3', - description='Retrieve metadata information relating to projects, cohorts, and other data', - allowed_client_ids=[INSTALLED_APP_CLIENT_ID, endpoints.API_EXPLORER_CLIENT_ID]) - -@Meta_Endpoints_v3.api_class(resource_name='meta_endpoints') -class Meta_Endpoints_API_v3(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer( - MetadataAttr, - token=messages.StringField(4), - ) - @endpoints.method(GET_RESOURCE, MetadataAttrList, - path='attributes', http_method='GET', - name='meta.attr_list') - def metadata_attr_list(self, request): - - cursor = None - db = None - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - query_dict = {} - value_tuple = () - for key, value in MetadataAttr.__dict__.items(): - if not key.startswith('_'): - if request.__getattribute__(key) != None: - query_dict[key] = request.__getattribute__(key) - - if len(query_dict) == 0: - query_str = 'SELECT * FROM metadata_attr' - else: - query_str = 'SELECT * FROM metadata_attr where' - where_clause = build_where_clause(query_dict) - query_str += where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(query_str, value_tuple) - data = [] - for row in cursor.fetchall(): - data.append(MetadataAttr(attribute=str(row['attribute']), - code=str(row['code']), - spec=str(row['spec']), - key=str(row['spec']) + ':' + str(row['attribute']) - )) - - if user: - studies = Project.get_user_studies(user) - feature_defs = User_Feature_Definitions.objects.filter(project__in=studies) - for feature in feature_defs: - data_table = User_Data_Tables.objects.get(project=feature.project).metadata_samples_table - name = feature.feature_name - key = 'study:' + str(feature.project_id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - - data.append(MetadataAttr(attribute=name, - code='N' if feature.is_numeric else 'C', - spec='USER', - key=key - )) - - cursor.close() - db.close() - request_finished.send(self) - return MetadataAttrList(items=data, count=len(data)) - - except (IndexError, TypeError): - raise endpoints.InternalServerErrorException('Error retrieving attribute list') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsItem, - path='metadata_counts', http_method='POST', - name='meta.metadata_counts') - def metadata_counts(self, request): - filters = {} - sample_ids = None - cohort_id = None - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for reqFilter in tmp: - key = reqFilter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(reqFilter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - start = time.time() - counts_and_totals = count_metadata(user, cohort_id, sample_ids, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts" - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - request_finished.send(self) - return MetadataCountsItem(count=counts_and_totals['counts'], total=counts_and_totals['total']) - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - @endpoints.method(POST_RESOURCE, SampleBarcodeList, - path='metadata_sample_list', http_method='POST', - name='meta.metadata_sample_list') - def metadata_list(self, request): - filters = {} - valid_attrs = {} - sample_tables = {} - table_key_map = {} - sample_ids = None - project_ids = () - cohort_id = None - cursor = None - db = None - mutation_filters = None - mutation_results = {} - - user = get_current_user(request) - if user is None: - request_finished.send(self) - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if 'MUT:' in key: - if not mutation_filters: - mutation_filters = {} - if not key in mutation_filters: - mutation_filters[key] = [] - mutation_filters[key].append(filter['value']) - else: - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - request_finished.send(self) - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - django.setup() - - # TODO enable filtering based off of this - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_ids = query_samples_and_studies(cohort_id, 'project_id') - - if mutation_filters: - mutation_where_clause = build_where_clause(mutation_filters) - print >> sys.stdout, mutation_where_clause - cohort_join_str = '' - cohort_where_str = '' - bq_cohort_table = '' - bq_cohort_dataset = '' - cohort = '' - query_template = None - - if cohort_id is not None: - query_template = \ - ("SELECT ct.sample_barcode" - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] ct" - " JOIN (SELECT Tumor_SampleBarcode AS barcode " - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY barcode) mt" - " ON mt.barcode = ct.sample_barcode" - " WHERE ct.cohort_id = {cohort};") - bq_cohort_table = settings.BIGQUERY_COHORT_TABLE_ID - bq_cohort_dataset = settings.COHORT_DATASET_ID - cohort = cohort_id - else: - query_template = \ - ("SELECT Tumor_SampleBarcode" - " FROM [{project_name}:{dataset_name}.{table_name}]" - " WHERE " + mutation_where_clause['big_query_str'] + - " GROUP BY Tumor_SampleBarcode; ") - - params = mutation_where_clause['value_tuple'][0] - - query = query_template.format(dataset_name=settings.BIGQUERY_DATASET, - project_name=settings.BIGQUERY_PROJECT_NAME, - table_name="Somatic_Mutation_calls", hugo_symbol=str(params['gene']), - var_class=params['var_class'], cohort_dataset=bq_cohort_dataset, - cohort_table=bq_cohort_table, cohort=cohort) - - bq_service = authorize_credentials_with_Google() - query_job = submit_bigquery_job(bq_service, settings.BQ_PROJECT_ID, query) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - retries = 0 - - while not job_is_done and retries < 10: - retries += 1 - sleep(1) - job_is_done = is_bigquery_job_finished(bq_service, settings.BQ_PROJECT_ID, - query_job['jobReference']['jobId']) - - results = get_bq_job_results(bq_service, query_job['jobReference']) - - # for-each result, add to list - - if results.__len__() > 0: - for barcode in results: - mutation_results[str(barcode['f'][0]['v'])] = 1 - - else: - print >> sys.stdout, "Mutation filter result was empty!" - # Put in one 'not found' entry to zero out the rest of the queries - barcodes = ['NONE_FOUND', ] - - # Add TCGA attributes to the list of available attributes - if 'user_studies' not in filters or 'tcga' in filters['user_studies']['values']: - sample_tables['metadata_samples'] = {'features': {}, 'barcode': 'sample_barcode', 'project_id': None} - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute('SELECT attribute, spec FROM metadata_attr') - for row in cursor.fetchall(): - key = row['spec'] + ':' + row['attribute'] - valid_attrs[key] = {'name': row['attribute']} - sample_tables['metadata_samples']['features'][key] = row['attribute'] - if key in filters: - filters[key]['tables'] += ('metadata_samples',) - cursor.close() - - # If we have a user, get a list of valid studies - if user: - for project in Project.get_user_studies(user): - if 'user_studies' not in filters or project.id in filters['user_studies']['values']: - project_ids += (project.id,) - - # Add all tables from each project - for tables in User_Data_Tables.objects.filter(project=project): - sample_tables[tables.metadata_samples_table] = { - 'features':{}, - 'barcode':'sample_barcode', - 'project_id': project.id - } - - # Record features that should be in each sample table so we can know how and when we need to query - for feature in User_Feature_Definitions.objects.filter(project=project): - name = feature.feature_name - key = 'study:' + str(project.id) + ':' + name - - if feature.shared_map_id: - key = feature.shared_map_id - name = feature.shared_map_id.split(':')[-1] - - if key not in valid_attrs: - valid_attrs[key] = {'name': name} - - for tables in User_Data_Tables.objects.filter(project=feature.project_id): - sample_tables[tables.metadata_samples_table]['features'][key] = feature.feature_name - - if key in filters: - filters[key]['tables'] += (tables.metadata_samples_table,) - else: - print "User not authenticated with Metadata Endpoint API" - - # Now that we're through the Studies filtering area, delete it so it doesn't get pulled into a query - if 'user_studies' in filters: - del filters['user_studies'] - - results = [] - # Loop through the sample tables - for table, table_settings in sample_tables.items(): - # Make sure we should run the query here, or if we have filters that won't return anything, skip - should_be_queried = True - for key, filter in filters.items(): - if table not in filter['tables']: - should_be_queried = False - break - - if not should_be_queried: - continue - - filter_copy = copy.deepcopy(filters) - where_clause = build_where_clause(filter_copy, table_settings['features']) - query = 'SELECT DISTINCT %s FROM %s' % (table_settings['barcode'], table) - if where_clause['query_str']: - query += ' WHERE ' + where_clause['query_str'] - cursor = db.cursor() - cursor.execute(query, where_clause['value_tuple']) - for row in cursor.fetchall(): - project_id = table_settings['project_id'] - if cohort_id and (project_id not in sample_ids or row[0] not in sample_ids[project_id]): - # This barcode was not in our cohort's list of barcodes, skip it - continue - if mutation_filters: - if row[0] in mutation_results: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id'])) - else: - results.append(SampleBarcodeItem(sample_barcode=row[0], study_id=table_settings['project_id']) ) - cursor.close() - - db.close() - request_finished.send(self) - return SampleBarcodeList( items=results, count=len(results) ) - - GET_RESOURCE = endpoints.ResourceContainer( - cohort_id=messages.IntegerField(1, required=True), - ) - @endpoints.method(GET_RESOURCE, SampleBarcodeList, - path='metadata_participant_list', http_method='GET', - name='meta.metadata_participant_list') - def metadata_participant_list(self, request): - cursor = None - db = None - - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode, project_id FROM cohorts_samples WHERE cohort_id=%s;' - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(sample_query_str, (cohort_id,)) - sample_ids = [] - - for row in cursor.fetchall(): - sample_ids.append(row['sample_barcode']) - - participant_query = 'SELECT DISTINCT case_barcode from metadata_samples where sample_barcode in (' - first = True - value_tuple = () - for barcode in sample_ids: - value_tuple += (barcode,) - if first: - participant_query += '%s' - first = False - else: - participant_query += ',%s' - - participant_query += ');' - results = [] - cursor.execute(participant_query, value_tuple) - for row in cursor.fetchall(): - results.append(SampleBarcodeItem(sample_barcode=row['case_barcode'], study_id=0)) - - cursor.close() - db.close() - return SampleBarcodeList(items=results, count=len(results)) - - except (TypeError, IndexError) as e: - raise endpoints.NotFoundException('Error in retrieving barcodes.') - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataPlatformItemList, - path='metadata_platform_list', http_method='POST', - name='meta.metadata_platform_list') - def metadata_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - cursor = None - - if request.__getattribute__('filters')is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values':[], 'tables':[] } - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException('Filters must be a valid JSON formatted array with objects containing both key and value properties') - - db = sql_connection() - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - sample_query_str = 'SELECT sample_barcode FROM cohorts_samples WHERE cohort_id=%s;' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(sample_query_str, (cohort_id,)) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query sample IDs in metadata_platform_list for cohort '" + cohort_id + "': " + (stop - start).__str__()) - sample_ids = () - - for row in cursor.fetchall(): - sample_ids += (row['sample_barcode'],) - - except (TypeError, IndexError) as e: - print e - cursor.close() - db.close() - raise endpoints.NotFoundException('Error in retrieving barcodes.') - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_platform_list for cohort '" + str(request.cohort_id) + "': " + (stop - start).__str__()) - data = [] - for row in cursor.fetchall(): - - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataPlatformItemList(items=data) - - except (IndexError, TypeError) as e: - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Sample not found.') - - POST_RESOURCE = endpoints.ResourceContainer(IncomingMetadataCount) - - @endpoints.method(POST_RESOURCE, MetadataCountsPlatformItem, - path='metadata_counts_platform_list', http_method='POST', - name='meta.metadata_counts_platform_list') - def metadata_counts_platform_list(self, request): - """ Used by the web application.""" - filters = {} - sample_ids = None - samples_by_project = None - cohort_id = None - participants = 0 - user = get_current_user(request) - - if request.__getattribute__('filters') is not None: - try: - tmp = json.loads(request.filters) - for filter in tmp: - key = filter['key'] - if key not in filters: - filters[key] = {'values': [], 'tables': []} - filters[key]['values'].append(filter['value']) - - except Exception, e: - print traceback.format_exc() - raise endpoints.BadRequestException( - 'Filters must be a valid JSON formatted array with objects containing both key and value properties') - - # Check for passed in saved search id - if request.__getattribute__('cohort_id') is not None: - cohort_id = str(request.cohort_id) - samples = query_samples_and_studies(cohort_id, ) - - sample_ids = () - samples_by_project = {} - - for sample in samples: - sample_ids += (sample['sample_id'],) - if sample['project_id'] not in samples_by_project: - samples_by_project[sample['project_id']] = [] - samples_by_project[sample['project_id']].append(sample['sample_id']) - - participants = get_participant_count(sample_ids) - - start = time.time() - counts_and_total = count_metadata(user, cohort_id, samples_by_project, filters) - stop = time.time() - logger.debug( - "[BENCHMARKING] In api/metadata, time to query metadata_counts " - + (" for cohort "+cohort_id if cohort_id is not None else "") - + (" and" if cohort_id is not None and filters.__len__() > 0 else "") - + (" filters "+filters.__str__() if filters.__len__() > 0 else "") - + ": " + (stop - start).__str__() - ) - - db = sql_connection() - - query_str = "SELECT " \ - "IF(has_Illumina_DNASeq=1, " \ - "'Yes', 'None'" \ - ") AS DNAseq_data," \ - "IF (has_SNP6=1, 'Genome_Wide_SNP_6', 'None') as cnvrPlatform," \ - "CASE" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'HiSeq/BCGSC'" \ - " WHEN has_BCGSC_HiSeq_RNASeq=1 and has_UNC_HiSeq_RNASeq=1" \ - " THEN 'HiSeq/BCGSC and UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=1" \ - " THEN 'GA and HiSeq/UNC V2'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=0 and has_BCGSC_GA_RNASeq=1 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and GA/BCGSC'" \ - " WHEN has_UNC_HiSeq_RNASeq=1 and has_BCGSC_HiSeq_RNASeq=1 and has_BCGSC_GA_RNASeq=0 and has_UNC_GA_RNASeq=0" \ - " THEN 'HiSeq/UNC V2 and BCGSC'" \ - " WHEN has_BCGSC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/BCGSC'" \ - " WHEN has_UNC_GA_RNASeq=1 and has_UNC_HiSeq_RNASeq=0" \ - " THEN 'GA/UNC V2'" \ - " ELSE 'None'" \ - "END AS gexpPlatform," \ - "CASE " \ - " WHEN has_27k=1 and has_450k=0" \ - " THEN 'HumanMethylation27'" \ - " WHEN has_27k=0 and has_450k=1" \ - " THEN 'HumanMethylation450'" \ - " WHEN has_27k=1 and has_450k=1" \ - " THEN '27k and 450k'" \ - " ELSE 'None'" \ - "END AS methPlatform," \ - "CASE " \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=0" \ - " THEN 'IlluminaHiSeq_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=0 and has_GA_miRNASeq=1" \ - " THEN 'IlluminaGA_miRNASeq'" \ - " WHEN has_HiSeq_miRnaSeq=1 and has_GA_miRNASeq=1" \ - " THEN 'GA and HiSeq'" \ - " ELSE 'None'" \ - "END AS mirnPlatform," \ - "IF (has_RPPA=1, 'MDA_RPPA_Core', 'None') AS rppaPlatform " \ - "FROM metadata_samples " - - value_tuple = () - if len(filters) > 0: - where_clause = build_where_clause(filters) - query_str += ' WHERE ' + where_clause['query_str'] - value_tuple = where_clause['value_tuple'] - - if sample_ids: - if query_str.rfind('WHERE') >= 0: - query_str += ' and sample_barcode in %s' % (sample_ids,) - else: - query_str += ' WHERE sample_barcode in %s' % (sample_ids,) - - query_str += ';' - - data = [] - - try: - cursor = db.cursor(MySQLdb.cursors.DictCursor) - start = time.time() - cursor.execute(query_str, value_tuple) - stop = time.time() - logger.debug("[BENCHMARKING] In api/metadata, time to query platforms in metadata_counts_platform_list for cohort '" + str( - request.cohort_id) + "': " + (stop - start).__str__()) - for row in cursor.fetchall(): - item = MetadataPlatformItem( - DNAseq_data=str(row['DNAseq_data']), - cnvrPlatform=str(row['cnvrPlatform']), - gexpPlatform=str(row['gexpPlatform']), - methPlatform=str(row['methPlatform']), - mirnPlatform=str(row['mirnPlatform']), - rppaPlatform=str(row['rppaPlatform']), - ) - data.append(item) - - cursor.close() - db.close() - - return MetadataCountsPlatformItem(items=data, count=counts_and_total['counts'], - participants=counts_and_total['participants'], - total=counts_and_total['total']) - - except Exception as e: - print traceback.format_exc() - if cursor: cursor.close() - if db and db.open: db.close() - raise endpoints.NotFoundException('Exception in metadata_counts_platforms_list.') diff --git a/api_3/pairwise.py b/api_3/pairwise.py deleted file mode 100755 index acb7f8ba..00000000 --- a/api_3/pairwise.py +++ /dev/null @@ -1,144 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import json -import base64 -import logging -import urllib -import traceback - -from google.appengine.api import urlfetch -from django.conf import settings - -from bq_data_access.data_access import get_feature_vector -from bq_data_access.feature_value_types import ValueType -from bq_data_access.utils import VectorMergeSupport - -logger = logging.getLogger(__name__) - -class PairwiseInputVector(object): - def __init__(self, feature_id, value_type, data): - self.feature_id = feature_id - self.value_type = value_type - self.data = data - - -class Pairwise(object): - def __init__(self): - pass - - @classmethod - def prepare_features(self, cohort_id, features): - # Get the feature data - feature_vector_mapping = {} - vectors = [] - for feature in features: - value_type, vector = get_feature_vector(feature, cohort_id) - - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - vms = VectorMergeSupport('NA', 'sample_id', row_ids=features) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def prepare_feature_vector(self, input_vectors): - feature_vector_mapping = {} - vectors = [] - for item in input_vectors: - feature_id, value_type, vector = item.feature_id, item.value_type, item.data - if value_type == ValueType.INTEGER or value_type == ValueType.FLOAT: - value_type = "N" - elif value_type == ValueType.STRING: - value_type = "C" - else: - value_type = "B" - - feature_vector_mapping[feature_id] = (value_type, vector) - vectors.append(vector) - - # Create merged feature vectors - feature_ids = [v.feature_id for v in input_vectors] - - vms = VectorMergeSupport('NA', 'sample_id', 'case_id', row_ids=feature_ids) - - for feature in feature_vector_mapping.keys(): - vms.add_dict_array(feature_vector_mapping[feature][1], feature, 'value') - - merged = vms.get_merged_dict() - - rows = [] - - for feature in feature_vector_mapping.keys(): - current_row = [feature_vector_mapping[feature][0] + ":" + feature] - - for item in merged: - current_row.append(item[feature]) - - rows.append("\t".join(current_row)) - - return rows - - @classmethod - def run_pairwise(self, feature_rows): - url = settings.PAIRWISE_SERVICE_URL - - data_dict = {} - row_count = 1 - for row in feature_rows: - label = "row_{count}".format(count=row_count) - data_dict[label] = row - row_count += 1 - - # Encode the data to be sent to the service - data = urllib.urlencode(data_dict) - decoded_response = None - - try: - pairwise_response = urlfetch.fetch(url=url, payload=data, method=urlfetch.POST) - response = pairwise_response.content - decoded_response = json.loads(base64.b64decode(response)) - except Exception as e: - decoded_response = None - logger.error(traceback.format_exc()) - - return decoded_response diff --git a/api_3/pairwise_api.py b/api_3/pairwise_api.py deleted file mode 100644 index 2e5a5e19..00000000 --- a/api_3/pairwise_api.py +++ /dev/null @@ -1,166 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -import endpoints -from protorpc import messages, remote, message_types -from django.conf import settings - -from api_3.api_helpers import authorize_credentials_with_Google -from api_3.pairwise import Pairwise - -package = 'pairwise' - - -class PairwiseJobRequest(messages.Message): - cohort_id = messages.StringField(1, required=True) - feature = messages.StringField(2, repeated=True) - - -class PairwiseResultVector(messages.Message): - feature_1 = messages.StringField(1, required=True) - feature_2 = messages.StringField(2, required=True) - comparison_type = messages.StringField(3, required=True) - correlation_coefficient = messages.StringField(4, required=True) - n = messages.IntegerField(5, required=True) - _logp = messages.FloatField(6, required=True) - n_A = messages.IntegerField(7, required=True) - p_A = messages.FloatField(8, required=True) - n_B = messages.IntegerField(9, required=True) - p_B = messages.FloatField(10, required=True) - exclusion_rules = messages.StringField(11, required=True) - - -class PairwiseFilterMessage(messages.Message): - filter_message = messages.StringField(1, required=True) - - -class PairwiseResults(messages.Message): - result_vectors = messages.MessageField(PairwiseResultVector, 1, repeated=True) - filter_messages = messages.MessageField(PairwiseFilterMessage, 2, repeated=True) - - -class Feature(messages.Message): - annotated_type = messages.StringField(1) - chr = messages.StringField(2) - start = messages.IntegerField(3) - end = messages.IntegerField(4) - label = messages.StringField(5) - mutation_count = messages.IntegerField(6) - source = messages.StringField(7) - - -class Association(messages.Message): - node1 = messages.MessageField(Feature, 1) - node2 = messages.MessageField(Feature, 2) - logged_pvalue = messages.FloatField(3) - - -class CircvizOutput(messages.Message): - items = messages.MessageField(Association, 1, repeated=True) - - -Pairwise_Endpoints = endpoints.api(name='pairwise', version='v3') - - -@Pairwise_Endpoints.api_class(resource_name='pairwise_api') -class PairwiseApi(remote.Service): - """Pairwise API v1""" - - @endpoints.method(PairwiseJobRequest, PairwiseResults, name="run", http_method="POST") - def run_job(self, request): - """ Used by the web application.""" - features = [] - count = len(request.feature) - 1 - while count >= 0: - features.append(str(request.feature[count])) - count -= 1 - - prepped_features = Pairwise.prepare_features(request.cohort_id, features) - outputs = Pairwise.run_pairwise(prepped_features) - - results = PairwiseResults(result_vectors=[], filter_messages=[]) - logging.info(results) - - for row_label, row in outputs.items(): - if type(row) is dict: - results.result_vectors.append(PairwiseResultVector(feature_1=row['feature_A'], - feature_2=row['feature_B'], - comparison_type=row['comparison_type'], - correlation_coefficient=row['correlation_coefficient'], - n=int(row['n']), - _logp=float(row['_logp']), - n_A=int(row['n_A']), - p_A=float(row['p_A']), - n_B=int(row['n_B']), - p_B=float(row['p_B']), - exclusion_rules=row['exclusion_rules'])) - elif type(row) is unicode: - results.filter_messages.append(PairwiseFilterMessage(filter_message=row[0])) - - return results - - @endpoints.method(message_types.VoidMessage, CircvizOutput, - path='precomp', http_method='GET', name='precomputed') - def precomputed_results(self, request): - """ Used by the web application.""" - bq_table = 'brca_pwpv' - query = 'SELECT A_valueType, A_chr, A_startPos, A_endPos, A_featureName, A_N, A_dataType,' \ - 'B_valueType, B_chr, B_startPos, B_endPos, B_featureName, B_N, B_dataType,' \ - 'logP FROM [isb-cgc:test.brca_pwpv] ' \ - 'where B_chr != "null" ' \ - 'and A_chr != "null"' \ - 'and A_startPos != "null" and A_endPos != "null"' \ - 'and B_startPos != "null" and B_endPos != "null"' \ - 'LIMIT 50;' - query_body = { - 'query': query - } - bigquery_service = authorize_credentials_with_Google() - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=settings.BQ_PROJECT_ID, body=query_body).execute() - association_list = [] - feature_list = [] - for row in query_response['rows']: - node1 = Feature( - annotated_type=row['f'][0]['v'].encode('utf-8') if row['f'][0]['v'] else None, - chr=row['f'][1]['v'].encode('utf-8').replace('chr','') if row['f'][1]['v'] else None, - start=int(row['f'][2]['v']) if row['f'][2]['v'] else None, - end=int(row['f'][3]['v']) if row['f'][3]['v'] else None, - label=row['f'][4]['v'].encode('utf-8') if row['f'][4]['v'] else '', - mutation_count=int(row['f'][5]['v']) if row['f'][5]['v'] else None, - source=row['f'][6]['v'].encode('utf-8') if row['f'][6]['v'] else None - ) - node2 = Feature( - annotated_type=row['f'][7]['v'].encode('utf-8') if row['f'][7]['v'] else None, - chr=row['f'][8]['v'].encode('utf-8').replace('chr','') if row['f'][8]['v'] else None, - start=int(row['f'][9]['v']) if row['f'][9]['v'] else None, - end=int(row['f'][10]['v']) if row['f'][10]['v'] else None, - label=row['f'][11]['v'].encode('utf-8') if row['f'][11]['v'] else '', - mutation_count=int(row['f'][12]['v']) if row['f'][12]['v'] else None, - source=row['f'][13]['v'].encode('utf-8') if row['f'][13]['v'] else None - ) - logP = float(row['f'][14]['v']) - association_list.append(Association(node1=node1, node2=node2, logged_pvalue=logP)) - feature_list.append(node1) - feature_list.append(node2) - return CircvizOutput(items=association_list) - - -APPLICATION = endpoints.api_server([PairwiseApi]) diff --git a/api_3/patients_get_helper.py b/api_3/patients_get_helper.py deleted file mode 100755 index ed7712ab..00000000 --- a/api_3/patients_get_helper.py +++ /dev/null @@ -1,205 +0,0 @@ -''' -Created on Apr 5, 2017 - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import endpoints -import logging -import MySQLdb - -from django.core.signals import request_finished -from protorpc import remote, messages - -from api_3.cohort_endpoint_helpers import build_constructor_dict_for_message -from api_3.api_helpers import sql_connection - -logger = logging.getLogger(__name__) - - -class CasesGetQueryBuilder(object): - - def build_queries(self, program, genomic_builds, count=1): - - clin_case_clause = 'case_barcode=%s' - case_clause = 'case_barcode=%s' - - if count > 1: - clin_case_clause = 'case_barcode in ({})'.format(",".join(['%s']*count)) - - clinical_query_str = 'select * ' \ - 'from {}_metadata_clinical ' \ - 'where {}'.format(program, clin_case_clause) - - sample_query_str = 'select sample_barcode ' \ - 'from {}_metadata_biospecimen ' \ - 'where {} ' \ - 'group by sample_barcode ' \ - 'order by sample_barcode'.format(program, case_clause) - - aliquot_query_str = '' - for genomic_build in genomic_builds: - part_aliquot_query_str = 'select aliquot_barcode ' \ - 'from {}_metadata_data_{}_r14 ' \ - 'where {} and aliquot_barcode is not null ' \ - 'group by aliquot_barcode '.format(program, genomic_build, case_clause) - if 0 < len(aliquot_query_str): - aliquot_query_str += ' UNION DISTINCT ' - aliquot_query_str += part_aliquot_query_str - - aliquot_query_str += 'order by aliquot_barcode' - - return clinical_query_str, sample_query_str, aliquot_query_str - - -class CaseGetListFilters(messages.Message): - case_barcodes = messages.StringField(1, repeated=True) - -class CasesGetHelper(remote.Service): - - GET_RESOURCE = endpoints.ResourceContainer(case_barcode=messages.StringField(1, required=True)) - - POST_RESOURCE = endpoints.ResourceContainer( - CaseGetListFilters - ) - - def get(self, request, CaseDetails, MetadataItem, program): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case (*eg* TCGA-B9-7268 for TCGA) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - case_barcode = request.get_assigned_value('case_barcode') - query_tuple = (str(case_barcode),) - genomic_builds = ['HG19'] - if program != 'CCLE': - genomic_builds.append('HG38') - - clinical_query_str, sample_query_str, aliquot_query_str = CasesGetQueryBuilder().build_queries(program, genomic_builds) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build clinical data message - cursor.execute(clinical_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - logger.warn("Case barcode {} not found in {}_metadata_clinical table.".format(case_barcode, program)) - raise endpoints.NotFoundException("Case barcode {} not found".format(case_barcode)) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - clinical_data_item = MetadataItem(**constructor_dict) - - # get list of samples - cursor.execute(sample_query_str, query_tuple) - sample_list = [row['sample_barcode'] for row in cursor.fetchall()] - - # get list of aliquots - # cursor.execute(aliquot_query_str, (query_tuple * len(genomic_builds))) - # aliquot_list = [row['aliquot_barcode'] for row in cursor.fetchall()] - aliquot_list = None - - return CaseDetails(clinical_data=clinical_data_item, samples=sample_list, aliquots=aliquot_list if aliquot_list else []) - except (IndexError, TypeError), e: - logger.info("Case {} not found. Error: {}".format(case_barcode, e)) - raise endpoints.NotFoundException("Case {} not found.".format(case_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving case, sample, or aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving case, sample, or aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) - - - def get_list(self, request, CaseSetDetails, CaseDetails, MetadataItem, program): - """ - Returns information about a specific case, - including a list of samples and aliquots derived from this case. - Takes a case (*eg* TCGA-B9-7268 for TCGA) as a required parameter. - User does not need to be authenticated. - """ - - cursor = None - db = None - - case_barcodes = request.get_assigned_value('case_barcodes') if 'case_barcodes' in [k.name for k in request.all_fields()] else None - - if not case_barcodes or not len(case_barcodes): - raise endpoints.BadRequestException("A list of case barcodes is required.") - elif len(case_barcodes) > 500: - raise endpoints.BadRequestException("The limit on barcodes per request is 500.") - - query_tuple = [x for x in case_barcodes] - genomic_builds = ['HG19'] - if program != 'CCLE': - genomic_builds.append('HG38') - - clinical_query_str, sample_query_str, aliquot_query_str = CasesGetQueryBuilder().build_queries(program, genomic_builds, len(case_barcodes)) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build clinical data message - cursor.execute(clinical_query_str, query_tuple) - rows = cursor.fetchall() - if not len(rows): - cursor.close() - db.close() - logger.warn("None of the case barcodes were found in the {}_metadata_clinical table.".format(program)) - raise endpoints.NotFoundException("None of the case barcodes were found") - - case_details = [] - - for row in rows: - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - clinical_data_item = MetadataItem(**constructor_dict) - - # get list of samples - cursor.execute(sample_query_str, (row['case_barcode'],)) - sample_list = [sample_row['sample_barcode'] for sample_row in cursor.fetchall()] - - # get list of aliquots - # cursor.execute(aliquot_query_str, ((row['case_barcode'],) * len(genomic_builds))) - # aliquot_list = [aliquot_row['aliquot_barcode'] for aliquot_row in cursor.fetchall()] - aliquot_list = None - - case_details.append( - CaseDetails( - clinical_data=clinical_data_item, samples=sample_list, - aliquots=aliquot_list if aliquot_list else [], case_barcode=row['case_barcode']) - ) - - return CaseSetDetails(cases=case_details) - except (IndexError, TypeError), e: - logger.exception(e) - logger.info("The cases supplied were not found. Error: {}".format(e)) - raise endpoints.NotFoundException("The cases provided were not found not found.") - except MySQLdb.ProgrammingError as e: - logger.warn("Error retrieving case, sample, or aliquot data: {}".format(e)) - raise endpoints.BadRequestException("Error retrieving case, sample, or aliquot data: {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - request_finished.send(self) diff --git a/api_3/samples_get_helper.py b/api_3/samples_get_helper.py deleted file mode 100755 index 0ddc34ee..00000000 --- a/api_3/samples_get_helper.py +++ /dev/null @@ -1,341 +0,0 @@ -''' -Created on Apr 5, 2017 - -opyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -@author: michael -''' -import endpoints -import logging -import MySQLdb -from protorpc import remote, messages -from api_3.isb_cgc_api_TCGA.message_classes import BiospecimenMetadataItem - -from api_3.api_helpers import sql_connection -from api_3.cohort_endpoint_helpers import build_constructor_dict_for_message - -logger = logging.getLogger(__name__) - -class SamplesGetQueryBuilder(object): - def build_aliquot_query(self, program, param_list, count=1): - if 'CCLE' == program: - genomic_builds = ['HG19'] - else: - genomic_builds = ['HG19', 'HG38'] - - aliquot_query_str = '' - for genomic_build in genomic_builds: - part_aliquot_query_str = 'select sample_barcode, case_barcode, aliquot_barcode, aliquot_gdc_id ' \ - 'from {}_metadata_data_{}_r14 ' \ - 'where file_name_key is not null and file_name_key !="" '.format(program, genomic_build) - for column in param_list: - if column == 'sample_barcode' and count>1: - part_aliquot_query_str += ' and {} IN ({}) '.format(column, ','.join(["%s"]*count)) - else: - part_aliquot_query_str += ' and {}=%s '.format(column) - if 0 < len(aliquot_query_str): - aliquot_query_str += ' union ' - aliquot_query_str += part_aliquot_query_str - - return aliquot_query_str - - def build_biospecimen_query(self, program, count=1): - - biospecimen_query_str = 'select * ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode=%s'.format(program) - if count>1: - biospecimen_query_str = 'select * ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode in ({})'.format(program, ','.join(["%s"] * count)) - - return biospecimen_query_str - - def build_data_query(self, program, datadict_class, param_list, count=1): - if 'CCLE' == program: - genomic_builds = ['HG19'] - else: - genomic_builds = ['HG19', 'HG38'] - - data_query_str = '' - for genomic_build in genomic_builds: - part_data_query_str = 'select {0} ' \ - 'from {1}_metadata_data_{2}_r14 ' \ - 'where file_name_key is not null and file_name_key !="" '.format(', '.join(field.name for field in datadict_class.all_fields()), program, genomic_build) - for column in param_list: - if column == 'sample_barcode' and count > 1: - part_data_query_str += ' and {} IN ({}) '.format(column, ','.join(["%s"]*count)) - else: - part_data_query_str += ' and {}=%s '.format(column) - if 0 < len(data_query_str): - data_query_str += ' union ' - data_query_str += part_data_query_str - - return data_query_str - - def build_case_query(self, program, count=1): - - case_query_str = 'select case_barcode, case_gdc_id ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode=%s ' \ - 'group by case_barcode, case_gdc_id'.format(program) - - if count>1: - case_query_str = 'select case_barcode, case_gdc_id ' \ - 'from {}_metadata_biospecimen ' \ - 'where sample_barcode in ({}) ' \ - 'group by case_barcode, case_gdc_id'.format(program, ','.join(["%s"] * count)) - - return case_query_str - - -class DataDetails(messages.Message): - file_gdc_id = messages.StringField(1) - file_name_key = messages.StringField(3) - file_size = messages.IntegerField(4, variant=messages.Variant.INT64) - sample_gdc_id = messages.StringField(5) - sample_barcode = messages.StringField(6) - project_short_name = messages.StringField(8) - disease_code = messages.StringField(9) - program_name = messages.StringField(11) - data_type = messages.StringField(12) - data_category = messages.StringField(13) - experimental_strategy = messages.StringField(14) - data_format = messages.StringField(15) - access = messages.StringField(16) - platform = messages.StringField(17) - index_file_name_key = messages.StringField(20) - - -class SampleGetListFilters(messages.Message): - sample_barcodes = messages.StringField(1, repeated=True) - disease_code = messages.StringField(2, repeated=True) - experimental_strategy = messages.StringField(3, repeated=True) - platform = messages.StringField(4, repeated=True) - data_category = messages.StringField(5, repeated=True) - data_type = messages.StringField(6, repeated=True) - data_format = messages.StringField(7, repeated=True) - project_short_name = messages.StringField(8, repeated=True) - - -class SamplesGetAPI(remote.Service): - GET_RESOURCE = endpoints.ResourceContainer( - sample_barcode=messages.StringField(1, required=True), - data_type = messages.StringField(2), - data_category = messages.StringField(3), - experimental_strategy = messages.StringField(4), - data_format = messages.StringField(5), - platform = messages.StringField(6), - endpoint_type = messages.StringField(7), - analysis_workflow_type = messages.StringField(8) - ) - - POST_RESOURCE = endpoints.ResourceContainer( - SampleGetListFilters - ) - - def get(self, request, program, SampleDetails, MetadataItem): - """ - Given a sample barcode (of length 16, *eg* TCGA-B9-7268-01A, for TCGA), this endpoint returns - all available "biospecimen" information about this sample, - the associated case barcode, a list of associated aliquots, - and a list of "data_details" blocks describing each of the data files associated with this sample - """ - cursor = None - db = None - - sample_barcode = request.get_assigned_value('sample_barcode') - param_list = ['sample_barcode'] - query_tuple = [sample_barcode] - extra_query_tuple = [sample_barcode] - for field in request.all_fields(): - if 'sample_barcode' != field.name and request.get_assigned_value(field.name): - param_list += [field.name] - extra_query_tuple += [request.get_assigned_value(field.name)] - - if 'CCLE' != program: - extra_query_tuple += extra_query_tuple - - # need to take into account params used in the union between the genomic builds - - aliquot_query_str = SamplesGetQueryBuilder().build_aliquot_query(program, param_list) - biospecimen_query_str = SamplesGetQueryBuilder().build_biospecimen_query(program) - data_query_str = SamplesGetQueryBuilder().build_data_query(program, DataDetails(), param_list) - case_query_str = SamplesGetQueryBuilder().build_case_query(program) - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build biospecimen data message - cursor.execute(biospecimen_query_str, query_tuple) - row = cursor.fetchone() - if row is None: - cursor.close() - db.close() - error_message = "Sample barcode {} not found in {}_metadata_biospecimen table.".format(query_tuple[0], program) - raise endpoints.NotFoundException(error_message) - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - biospecimen_data_item = MetadataItem(**constructor_dict) - - # get list of aliquots - # cursor.execute(aliquot_query_str, extra_query_tuple) - # aliquot_list = [row['aliquot_barcode'] for row in cursor.fetchall()] - aliquot_list = [] - - # get case barcode (superfluous?) - cursor.execute(case_query_str, query_tuple) - row = cursor.fetchone() - case_barcode = str(row["case_barcode"]) - case_gdc_id = str(row["case_gdc_id"]) - - # prepare to build list of data details messages - cursor.execute(data_query_str, extra_query_tuple) - cursor_rows = cursor.fetchall() - - # build a data details message for each row returned from metadata_data table - data_details_list = [] - for row in cursor_rows: - constructor_dict = build_constructor_dict_for_message(DataDetails(), row) - data_details_item = DataDetails(**constructor_dict) - data_details_list.append(data_details_item) - - return SampleDetails(aliquots=aliquot_list, - biospecimen_data=biospecimen_data_item, - data_details=data_details_list, - data_details_count=len(data_details_list), - case_barcode=case_barcode, - case_gdc_id=case_gdc_id) - - except (IndexError, TypeError) as e: - logger.info("Sample details for barcode {} not found. Error: {}, \nSQL: {}, \nParams: {}".format(sample_barcode, e, aliquot_query_str, extra_query_tuple)) - raise endpoints.NotFoundException( - "Sample details for barcode {} not found.".format(sample_barcode)) - except MySQLdb.ProgrammingError as e: - logger.warn(e) - raise endpoints.BadRequestException("Error retrieving biospecimen, case, or other data. {}".format(e)) - finally: - if cursor: cursor.close() - if db and db.open: db.close() - - def get_list(self, request, program, SampleSetDetails, SampleDetails, MetadataItem): - """ - Given a set of sample barcodes (of length 16, *eg* TCGA-B9-7268-01A, for TCGA), this endpoint returns - all available "biospecimen" information about these samples, the associated case barcodes, - a list of associated aliquots, and a list of "data_details" blocks describing each of the data files - associated with this sample - """ - cursor = None - db = None - sample_barcodes = request.get_assigned_value('sample_barcodes') if 'sample_barcodes' in [k.name for k in - request.all_fields()] else None - - if not sample_barcodes: - raise endpoints.BadRequestException("A list of sample barcodes is required.") - elif len(sample_barcodes) > 500: - raise endpoints.BadRequestException("There is a 500 barcode limit per quest.") - - param_list = ['sample_barcode'] - query_tuple = [x for x in sample_barcodes] - extra_query_tuple = [x for x in sample_barcodes] - for field in request.all_fields(): - if 'sample_barcodes' not in field.name and request.get_assigned_value(field.name): - param_list += [field.name] - extra_query_tuple += [request.get_assigned_value(field.name)] - - if 'CCLE' != program: - extra_query_tuple += extra_query_tuple - - # need to take into account params used in the union between the genomic builds - - aliquot_query_str = SamplesGetQueryBuilder().build_aliquot_query(program, param_list, len(sample_barcodes)) - biospecimen_query_str = SamplesGetQueryBuilder().build_biospecimen_query(program, len(sample_barcodes)) - data_query_str = SamplesGetQueryBuilder().build_data_query(program, DataDetails(), param_list, len(sample_barcodes)) - - samples = SampleSetDetails() - - sample_data = {} - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - # build biospecimen data message - cursor.execute(biospecimen_query_str, query_tuple) - rows = cursor.fetchall() - if not len(rows): - cursor.close() - db.close() - error_message = "These sample barcodes were not found in the {}_metadata_biospecimen table.".format(program,) - raise endpoints.NotFoundException(error_message) - - for row in rows: - sample_data[row['sample_barcode']] = {} - constructor_dict = build_constructor_dict_for_message(MetadataItem(), row) - biospecimen_data_item = MetadataItem(**constructor_dict) - sample_data[row['sample_barcode']]['biospecimen'] = biospecimen_data_item - sample_data[row['sample_barcode']]['case_barcode'] = row['case_barcode'] - sample_data[row['sample_barcode']]['case_gdc_id'] = row['case_gdc_id'] - - # get list of aliquots - # cursor.execute(aliquot_query_str, extra_query_tuple) - # rows = cursor.fetchall() - # for row in rows: - # if 'aliquots' not in sample_data[row['sample_barcode']]: - # sample_data[row['sample_barcode']]['aliquots'] = [] - # sample_data[row['sample_barcode']]['aliquots'].append(row['aliquot_barcode']) - - # prepare to build list of data details messages - cursor.execute(data_query_str, extra_query_tuple) - cursor_rows = cursor.fetchall() - - # build a data details message for each row returned from metadata_data table - for row in cursor_rows: - if 'data_rows' not in sample_data[row['sample_barcode']]: - sample_data[row['sample_barcode']]['data_rows'] = [] - constructor_dict = build_constructor_dict_for_message(DataDetails(), row) - data_details_item = DataDetails(**constructor_dict) - sample_data[row['sample_barcode']]['data_rows'].append(data_details_item) - - for sample in sample_data: - samples.samples.append( - SampleDetails( - aliquots=sample_data[sample]['aliquots'] if 'aliquots' in sample_data[sample] else [], - biospecimen_data=sample_data[sample]['biospecimen'], - data_details=sample_data[sample]['data_rows'] if 'data_rows' in sample_data[sample] else [], - data_details_count=len(sample_data[sample]['data_rows']) if 'data_rows' in sample_data[sample] else 0, - case_barcode=sample_data[sample]['case_barcode'], - case_gdc_id=sample_data[sample]['case_gdc_id'], - sample_barcode=sample - ) - ) - - return samples - - except (IndexError, TypeError) as e: - logger.exception(e) - logger.error("Error: {}, \nSQL: {}, \nParams: {}".format(e,aliquot_query_str,extra_query_tuple)) - raise endpoints.NotFoundException( - "There was an error while processing your request--please contact us at feedback@isb-cgc.org" - ) - except MySQLdb.ProgrammingError as e: - logger.warn(e) - raise endpoints.BadRequestException("Error retrieving biospecimen, case, or other data. {}".format(e)) - except Exception as e: - logger.error("[ERROR] While processing samples.get_list: ") - logger.exception(e) - finally: - if cursor: cursor.close() - if db and db.open: db.close() diff --git a/api_3/schema/__init__.py b/api_3/schema/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/schema/tcga_clinical.py b/api_3/schema/tcga_clinical.py deleted file mode 100755 index 0725770f..00000000 --- a/api_3/schema/tcga_clinical.py +++ /dev/null @@ -1,283 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -# Updated from -# https://github.com/isb-cgc/data-prototyping/blob/537c5c24646f87bda804ca95dee6cf479f0b1fb9/tcga_etl_pipeline/schemas/clinical.json - -schema = [ - { - "type": "STRING", - "name": "ParticipantBarcode" - }, - { - "type": "STRING", - "name": "Study" - }, - { - "type": "STRING", - "name": "Project" - }, - { - "type": "STRING", - "name": "ParticipantUUID" - }, - { - "type": "STRING", - "name": "TSSCode" - }, - { - "type": "INTEGER", - "name": "age_at_diagnosis" - }, - { - "type": "STRING", - "name": "anatomic_neoplasm_subdivision" - }, - { - "type": "INTEGER", - "name": "batch_number" - }, - { - "type": "STRING", - "name": "bcr" - }, - { - "type": "STRING", - "name": "clinical_M" - }, - { - "type": "STRING", - "name": "clinical_N" - }, - { - "type": "STRING", - "name": "clinical_T" - }, - { - "type": "STRING", - "name": "clinical_stage" - }, - { - "type": "STRING", - "name": "colorectal_cancer" - }, - { - "type": "STRING", - "name": "country" - }, - { - "type": "STRING", - "name": "vital_status" - }, - { - "type": "INTEGER", - "name": "days_to_birth" - }, - { - "type": "INTEGER", - "name": "days_to_death" - }, - { - "type": "INTEGER", - "name": "days_to_last_known_alive" - }, - { - "type": "INTEGER", - "name": "days_to_last_followup" - }, - { - "type": "INTEGER", - "name": "days_to_initial_pathologic_diagnosis" - }, - { - "type": "INTEGER", - "name": "days_to_submitted_specimen_dx" - }, - { - "type": "STRING", - "name": "ethnicity" - }, - { - "type": "STRING", - "name": "frozen_specimen_anatomic_site" - }, - { - "type": "STRING", - "name": "gender" - }, - { - "type": "FLOAT", - "name": "gleason_score_combined" - }, - { - "type": "STRING", - "name": "histological_type" - }, - { - "type": "STRING", - "name": "history_of_colon_polyps" - }, - { - "type": "STRING", - "name": "history_of_neoadjuvant_treatment" - }, - { - "type": "STRING", - "name": "hpv_calls" - }, - { - "type": "STRING", - "name": "hpv_status" - }, - { - "type": "STRING", - "name": "icd_10" - }, - { - "type": "STRING", - "name": "icd_o_3_histology" - }, - { - "type": "STRING", - "name": "icd_o_3_site" - }, - { - "type": "STRING", - "name": "lymphatic_invasion" - }, - { - "type": "STRING", - "name": "lymphnodes_examined" - }, - { - "type": "STRING", - "name": "lymphovascular_invasion_present" - }, - { - "type": "STRING", - "name": "menopause_status" - }, - { - "type": "STRING", - "name": "mononucleotide_and_dinucleotide_marker_panel_analysis_status" - }, - { - "type": "FLOAT", - "name": "mononucleotide_marker_panel_analysis_status" - }, - { - "type": "STRING", - "name": "neoplasm_histologic_grade" - }, - { - "type": "STRING", - "name": "new_tumor_event_after_initial_treatment" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_examined" - }, - { - "type": "FLOAT", - "name": "number_of_lymphnodes_positive_by_he" - }, - { - "type": "FLOAT", - "name": "number_pack_years_smoked" - }, - { - "type": "INTEGER", - "name": "year_of_initial_pathologic_diagnosis" - }, - { - "type": "STRING", - "name": "pathologic_M" - }, - { - "type": "STRING", - "name": "pathologic_N" - }, - { - "type": "STRING", - "name": "pathologic_T" - }, - { - "type": "STRING", - "name": "pathologic_stage" - }, - { - "type": "STRING", - "name": "person_neoplasm_cancer_status" - }, - { - "type": "STRING", - "name": "pregnancies" - }, - { - "type": "STRING", - "name": "primary_neoplasm_melanoma_dx" - }, - { - "type": "STRING", - "name": "primary_therapy_outcome_success" - }, - { - "type": "STRING", - "name": "prior_dx" - }, - { - "type": "FLOAT", - "name": "psa_value" - }, - { - "type": "STRING", - "name": "race" - }, - { - "type": "STRING", - "name": "residual_tumor" - }, - { - "type": "STRING", - "name": "tobacco_smoking_history" - }, - { - "type": "STRING", - "name": "tumor_tissue_site" - }, - { - "type": "STRING", - "name": "tumor_type" - }, - { - "type": "STRING", - "name": "venous_invasion" - }, - { - "type": "FLOAT", - "name": "weight" - }, - { - "type": "FLOAT", - "name": "height" - }, - { - "type": "FLOAT", - "name": "BMI" - } -] \ No newline at end of file diff --git a/api_3/seqpeek_api.py b/api_3/seqpeek_api.py deleted file mode 100644 index 7aee1aa7..00000000 --- a/api_3/seqpeek_api.py +++ /dev/null @@ -1,94 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from api_3.data_access import PlotDataCohortInfo - -from endpoints import api as endpoints_api, method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class MAFRecord(Message): - sample_id = StringField(1) - patient_id = StringField(2) - aliquot_id = StringField(3) - hugo_symbol = StringField(4) - uniprot_aapos = IntegerField(5, variant=Variant.INT32) - uniprot_id = StringField(6) - variant_classification = StringField(7) - cohort = IntegerField(8, repeated=True) - - -class MAFRecordList(Message): - items = MessageField(MAFRecord, 1, repeated=True) - cohort_set = MessageField(PlotDataCohortInfo, 2, repeated=True) - -SeqPeekDataEndpointsAPI = endpoints_api(name='seqpeek_data_api', version='v1', - description='Endpoints used by the seqpeek visualization in the web application.') - - -def maf_array_to_record(maf_array): - data_points = [] - for item in maf_array: - data_points.append(MAFRecord(**item)) - - return data_points - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekDataAccessAPI(remote.Service): - - def create_response(self, maf_with_cohorts): - - data_points = maf_array_to_record(maf_with_cohorts.maf_vector) - - cohort_info_obj_array = [] - for item in maf_with_cohorts.cohort_info: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return MAFRecordList(items=data_points, cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, MAFRecordList, - path='by_gnab_feature', http_method='GET', name='seqpeek.getMAFDataWithCohorts') - def data_access_by_gnab_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - - maf_with_cohorts = SeqPeekMAFDataFormatter().format_maf_vector_for_view(feature_id, cohort_id_array) - response = self.create_response(maf_with_cohorts) - return response - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/seqpeek_view_api.py b/api_3/seqpeek_view_api.py deleted file mode 100644 index 8e69d7be..00000000 --- a/api_3/seqpeek_view_api.py +++ /dev/null @@ -1,238 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from endpoints import method as endpoints_method -from endpoints import InternalServerErrorException -from protorpc import remote -from protorpc.messages import IntegerField, Message, MessageField, StringField, Variant - -from bq_data_access.seqpeek.seqpeek_view import SeqPeekViewDataBuilder -from bq_data_access.data_access import get_feature_vectors_tcga_only -from api_3.seqpeek_api import SeqPeekDataEndpointsAPI, MAFRecord, maf_array_to_record -from bq_data_access.seqpeek.seqpeek_maf_formatter import SeqPeekMAFDataFormatter -from bq_data_access.seqpeek_maf_data import SeqPeekDataProvider -from bq_data_access.data_access import ProviderClassQueryDescription -from api_3.data_access import fetch_isbcgc_project_set -from api_3.api_helpers import sql_connection - -from projects.models import Project - -class SeqPeekViewDataRequest(Message): - hugo_symbol = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class InterproMatchLocation(Message): - # TODO this should likely be a float - score = IntegerField(1, variant=Variant.INT32) - start = IntegerField(2, variant=Variant.INT32) - end = IntegerField(3, variant=Variant.INT32) - - -class InterproMatch(Message): - status = StringField(1) - name = StringField(2) - evd = StringField(3) - locations = MessageField(InterproMatchLocation, 4, repeated=True) - dbname = StringField(5) - id = StringField(6) - - -class InterproJson(Message): - matches = MessageField(InterproMatch, 1, repeated=True) - uniprot_id = StringField(2) - length = IntegerField(3, variant=Variant.INT32) - name = StringField(4) - - -class InterproItem(Message): - uniprot_id = StringField(1) - interpro_json = MessageField(InterproJson, 2, repeated=False) - - -class SeqPeekRegionRecord(Message): - type = StringField(1, required=True) - start = IntegerField(2, required=True, variant=Variant.INT32) - end = IntegerField(3, required=True, variant=Variant.INT32) - - -class SeqPeekTrackRecord(Message): - mutations = MessageField(MAFRecord, 1, repeated=True) - type = StringField(2, required=True) - label = StringField(3, required=True) - number_of_samples = IntegerField(4, required=True) - mutated_positions = IntegerField(5, required=True) - cohort_size = IntegerField(6, required=False) - row_id = StringField(7, required=True) - - -class SeqPeekViewPlotDataRecord(Message): - tracks = MessageField(SeqPeekTrackRecord, 1, repeated=True) - protein = MessageField(InterproItem, 2, required=False) - regions = MessageField(SeqPeekRegionRecord, 3, repeated=True) - - -class SeqPeekRemovedRow(Message): - name = StringField(1, required=True) - num = IntegerField(2, required=True) - - -class SeqPeekViewRecord(Message): - cohort_id_list = StringField(1, repeated=True) - hugo_symbol = StringField(2, required=True) - plot_data = MessageField(SeqPeekViewPlotDataRecord, 3, required=True) - removed_row_statistics = MessageField(SeqPeekRemovedRow, 4, repeated=True) - - -def create_interpro_record(interpro_literal): - match_data = [] - for match in interpro_literal['matches']: - - match_location_data = [] - for location in match['locations']: - match_location_data.append(InterproMatchLocation( - score=int(location['score']), - start=int(location['start']), - end=int(location['end']) - )) - - match_data.append(InterproMatch( - status=str(match['status']), - name=str(match['name']), - evd=str(match['evd']), - locations=match_location_data, - dbname=str(match['dbname']), - id=str(match['id']), - )) - - interpro_json = InterproJson( - matches=match_data, - uniprot_id=str(interpro_literal['uniprot_id']), - length=int(interpro_literal['length']), - name=str(interpro_literal['name']) - ) - - return InterproItem( - interpro_json=interpro_json - ) - - -@SeqPeekDataEndpointsAPI.api_class(resource_name='data_endpoints') -class SeqPeekViewDataAccessAPI(remote.Service): - def build_gnab_feature_id(self, gene_label): - return "GNAB:{gene_label}:variant_classification".format(gene_label=gene_label) - - def create_response(self, seqpeek_view_data): - plot_data = seqpeek_view_data['plot_data'] - tracks = [] - for track in plot_data['tracks']: - mutations = maf_array_to_record(track['mutations']) - tracks.append(SeqPeekTrackRecord(mutations=mutations, label=track['label'], type=track["type"], - row_id=track['render_info']['row_id'], - number_of_samples=track['statistics']['samples']['numberOf'], - mutated_positions=track['statistics']['samples']['mutated_positions'], - cohort_size=track['statistics']['cohort_size'])) - - region_records = [] - for region in plot_data['regions']: - region_records.append(SeqPeekRegionRecord(**region)) - - protein = create_interpro_record(plot_data['protein']) - plot_data_record = SeqPeekViewPlotDataRecord(tracks=tracks, protein=protein, regions=region_records) - - removed_row_statistics = [] - for item in seqpeek_view_data['removed_row_statistics']: - removed_row_statistics.append(SeqPeekRemovedRow(**item)) - - return SeqPeekViewRecord(plot_data=plot_data_record, hugo_symbol=seqpeek_view_data['hugo_symbol'], - cohort_id_list=seqpeek_view_data['cohort_id_list'], - removed_row_statistics=removed_row_statistics) - - @endpoints_method(SeqPeekViewDataRequest, SeqPeekViewRecord, - path='view_data', http_method='GET', name='seqpeek.getViewData') - def seqpeek_view_data(self, request): - try: - hugo_symbol = request.hugo_symbol - cohort_id_array = request.cohort_id - - gnab_feature_id = self.build_gnab_feature_id(hugo_symbol) - logging.debug("GNAB feature ID for SeqPeke: {0}".format(gnab_feature_id)) - - # Get the project IDs these cohorts' samples come from - cohort_vals = tuple(int(i) for i in cohort_id_array) - cohort_params = ('%s,' * len(cohort_id_array))[:-1] - - db = sql_connection() - cursor = db.cursor() - - isbcgc_projects = fetch_isbcgc_project_set() - - cursor.execute("SELECT DISTINCT project_id FROM cohorts_samples WHERE cohort_id IN (%s);" % cohort_params, - cohort_vals) - - # Only samples whose source studies are TCGA studies, or extended from them, should be used - confirmed_project_ids = [] - unconfirmed_project_ids = [] - - for row in cursor.fetchall(): - if row[0] in isbcgc_projects: - if row[0] not in confirmed_project_ids: - confirmed_project_ids.append(row[0]) - elif row[0] not in unconfirmed_project_ids: - unconfirmed_project_ids.append(row[0]) - - if len(unconfirmed_project_ids) > 0: - projects = Project.objects.filter(id__in=unconfirmed_project_ids) - - for proj in projects: - if proj.get_my_root_and_depth()['root'] in isbcgc_projects: - confirmed_project_ids.append(proj.id) - - async_params = [ProviderClassQueryDescription(SeqPeekDataProvider, gnab_feature_id, cohort_id_array, confirmed_project_ids)] - maf_data_result = get_feature_vectors_tcga_only(async_params, skip_formatting_for_plot=True) - - maf_data_vector = maf_data_result[gnab_feature_id]['data'] - - if len(maf_data_vector) > 0: - # Since the gene (hugo_symbol) parameter is part of the GNAB feature ID, - # it will be sanity-checked in the SeqPeekMAFDataAccess instance. - seqpeek_data = SeqPeekMAFDataFormatter().format_maf_vector_for_view(maf_data_vector, cohort_id_array) - - seqpeek_maf_vector = seqpeek_data.maf_vector - seqpeek_cohort_info = seqpeek_data.cohort_info - removed_row_statistics_dict = seqpeek_data.removed_row_statistics - - seqpeek_view_data = SeqPeekViewDataBuilder().build_view_data(hugo_symbol, - seqpeek_maf_vector, - seqpeek_cohort_info, - cohort_id_array, - removed_row_statistics_dict) - - response = self.create_response(seqpeek_view_data) - return response - else: - # No data found - return SeqPeekViewRecord(plot_data=SeqPeekViewPlotDataRecord(tracks=[], protein=None, regions=[]), - hugo_symbol=hugo_symbol, cohort_id_list=[str(i) for i in cohort_id_array], - removed_row_statistics=[]) - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() - diff --git a/api_3/single_feature_access.py b/api_3/single_feature_access.py deleted file mode 100644 index 916cb48c..00000000 --- a/api_3/single_feature_access.py +++ /dev/null @@ -1,131 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import time - -from api_3.data_access import FeatureDataEndpointsAPI, PlotDataCohortInfo - -from endpoints import method as endpoints_method -from endpoints import NotFoundException, InternalServerErrorException -from protorpc import remote -from protorpc.messages import EnumField, IntegerField, Message, MessageField, StringField - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType -from bq_data_access.data_access import is_valid_feature_identifier, get_feature_vectors_with_user_data -from bq_data_access.utils import VectorMergeSupport -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess - - -class DataRequest(Message): - feature_id = StringField(1, required=True) - cohort_id = IntegerField(2, repeated=True) - - -class DataPoint(Message): - sample_id = StringField(1) - value = StringField(2) - cohort = IntegerField(3, repeated=True) - - -class DataPointList(Message): - type = EnumField(ValueType, 1, required=True) - items = MessageField(DataPoint, 2, repeated=True) - label = StringField(3, required=True) - cohort_set = MessageField(PlotDataCohortInfo, 4, repeated=True) - - -@FeatureDataEndpointsAPI.api_class(resource_name='feature_data_endpoints') -class SingleFeatureDataAccess(remote.Service): - def get_feature_vector(self, feature_id, cohort_id_array): - start = time.time() - - async_params = [(feature_id, cohort_id_array)] - async_result = get_feature_vectors_with_user_data(async_params) - - feature_type, feature_vec = async_result[feature_id]['type'], async_result[feature_id]['data'] - - end = time.time() - time_elapsed = end-start - logging.info('Time elapsed: ' + str(time_elapsed)) - - vms = VectorMergeSupport('NA', 'sample_id', [feature_id]) - vms.add_dict_array(feature_vec, feature_id, 'value') - - merged = vms.get_merged_dict() - - return feature_type, merged - - def annotate_vector_with_cohorts(self, cohort_id_array, merged): - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - for value_bundle in merged: - sample_id = value_bundle['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - value_bundle['cohort'] = cohort_set - - def get_cohort_information(self, cohort_id_array): - # Get the name and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - - return cohort_info_array - - def create_response(self, feature_id, vector_type, vector, cohort_info_array): - data_points = [] - for item in vector: - data_points.append(DataPoint( - sample_id=item['sample_id'], - value=item[feature_id], - cohort=item['cohort'] - )) - - cohort_info_obj_array = [] - for item in cohort_info_array: - cohort_info_obj_array.append(PlotDataCohortInfo(**item)) - - return DataPointList(type=vector_type, items=data_points, label=feature_id, - cohort_set=cohort_info_obj_array) - - @endpoints_method(DataRequest, DataPointList, - path='feature_data', http_method='GET', name='feature_access.getFeatureData') - def data_access_by_feature(self, request): - """ Used by the web application.""" - try: - feature_id = request.feature_id - cohort_id_array = request.cohort_id - vector_type, vector = self.get_feature_vector(feature_id, cohort_id_array) - - self.annotate_vector_with_cohorts(cohort_id_array, vector) - - cohort_info = self.get_cohort_information(cohort_id_array) - response = self.create_response(feature_id, vector_type, vector, cohort_info) - return response - except FeatureNotFoundException as fnf: - logging.error("Invalid internal feature ID '{}'".format(str(fnf))) - raise NotFoundException() - except Exception as e: - logging.exception(e) - raise InternalServerErrorException() diff --git a/api_3/test/__init__.py b/api_3/test/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/api_3/users_get_common.py b/api_3/users_get_common.py deleted file mode 100755 index aea70154..00000000 --- a/api_3/users_get_common.py +++ /dev/null @@ -1,86 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import logging - -import endpoints -from protorpc import remote, messages - -from django.contrib.auth.models import User -from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned - -from accounts.models import AuthorizedDataset, NIH_User, UserAuthorizedDatasets - -logger = logging.getLogger(__name__) - -class UserGetAPIReturnJSON(messages.Message): - message = messages.StringField(1) - dbGaP_authorized = messages.BooleanField(2) - dbGaP_allowed = messages.BooleanField(3) - -class UserGetAPICommon(remote.Service): - def get(self, program): - ''' - Returns the dbGaP authorization status of the user. - ''' - user_email = None - - if endpoints.get_current_user() is not None: - user_email = endpoints.get_current_user().email() - - if user_email is None: - raise endpoints.UnauthorizedException("Authentication unsuccessful.") - - authorized = False - allowed = False - try: - user = User.objects.get(email=user_email) - except ObjectDoesNotExist, MultipleObjectsReturned: - user = None - - if user: - # FIND NIH_USER FOR USER - try: - nih_user = NIH_User.objects.filter(user=user).first() - except: - nih_user = None - - # IF USER HAS LINKED ERA COMMONS ID - if nih_user: - # FIND ALL DATASETS USER HAS ACCESS TO - user_auth_datasets = AuthorizedDataset.objects.filter(id__in=UserAuthorizedDatasets.objects.filter(nih_user_id=nih_user.id).values_list('authorized_dataset', flat=True)) - for dataset in user_auth_datasets: - if program in dataset.name: - authorized = True - allowed = True - - if not allowed: - return UserGetAPIReturnJSON(message="{} is not on the controlled-access google group.".format(user_email), - dbGaP_authorized=False, - dbGaP_allowed=False) - # since user has access to the program controlled data, include a warning - warn_message = 'You are reminded that when accessing controlled access information you are bound by the dbGaP DATA USE CERTIFICATION AGREEMENT (DUCA) for this dataset.' - if authorized: - return UserGetAPIReturnJSON(message="{} has dbGaP authorization for {} and is a member of the controlled-access google group. {}" - .format(user_email, program, warn_message), - dbGaP_authorized=True, - dbGaP_allowed=True) - else: - return UserGetAPIReturnJSON(message="{} has dbGaP authorization for {} but is not currently a member of the controlled-access google group. {}" - .format(user_email, program, warn_message), - dbGaP_authorized=False, - dbGaP_allowed=True) diff --git a/apiv4/__init__.py b/apiv4/__init__.py new file mode 100644 index 00000000..d8b2cf00 --- /dev/null +++ b/apiv4/__init__.py @@ -0,0 +1,104 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import os +from os.path import join, dirname +import sys +import ruamel.yaml +import json +from flask import Flask, jsonify, request +from flask_cors import cross_origin +from flask_talisman import Talisman + +app = Flask(__name__, static_folder='api_static') +Talisman(app, strict_transport_security_max_age=300, content_security_policy={ + 'default-src': [ + '\'self\'', + '*.googleapis.com', + '*.swagger.io', + '\'unsafe-inline\'', + 'data:' + ] +}) + +import django +django.setup() +from django.conf import settings + +from auth import auth_info +from main_routes import * +from cohorts_routes import * +from program_routes import * +from sample_case_routes import * +from file_routes import * +from user_routes import * + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.context_processor +def utilities(): + def load_spec(): + json_spec = "" + try: + yaml = ruamel.yaml.YAML(typ='safe') + logger.debug(os.path.split(os.path.abspath(dirname(__file__)))[0] + '/openapi-appengine.yaml') + with open(os.path.split(os.path.abspath(dirname(__file__)))[0] + '/openapi-appengine.yaml') as fpi: + data = yaml.load(fpi) + del data['paths']['/swagger'] + del data['paths']['/oauth2callback'] + # We need to adjust the security definition for use with Swagger UI itself (as opposed to the deployed API) + data['securityDefinitions']['google_id_token'] = { + 'type': 'oauth2', + 'authorizationUrl': "https://accounts.google.com/o/oauth2/v2/auth", + 'tokenUrl': 'https://www.googleapis.com/oauth2/v4/token', + 'flow': 'implicit', + 'scopes': {"https://www.googleapis.com/auth/userinfo.email": "User email address", "openid": "For OIDC"}, + 'x-tokenName': 'id_token' + } + # Escape the ' or the JS will be sad + json_spec = json.dumps(data).replace("'", "\\'") + except Exception as e: + logger.error("[ERROR] While reading YAML spec:") + logger.exception(e) + return json_spec + + return dict( + load_spec=load_spec, + static_uri=(settings.STATIC_URL.replace('/static/', '')), + api_base_uri=settings.BASE_API_URL, + ouath2_callback_path="oauth2callback", + api_client_id=settings.API_CLIENT_ID + ) + + +# Error handlers +@app.errorhandler(500) +def unexpected_error(e): + """Handle exceptions by returning swagger-compliant json.""" + logging.error('[ERROR] An error occurred while processing the request:') + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Exception: {}'.format(e) + }) + response.status_code = 500 + return response + + +if __name__ == '__main__': + app.run(host='127.0.0.1', port=8090, debug=True) diff --git a/apiv4/auth.py b/apiv4/auth.py new file mode 100644 index 00000000..cce6afaf --- /dev/null +++ b/apiv4/auth.py @@ -0,0 +1,167 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import base64 +import json +import requests +import django +from flask import request, jsonify +from django.conf import settings +# from cohorts.metadata_helpers import get_acls_by_uuid +from django.contrib.auth.models import User as Django_User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from cohorts.models import Cohort_Perms +from accounts.sa_utils import auth_dataset_whitelists_for_user +# from accounts.dcf_support import refresh_at_dcf, TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired + +logger = logging.getLogger(settings.LOGGER_NAME) + + +class UserValidationException(Exception): + pass + + +# BEGIN METHODS +def _base64_decode(encoded_str): + # Add paddings manually if necessary. + num_missed_paddings = 4 - len(encoded_str) % 4 + if num_missed_paddings != 4: + encoded_str += b'=' * num_missed_paddings + return base64.b64decode(encoded_str).decode('utf-8') + + +def auth_info(): + """Retrieves the authenication information from Google Cloud Endpoints.""" + encoded_info = request.headers.get('X-Endpoint-API-UserInfo', None) + + if encoded_info: + info_json = _base64_decode(encoded_info) + user_info = json.loads(info_json) + if 'email' not in user_info: + raise UserValidationException("Couldn't obtain user email - the correct scopes may not have been provided during authorization!") + else: + logger.info("[STATUS] No user encoded info found.") + user_info = {'id': 'anonymous', 'email': 'Anonymous'} + + return user_info + + +def get_user(user_email=None): + # Assume this is for the user information in the request header if none is + # provided (it could be for someone else on a project, etc.) + if not user_email: + user_email = auth_info()['email'] + + user = None + + django.setup() + try: + user = Django_User.objects.get(email=user_email) + except ObjectDoesNotExist as e: + logger.warn("User {} does not exist in our system.".format(user_email)) + raise UserValidationException( + "User {} wasn't found in our system.".format(user_email) + + " Please register with our Web Application first: " + ) + + return user + + +def validate_user(user_email=None, cohort_id=None, uuids=None): + user = get_user() + + if not user_email: + user_email = user.email + + try: + if cohort_id: + Cohort_Perms.objects.get(cohort_id=cohort_id, user_id=user.id) + except ObjectDoesNotExist as e: + logger.warn("Error retrieving cohort {} for user {}: {}".format(cohort_id, user_email, e)) + raise UserValidationException( + "User {} does not have access to cohort {}.".format(user_email, cohort_id) + + " Please contact this cohort owner to obtain permission." + ) + + # if uuids: + # acls_needed = get_acls_by_uuid(uuids) + # user_acls = get_user_acls(user) + # + # logger.info("User ACLs: {}".format(str(user_acls))) + # logger.info("ACLs needed: {}".format(str(acls_needed))) + # inaccessible = [] + # + # for acl in acls_needed: + # if acl not in user_acls: + # inaccessible.append(acl) + # + # if len(inaccessible): + # logger.warn( + # "User {} does not have access to one or more programs in this barcode set.".format(user_email)) + # raise UserValidationException( + # "User {} does not have access to one or more programs in this barcode set.".format(user_email) + + # " Please double-check that you have linked the email '{}' to your eRA Commons ID via DCF and are currently signed in." + # ) + + return user + + +def get_user_acls(user): + user_acls = auth_dataset_whitelists_for_user(user.id) + + if not user_acls: + raise UserValidationException("Couldn't verify user controlled data access for user {}.".format(user.email) + + " Please visit the web application at and attempt a login to DCF from" + + " your Account Settings page, then verify your controlled dataset access." + ) + + + # try: + # err_msg, expr_str, _ = refresh_at_dcf(user.id) + # exception_msg = None + # if err_msg: + # logger.warn(err_msg) + # exception_msg = "User {} not currently logged in via DCF and failed to refresh.".format(user.email) + \ + # " Please visit the web application at and attempt a login to DCF from" + \ + # " your Account Settings page." + # else: + # user_acls = auth_dataset_whitelists_for_user(user.id) + # + # if not user_acls: + # exception_msg = "Couldn't verify user controlled data access for user {}.".format(user.email) + \ + # " Please visit the web application at and attempt a login to DCF from" + \ + # " your Account Settings page, then verify your controlled dataset access." + # + # if exception_msg: + # raise UserValidationException(exception_msg) + # + # except (TokenFailure, InternalTokenError, DCFCommFailure, RefreshTokenExpired) as e: + # if type(e) is RefreshTokenExpired: + # raise UserValidationException( + # "Unable to refresh your 24 hour access to controlled data. Please log in to Web " + # + "Application at and visit your Account Details page to refresh " + # + "your controlled dataset access.") + # else: + # if type(e) is DCFCommFailure: + # msg = "Unable to communicate with DCF while attempting to refresh user access for {}.".format(user.email) + # else: + # msg = "There is an internal inconsistency with user tokens for user {}".format(user.email) + # raise Exception(msg) + + return user_acls + +# END METHODS diff --git a/apiv4/cohorts_routes.py b/apiv4/cohorts_routes.py new file mode 100644 index 00000000..a3608868 --- /dev/null +++ b/apiv4/cohorts_routes.py @@ -0,0 +1,264 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from cohorts_views import get_cohort_info, get_cohorts, get_file_manifest, get_cohort_counts, create_cohort, edit_cohort +from auth import auth_info, UserValidationException, validate_user +from django.conf import settings +from django.db import close_old_connections + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/v4/cohorts//', methods=['GET', 'PATCH', 'DELETE'], strict_slashes=False) +def cohort(cohort_id): + """ + GET: Retrieve extended information for a specific cohort + PATCH: Edit an extent cohort + """ + + try: + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + response = None + code = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + if request.method == 'GET': + include_barcodes = (request.args.get('include_barcodes', default="false", type=str).lower() == "true") + cohort_info = get_cohort_info(cohort_id, include_barcodes) + else: + cohort_info = edit_cohort(cohort_id, user, delete=(request.method == 'DELETE')) + + if cohort_info: + response_obj = {} + + if 'message' in cohort_info: + code = 400 + else: + code = 200 + + response_obj['data'] = cohort_info + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + else: + response = jsonify({ + 'code': 404, + 'message': "Cohort ID {} was not found.".format(str(cohort_id))}) + response.status_code = 404 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve this cohort\'s information.' + }) + response.status_code = 500 + finally: + close_old_connections() + + return response + + +@app.route('/v4/cohorts/', methods=['GET', 'POST'], strict_slashes=False) +def cohorts(): + """ + GET: Retrieve a user's list of cohorts + POST: Add a new cohort + """ + + response = None + info = None + code = None + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + if request.method == 'GET': + info = get_cohorts(user_info['email']) + else: + info = create_cohort(user) + + if info: + response_obj = {} + + if 'message' in info: + code = 400 + else: + code = 200 + + response_obj['data'] = info + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Error while attempting to {}.".format( + 'retrieve the cohort list' if request.method == 'GET' else 'create this cohort' + ) + }) + response.status_code = 500 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to {}.'.format( + "retrieve a list of cohorts" if request.method == 'GET' else "create this cohort" + ) + }) + response.status_code = 500 + finally: + close_old_connections() + + return response + + +@app.route('/v4/cohorts//file_manifest/', methods=['POST', 'GET'], strict_slashes=False) +def cohort_file_manifest(cohort_id): + """ + GET: Retrieve a cohort's file manifest + POST: Retrieve a cohort's file manifest with applied filters + """ + + response_obj = None + code = None + + try: + user_info = auth_info() + user = validate_user(user_info['email'], cohort_id) + + if not user: + response_obj = { + 'message': 'Encountered an error while attempting to identify this user.' + } + code = 500 + else: + file_manifest = get_file_manifest(cohort_id, user) + if file_manifest: + # Presence of a message means something went wrong with our request + if 'message' in file_manifest: + response_obj = file_manifest + code = 400 + else: + code = 200 + response_obj = { + 'data': file_manifest + } + else: + response_obj = { + 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)) + } + code = 500 + + except UserValidationException as e: + response_obj = { + 'message': str(e) + } + code = 403 + except Exception as e: + logger.exception(e) + response_obj = { + 'message': 'Encountered an error while attempting to identify this user.' + } + code = 500 + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response + + +@app.route('/v4/cohorts/preview/', methods=['POST'], strict_slashes=False) +def cohort_preview(): + """List the samples, cases, and counts a given set of cohort filters would produce""" + + code = None + response_obj = None + + try: + cohort_counts = get_cohort_counts() + + if cohort_counts: + # Presence of a message means something went wrong with the filters we received + if 'message' in cohort_counts: + response_obj = cohort_counts + code = 400 + else: + response_obj = { + 'data': cohort_counts + } + code = 200 + + # Lack of a valid object means something went wrong on the server + else: + response_obj = { + 'message': "Error while attempting to retrieve case and sample counts for these filters." + } + code = 500 + + except Exception as e: + logger.exception(e) + response_obj = { + 'message': 'Encountered an error while attempting to build this cohort preview.' + } + code = 500 + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response diff --git a/apiv4/cohorts_views.py b/apiv4/cohorts_views.py new file mode 100644 index 00000000..2b18e987 --- /dev/null +++ b/apiv4/cohorts_views.py @@ -0,0 +1,282 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +import django +import re + +from flask import request +from werkzeug.exceptions import BadRequest + +from django.contrib.auth.models import User as Django_User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.conf import settings + +from cohorts.models import Cohort_Perms, Cohort, Filters +from accounts.sa_utils import auth_dataset_whitelists_for_user +from cohorts.file_helpers import cohort_files +from cohorts.utils import get_sample_case_list_bq, create_cohort as make_cohort +from projects.models import Program + +from jsonschema import validate as schema_validate, ValidationError +from schemas.cohort_filter_schema import COHORT_FILTER_SCHEMA + +BLACKLIST_RE = settings.BLACKLIST_RE + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_file_manifest(cohort_id, user): + file_manifest = None + inc_filters = {} + + try: + has_access = auth_dataset_whitelists_for_user(user.id) + + params = { + 'limit': settings.MAX_FILE_LIST_REQUEST, + 'build': 'HG19', + 'access': has_access + } + + request_data = request.get_json() + + param_set = { + 'offset': {'default': 0, 'type': int, 'name': 'offset'}, + 'page': {'default': 1, 'type': int, 'name': 'page'}, + 'fetch_count': {'default': 5000, 'type': int, 'name': 'limit'}, + 'genomic_build': {'default': "HG19", 'type': str, 'name': 'build'} + } + + for param, parameter in param_set.items(): + default = parameter['default'] + param_type = parameter['type'] + name = parameter['name'] + params[name] = request_data[param] if (request_data and param in request_data) else request.args.get(param, default=default, type=param_type) if param in request.args else default + + if request_data: + inc_filters = { + filter: request_data[filter] + for filter in request_data.keys() + if filter not in list(param_set.keys()) + } + + response = cohort_files(cohort_id, user=user, inc_filters=inc_filters, **params) + + file_manifest = response['file_list'] if response and response['file_list'] else None + + except BadRequest as e: + logger.warn("[WARNING] Received bad request - couldn't load JSON.") + file_manifest = { + 'message': 'The JSON provided in this request appears to be improperly formatted.', + } + except Exception as e: + logger.error("[ERROR] File trieving the file manifest for Cohort {}:".format(str(cohort_id))) + logger.exception(e) + + return file_manifest + + +def get_cohort_info(cohort_id, get_barcodes=False): + cohort = None + try: + cohort_obj = Cohort.objects.get(id=cohort_id) + cohort = { + 'id': cohort_obj.id, + 'name': cohort_obj.name, + 'case_count': cohort_obj.case_size(), + 'sample_count': cohort_obj.sample_size(), + 'programs': cohort_obj.get_program_names(), + 'filters': cohort_obj.get_current_filters(True) + } + + if get_barcodes: + cohort['barcodes'] = get_sample_case_list_bq(cohort_id) + + except ObjectDoesNotExist as e: + logger.warn("Cohort with ID {} was not found!".format(str(cohort_id))) + except Exception as e: + logger.exception(e) + + return cohort + + +def get_cohorts(user_email): + + cohort_list = None + + try: + user = Django_User.objects.get(email=user_email) + cohort_perms = Cohort_Perms.objects.filter(user_id=user.id, cohort__active=1) + cohort_list = [] + for cohort_perm in cohort_perms: + cohort_list.append({ + 'id': cohort_perm.cohort.id, + 'name': cohort_perm.cohort.name, + 'permission': cohort_perm.perm, + 'filters': cohort_perm.cohort.get_current_filters(True) + }) + + except ObjectDoesNotExist as e: + logger.info("No cohorts found for user {}!".format(user_email)) + + return cohort_list + + +def get_cohort_counts(): + + cohort_counts = None + + try: + request_data = request.get_json() + schema_validate(request_data, COHORT_FILTER_SCHEMA) + + if 'filters' not in request_data: + cohort_counts = { + 'message': 'No filters were provided; ensure that the request body contains a \'filters\' property.' + } + else: + cohort_counts = get_sample_case_list_bq(None, request_data['filters']) + + if cohort_counts: + for prog in cohort_counts: + if cohort_counts[prog]['case_count'] <= 0: + cohort_counts[prog]['message'] = "No cases or samples found which meet the filter criteria for this program." + cohort_counts[prog]['provided_filters'] = request_data['filters'][prog] + + except BadRequest as e: + logger.warn("[WARNING] Received bad request - couldn't load JSON.") + cohort_counts = { + 'message': 'The JSON provided in this request appears to be improperly formatted.', + } + except ValidationError as e: + logger.warn('[WARNING] Filters rejected for improper formatting: {}'.format(e)) + cohort_counts = { + 'message': 'Filters were improperly formatted.' + } + except Exception as e: + logger.exception(e) + + return cohort_counts + + +def create_cohort(user): + cohort_info = None + + try: + request_data = request.get_json() + schema_validate(request_data, COHORT_FILTER_SCHEMA) + + if 'name' not in request_data: + cohort_info = { + 'message': 'A name was not provided for this cohort. The cohort was not made.', + } + return cohort_info + + if 'filters' not in request_data: + cohort_info = { + 'message': 'Filters were not provided; at least one filter must be provided for a cohort to be valid.' + + ' The cohort was not made.', + } + return cohort_info + + blacklist = re.compile(BLACKLIST_RE, re.UNICODE) + match = blacklist.search(str(request_data['name'])) + + if not match and 'desc' in request_data: + match = blacklist.search(str(request_data['desc'])) + + if match: + cohort_info = { + 'message': 'Your cohort\'s name or description contains invalid characters; please edit them and resubmit. ' + + '[Saw {}]'.format(str(match)), + } + + else: + result = make_cohort(user, **request_data) + + if 'message' in result: + cohort_info = result + else: + cohort_info = get_cohort_info(result['cohort_id']) + + except BadRequest as e: + logger.warn("[WARNING] Received bad request - couldn't load JSON.") + cohort_info = { + 'message': 'The JSON provided in this request appears to be improperly formatted.', + } + + except ValidationError as e: + logger.warn("[WARNING] Cohort information rejected for improper formatting: {}".format(e)) + cohort_info = { + 'message': 'Cohort information was improperly formatted - cohort not edited.', + } + + return cohort_info + + +def edit_cohort(cohort_id, user, delete=False): + match = None + + try: + if delete: + cohort = Cohort.objects.get(id=cohort_id) + cohort.active = False + cohort.save() + cohort_info = { + 'notes': 'Cohort {} (\'{}\') has been deleted.'.format(cohort_id, cohort.name), + 'data': {'filters': cohort.get_current_filters(unformatted=True)}, + } + else: + request_data = request.get_json() + if len(request_data.keys()): + schema_validate(request_data, COHORT_FILTER_SCHEMA) + + if 'name' in request_data: + blacklist = re.compile(BLACKLIST_RE, re.UNICODE) + match = blacklist.search(str(request_data['name'])) + + if match: + cohort_info = { + 'message': 'Your cohort\'s name or description contains invalid characters; please edit them and resubmit. ' + + '[Saw {}]'.format(str(match)), + } + else: + result = make_cohort(user, source_id=cohort_id, **request_data) + if 'message' in result: + cohort_info = result + else: + cohort_info = get_cohort_info(result['cohort_id']) + + + except BadRequest as e: + logger.warn("[WARNING] Received bad request - couldn't load JSON.") + cohort_info = { + 'message': 'The JSON provided in this request appears to be improperly formatted.', + } + + except ObjectDoesNotExist as e: + logger.error("[ERROR] During {} for cohort ID {}:".format(request.method,str(cohort_id))) + logger.error("Couldn't find a cohort with that ID!") + + except ValidationError as e: + logger.warn("[WARNING] Cohort information rejected for improper formatting: {}".format(e)) + cohort_info = { + 'message': 'Cohort information was improperly formatted - cohort not edited.', + } + + return cohort_info diff --git a/apiv4/file_routes.py b/apiv4/file_routes.py new file mode 100644 index 00000000..230d69ae --- /dev/null +++ b/apiv4/file_routes.py @@ -0,0 +1,202 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from django.conf import settings +from django.db import close_old_connections +from auth import validate_user, UserValidationException +from file_views import get_file_paths, get_signed_uris + +logger = logging.getLogger(settings.LOGGER_NAME) + + +# @app.route('/v4/files/signed_uris//', methods=['GET'], strict_slashes=False) +# def signed_uri(file_uuid): +# response = None +# +# request_data = request.get_json() +# +# try: +# user = validate_user(uuids=[file_uuid]) +# signed_uris = get_signed_uris(user, file_uuid) +# +# if signed_uris: +# response = jsonify({ +# 'code': 200, +# 'data': signed_uris, +# 'README': '' +# }) +# response.status_code = 200 +# else: +# response = jsonify({ +# 'code': 404, +# 'message': "File UUID {} was not found.".format(file_uuid)}) +# response.status_code = 404 +# +# except UserValidationException as e: +# response = jsonify({ +# 'code': 403, +# 'message': str(e) +# }) +# response.status_code = 403 +# +# except Exception as e: +# logger.exception(e) +# response = jsonify({ +# 'code': 500, +# 'message': 'Encountered an error while attempting to retrieve signed URIs for file UUID {}.'.format(file_uuid) +# }) +# response.status_code = 500 +# +# return response + + +# @app.route('/v4/files/signed_uris/', methods=['POST'], strict_slashes=False) +# def signed_uri_list(): +# +# response = None +# +# request_data = request.get_json() +# +# try: +# +# if 'uuids' not in request_data: +# response = jsonify({ +# 'code': 400, +# 'message': "File UUIDs not provided in data payload." +# }) +# response.status_code = 400 +# else: +# user = validate_user() +# signed_uris = get_signed_uris(user, request_data['uuids']) +# +# if signed_uris: +# response = jsonify({ +# 'code': 200, +# 'data': signed_uris, +# 'README': '' +# }) +# response.status_code = 200 +# else: +# response = jsonify({ +# 'code': 404, +# 'message': "The provided file UUIDs were not found."}) +# response.status_code = 404 +# +# except UserValidationException as e: +# response = jsonify({ +# 'code': 403, +# 'message': str(e) +# }) +# response.status_code = 403 +# +# except Exception as e: +# logger.exception(e) +# response = jsonify({ +# 'code': 500, +# 'message': 'Encountered an error while attempting to retrieve signed URIs for these file UUIDs.' +# }) +# response.status_code = 500 +# +# return response + + +@app.route('/v4/files/paths//', methods=['GET'], strict_slashes=False) +def file_path(file_uuid): + resp_obj = None + code = None + + try: + file_paths = get_file_paths(file_uuid) + + if file_paths: + if 'message' in file_paths: + resp_obj = file_paths + code = 400 + if 'not_found' in file_paths: + code = 404 + else: + resp_obj = { + 'data': file_paths + } + code = 200 + else: + resp_obj = { + 'message': 'Encountered an error while retrieving file paths.' + } + code = 500 + + except Exception as e: + logger.exception(e) + resp_obj = { + 'message': 'Encountered an error while retrieving file paths.' + } + code = 500 + finally: + close_old_connections() + + resp_obj['code'] = code + response = jsonify(resp_obj) + response.status_code = code + + return response + + +@app.route('/v4/files/paths/', methods=['POST'], strict_slashes=False) +def file_path_list(): + + response_obj = None + + try: + file_paths = get_file_paths() + + if file_paths: + # Presence of a message means something went wrong with our request + if 'message' in file_paths: + response_obj = file_paths + code = 400 + if 'not_found' in file_paths: + code = 404 + else: + response_obj = { + 'data': file_paths + } + code = 200 + + else: + response_obj = { + 'message': "Error while attempting to retrieve file manifest for cohort {}.".format(str(cohort_id)) + } + code = 500 + + except Exception as e: + logger.exception(e) + response_obj = { + 'message': 'Encountered an error while attempting to retrieve file paths for these file UUIDs.' + } + code = 500 + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response + diff --git a/apiv4/file_views.py b/apiv4/file_views.py new file mode 100644 index 00000000..2996b3e0 --- /dev/null +++ b/apiv4/file_views.py @@ -0,0 +1,85 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +import django + +from flask import request +from werkzeug.exceptions import BadRequest + +from django.conf import settings +from cohorts.metadata_helpers import get_paths_by_uuid + +from auth import UserValidationException + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_file_paths(uuid=None): + + result = None + uuids = None + + try: + + if uuid: + uuids = [uuid] + else: + request_data = request.get_json() + if 'uuids' in request_data: + uuids = request_data['uuids'] + + if not uuids or not len(uuids): + result = { + 'message': "File UUIDs not provided in data payload." + } + else: + paths, not_found = get_paths_by_uuid(uuids) + if not len(paths): + result = { + 'message': "No file paths were found for the provided UUIDs.", + 'not_found': uuids + } + else: + result = { + 'paths': paths + } + if len(not_found): + result['notes'] = "File paths were not found for all UUIDs. Please see 'uuids_not_found' for a list." + result['not_found'] = not_found + + except BadRequest as e: + logger.warn("[WARNING] Received bad request - couldn't load JSON.") + result = { + 'message': 'The JSON provided in this request appears to be improperly formatted.', + } + + return result + + +def get_signed_uris(user, file_uuids): + # if not user: + # logger.error("A user was not provided while attempting to obtained signed URIs!") + # raise UserValidationException("A user was not provided while attempting to obtained signed URIs!") + # if not file_uuids or not len(file_uuids): + # raise Exception("While attempting to obtain signed URIs, encountered an error: no file UUIDs were provided.") + # + return [] + + + + diff --git a/apiv4/main_routes.py b/apiv4/main_routes.py new file mode 100644 index 00000000..5fb3b23b --- /dev/null +++ b/apiv4/main_routes.py @@ -0,0 +1,51 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from flask import jsonify, request, render_template +from django.conf import settings +from apiv4 import app + +logger = logging.getLogger(settings.LOGGER_NAME) + +SCOPE = 'https://www.googleapis.com/auth/userinfo.email' + + +@app.route('/v4/about/', methods=['GET'], strict_slashes=False) +def apiv4(): + """Base response""" + response = jsonify({ + 'code': 200, + 'message': 'Welcome to the ISB-CGC API, Version 4.', + 'documentation': 'SwaggerUI interface available at <{}/swagger/>.'.format(settings.BASE_API_URL) + + 'Documentation available at ' + }) + response.status_code = 200 + return response + + +# Swagger UI +@app.route('/v4/swagger/', methods=['GET'], strict_slashes=False) +def swagger(): + return render_template('swagger/index.html') + + +@app.route('/v4/oauth2callback/', strict_slashes=False) +def oauth2callback(): + return render_template('swagger/oauth2-redirect.html') + + diff --git a/apiv4/program_routes.py b/apiv4/program_routes.py new file mode 100644 index 00000000..8f88c5dd --- /dev/null +++ b/apiv4/program_routes.py @@ -0,0 +1,60 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from django.conf import settings +from django.db import close_old_connections +from program_views import get_programs + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/v4/programs/', methods=['GET'], strict_slashes=False) +def programs(): + """Retrieve the list of programs and builds currently available for cohort creation.""" + response = None + + try: + + program_info = get_programs() + + if program_info: + response = jsonify({ + 'code': 200, + 'data': program_info + }) + response.status_code = 200 + else: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while retrieving the program list.' + }) + response.status_code = 500 + except Exception as e: + logger.error("[ERROR] While retrieving program information:") + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while retrieving the program list.' + }) + response.status_code = 500 + finally: + close_old_connections() + + return response diff --git a/apiv4/program_views.py b/apiv4/program_views.py new file mode 100644 index 00000000..dabb1d0d --- /dev/null +++ b/apiv4/program_views.py @@ -0,0 +1,45 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +import django + +from flask import request + +from django.conf import settings + +from projects.models import Program, Project + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_programs(): + django.setup() + program_info = None + try: + program_info = [ + { + 'name': x.name, + 'description': x.description, + 'projects': [{'name': y.name, 'description': y.description} for y in Project.objects.filter(program=x,active=1)] + } + for x in Program.get_public_programs() + ] + except Exception as e: + logger.exception(e) + + return program_info diff --git a/apiv4/sample_case_routes.py b/apiv4/sample_case_routes.py new file mode 100644 index 00000000..c1740143 --- /dev/null +++ b/apiv4/sample_case_routes.py @@ -0,0 +1,195 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from django.conf import settings +from django.db import close_old_connections +from sample_case_views import get_metadata +from auth import validate_user, UserValidationException + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/v4/samples//', methods=['GET'], strict_slashes=False) +def sample_metadata(sample_barcode): + + response = None + + try: + + metadata = get_metadata(sample_barcode, 'sample') + + if metadata: + if 'message' in metadata: + resp_obj = metadata + code = 400 + if 'barcodes_not_found' in metadata: + code = 404 + else: + resp_obj = { + 'data': metadata + } + code = 200 + else: + resp_obj = { + 'message': 'Encountered an error while retrieving sample metadata.' + } + code = 500 + except Exception as e: + logger.error("[ERROR] While fetching sample metadata:") + logger.exception(e) + resp_obj = { + 'message': 'Encountered an error while retrieving sample metadata.' + } + code = 500 + finally: + close_old_connections() + + resp_obj['code'] = code + response = jsonify(resp_obj) + response.status_code = code + + return response + + +@app.route('/v4/cases//', methods=['GET'], strict_slashes=False) +def case_metadata(case_barcode): + + resp_obj = None + + try: + metadata = get_metadata(case_barcode, 'case') + + if metadata: + if 'message' in metadata: + resp_obj = metadata + code = 400 + if 'barcodes_not_found' in metadata: + code = 404 + else: + resp_obj = { + 'data': metadata + } + code = 200 + else: + resp_obj = { + 'message': 'Encountered an error while retrieving case metadata.' + } + code = 500 + except Exception as e: + logger.error("[ERROR] While fetching case metadata:") + logger.exception(e) + resp_obj = { + 'message': 'Encountered an error while retrieving case metadata for {}.'.format(case_barcode) + } + code = 500 + finally: + close_old_connections() + + resp_obj['code'] = code + response = jsonify(resp_obj) + response.status_code = code + + return response + + +@app.route('/v4/samples/', methods=['POST'], strict_slashes=False) +def sample_metadata_list(): + + resp_obj = None + code = None + + try: + + metadata = get_metadata(type='sample') + + if metadata: + if 'message' in metadata: + resp_obj = metadata + code = 400 + if 'barcodes_not_found' in metadata: + code = 404 + else: + resp_obj = { + 'data': metadata + } + code = 200 + else: + resp_obj = { + 'message': 'Encountered an error while retrieving sample metadata.' + } + code = 500 + + except Exception as e: + logger.error("[ERROR] While fetching sample metadata:") + logger.exception(e) + resp_obj = { + 'message': 'Encountered an error while retrieving sample metadata.' + } + code = 500 + finally: + close_old_connections() + + resp_obj['code'] = code + response = jsonify(resp_obj) + response.status_code = code + + return response + + +@app.route('/v4/cases/', methods=['POST'], strict_slashes=False) +def case_metadata_list(): + + resp_obj = None + code = None + + try: + metadata = get_metadata(type='case') + + if metadata: + if 'message' in metadata: + resp_obj = metadata + code = 400 + if 'barcodes_not_found' in metadata: + code = 404 + else: + resp_obj = { + 'data': metadata + } + code = 200 + else: + resp_obj = { + 'message': 'Encountered an error while retrieving case metadata.' + } + code = 500 + except Exception as e: + logger.error("[ERROR] While fetching case metadata:") + logger.exception(e) + resp_obj = { + 'message': 'Encountered an error while retrieving case metadata.' + } + code = 500 + finally: + close_old_connections() + + resp_obj['code'] = code + response = jsonify(resp_obj) + response.status_code = code + + return response diff --git a/apiv4/sample_case_views.py b/apiv4/sample_case_views.py new file mode 100644 index 00000000..c40b321c --- /dev/null +++ b/apiv4/sample_case_views.py @@ -0,0 +1,73 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +import django + +from flask import request +from werkzeug.exceptions import BadRequest + +from django.conf import settings +from cohorts.metadata_helpers import get_full_case_metadata, get_full_sample_metadata + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_metadata(barcode=None, type=None): + + result = None + barcodes = None + + try: + if barcode: + barcodes = [barcode] + else: + request_data = request.get_json() + if 'barcodes' in request_data: + barcodes = request_data['barcodes'] + + if not barcodes or not len(barcodes): + result = { + 'message': 'A list of {} barcodes was not found in this request. Please double-check the expected request JSON format.'.format(type) + } + else: + if type == 'sample': + result = get_full_sample_metadata(barcodes) + else: + result = get_full_case_metadata(barcodes) + if not result or not result['total_found']: + if not result: + result = {} + else: + del result['total_found'] + result['message'] = "No metadata was found for the supplied {} barcodes.".format(type) + else: + if 'not_found' in result: + result['notes'] = "Some {} barcodes provided were not found. See 'not_found' for a list.".format(type) + + except BadRequest as e: + logger.warn("[WARNING] Received bad request - couldn't load JSON.") + result = { + 'message': 'The JSON provided in this request appears to be improperly formatted.', + } + + except Exception as e: + logger.error("[ERROR] While fetching {} metadata: ".format(type)) + logger.exception(e) + + return result + diff --git a/apiv4/schemas/cohort_filter_schema.py b/apiv4/schemas/cohort_filter_schema.py new file mode 100644 index 00000000..f5c18e5c --- /dev/null +++ b/apiv4/schemas/cohort_filter_schema.py @@ -0,0 +1,1907 @@ +COHORT_FILTER_SCHEMA = { + 'type': 'object', + 'properties': { + 'filters': { + 'type': 'object', + 'properties': { + 'TCGA': { + 'type': 'object', + 'properties': { + 'avg_percent_lymphocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_lymphocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_lymphocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_lymphocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_monocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_necrosis_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_neutrophil_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_normal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_stromal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'avg_percent_tumor_nuclei_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'batch_number': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'bcr': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'days_to_collection': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_sample_procurement': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'max_percent_lymphocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_lymphocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_lymphocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_lymphocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_monocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_necrosis_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_neutrophil_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_normal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_stromal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'max_percent_tumor_nuclei_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_lymphocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'min_percent_lymphocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei_lte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'min_percent_lymphocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei_gte': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'min_percent_lymphocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_monocyte_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_necrosis_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_neutrophil_infiltration_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_normal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_stromal_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_cells_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'min_percent_tumor_nuclei_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'number'] } + }, + 'num_portions_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'num_slides_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'pathology_report_uuid': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'preservation_method': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'weight_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_tobacco_smoking_onset_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_began_smoking_in_years_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'batch_number_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'bmi_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'days_to_birth_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_initial_pathologic_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_submitted_specimen_dx_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'height_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'psa_value_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'gleason_score_combined_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'weight_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_tobacco_smoking_onset_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_began_smoking_in_years_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'batch_number_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'bmi_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'days_to_birth_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_initial_pathologic_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_submitted_specimen_dx_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'height_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'psa_value_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'gleason_score_combined_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'weight_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'year_of_tobacco_smoking_onset_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'age_began_smoking_in_years_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'batch_number_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'bmi_btw': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'days_to_birth_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_initial_pathologic_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_submitted_specimen_dx_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'height_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'psa_value_btw': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'gleason_score_combined_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'weight': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'year_of_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'year_of_tobacco_smoking_onset': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'age_at_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'age_began_smoking_in_years': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'anatomic_neoplasm_subdivision': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'batch_number': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'bmi': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'bcr': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'days_to_birth': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_death': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_initial_pathologic_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_last_followup': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_last_known_alive': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'days_to_submitted_specimen_dx': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'height': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'psa_value': { + 'type': ['array', 'string', 'number'], + 'items': { 'type': ['string', 'number'] } + }, + 'gleason_score_combined': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'clinical_M': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'clinical_N': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'clinical_stage': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'clinical_T': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'colorectal_cancer': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'country': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'histological_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'history_of_colon_polyps': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'history_of_neoadjuvant_treatment': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'hpv_calls': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'hpv_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'h_pylori_infection': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'icd_10': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'icd_o_3_histology': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'icd_o_3_site': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'lymphatic_invasion': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'lymphnodes_examined': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'lymphovascular_invasion_present': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'menopause_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'mononucleotide_and_dinucleotide_marker_panel_analysis_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'neoplasm_histologic_grade': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'new_tumor_event_after_initial_treatment': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'number_of_lymphnodes_examined': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string','integer'] } + }, + 'number_of_lymphnodes_positive_by_he': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string','integer'] } + }, + 'number_pack_years_smoked': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string','integer'] } + }, + 'other_dx': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_malignancy_anatomic_site': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_malignancy_histological_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'other_malignancy_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_M': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_N': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_stage': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pathologic_T': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'person_neoplasm_cancer_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'pregnancies': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'primary_neoplasm_melanoma_dx': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'primary_therapy_outcome_success': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'race': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'residual_tumor': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'stopped_smoking_year': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tobacco_smoking_history': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tss_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tumor_tissue_site': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tumor_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'venous_invasion': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'vital_status': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'ethnicity': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'gender': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + } + }, + 'TARGET': { + 'type': 'object', + 'properties': { + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_type': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'tumor_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'program_name': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'program_dbgap_accession_number': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'project_name': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'gender': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'vital_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'race': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ethnicity': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'age_at_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'age_at_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_birth': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_birth_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_birth_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_birth_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_followup_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_last_known_alive_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'days_to_death_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'protocol': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'year_of_diagnosis': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_follow_up': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_follow_up_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_follow_up_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'year_of_last_follow_up_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_time_in_days': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_time_in_days_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_time_in_days_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'event_free_survival_time_in_days_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'first_event': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'WBC_at_diagnosis': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'WBC_at_diagnosis_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'WBC_at_diagnosis_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'WBC_at_diagnosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'MLL_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'CNS_site_of_relapse_or_induction_failure': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'CNS_status_at_diagnosis': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'bone_marrow_site_of_relapse_or_induction_failure': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'other_site_of_relapse_or_induction_failure': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'histology': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'BCR_ABL1_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'BMA_blasts_day_8': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_8_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_8_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_8_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_15': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_15_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_15_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_15_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_29': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_29_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_29_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_29_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_43': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_43_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_43_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'BMA_blasts_day_43_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'DNA_index': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'DNA_index_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'DNA_index_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'DNA_index_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'Down_syndrome': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'MRD_at_end_of_course_1': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_1_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_1_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_1_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_1_YN': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'MRD_at_end_of_course_2': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_2_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_2_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_2_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'MRD_at_end_of_course_2_YN': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'MRD_day_8': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_8_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_8_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_8_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_29': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_29_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_29_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_29_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_43': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_43_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_43_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_day_43_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'MRD_end_consolidation': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_end_consolidation_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_end_consolidation_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'MRD_end_consolidation_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'cell_of_origin': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'testes_site_of_relapse': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'testicular_involvement': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'comment': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'trisomies_4_10_Status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ETV6_RUNX1_fusion_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'karyotype': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'TCF3_PBX1_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'COG_risk_group': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ICDO_description': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'INSS_stage': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'MYCN_status': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ICDO': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'diagnostic_category': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ploidy': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'CEBPA_mutation': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'CNS_disease': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'CR_status_at_end_of_course_1': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'CR_status_at_end_of_course_2': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'FLT3_ITD_positive': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'FLT3_PM': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ISCN': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'MKI': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'NPM_mutation': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'WT1_mutation': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'bone_marrow_leukemic_blast_percentage': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'bone_marrow_leukemic_blast_percentage_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'bone_marrow_leukemic_blast_percentage_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'bone_marrow_leukemic_blast_percentage_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'chloroma': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'cytogenetic_complexity': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'del5q': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'del7q': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'del9q': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'grade': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'inv_16': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'minus_X': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'minus_Y': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'monosomy_5': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'monosomy_7': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'peripheral_blasts_pct': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'peripheral_blasts_pct_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'peripheral_blasts_pct_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'peripheral_blasts_pct_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'primary_cytogenetic_code': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'risk_group': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_10_11_p11_2_q23': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_11_19_q23_p13_1': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_3_5_q25_q34': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_6_11_q27_q23': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_6_9': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_8_21': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 't_9_11_p22_q23': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'trisomy_8': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'trisomy_21': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'FAB_category': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'SCT_in_1st_CR': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'percent_tumor': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ICD_O_3_M': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ICD_O_3_T': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'stage': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'discovery_or_validation': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'ALL_mol_subtype': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'chloroma_site_of_relapse_or_induction_failure': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'cytogenetic_site_of_relapse_or_induction_failure': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'percent_necrosis': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'percent_tumor_vs_stroma': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'site_of_relapse': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'alternate_therapy': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'c_Kit_mutation_exon8': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'c_Kit_mutation_exon17': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'cytogenetic_code_other': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'disease_at_diagnosis': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'primary_tumor_site': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'specific_tumor_site': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'time_to_first_event_in_days': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_event_in_days_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_event_in_days_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_event_in_days_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'FLT3_ITD_allelic_ratio': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'FLT3_ITD_allelic_ratio_lte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'FLT3_ITD_allelic_ratio_gte': { + 'type': ['array', 'string', 'number'], + 'items': {'type': ['string', 'number']} + }, + 'FLT3_ITD_allelic_ratio_btw': { + 'type': 'array', + 'items': {'type': ['string', 'number']} + }, + 'definitive_surgery': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'histologic_response': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'reason_for_death': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'specific_tumor_region': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'specific_tumor_side': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'time_to_first_relapse_in_days': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_relapse_in_days_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_relapse_in_days_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_relapse_in_days_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'alternate_therapy_other': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'metastasis_site': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'primary_site_progression': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'refractory_timepoint_sent_for_induction_failure_project': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'relapse_percent_necrosis': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'relapse_percent_necrosis_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'relapse_percent_necrosis_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'relapse_percent_necrosis_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'relapse_percent_tumor': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'relapse_type': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'therapy': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'time_to_first_SMN_in_days': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_SMN_in_days_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_SMN_in_days_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_SMN_in_days_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_enrollment_on_relapse_protocol_in_days': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_enrollment_on_relapse_protocol_in_days_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_enrollment_on_relapse_protocol_in_days_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'time_to_first_enrollment_on_relapse_protocol_in_days_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + + } + }, + 'CCLE': { + 'type': 'object', + 'properties': { + 'project_short_name': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'sample_type': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'summary_file_count': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_gte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_lte': { + 'type': ['array', 'string', 'integer'], + 'items': {'type': ['string', 'integer']} + }, + 'summary_file_count_btw': { + 'type': 'array', + 'items': {'type': ['string', 'integer']} + }, + 'gender': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'hist_subtype': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'histology': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'site_primary': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + 'source': { + 'type': ['array', 'string'], + 'items': {'type': 'string'} + }, + } + }, + }, + "additionalProperties": False + }, + 'name': { + 'type': 'string' + } + }, + "additionalProperties": False + } diff --git a/apiv4/schemas/file_filter_schema.py b/apiv4/schemas/file_filter_schema.py new file mode 100644 index 00000000..13e24cb9 --- /dev/null +++ b/apiv4/schemas/file_filter_schema.py @@ -0,0 +1,127 @@ +FILE_FILTER_SCHEMA = { + 'type': 'object', + 'properties': { + 'filters': { + 'type': 'object', + 'properties': { + 'access': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'acl': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'case_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'data_category': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'data_format': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'data_type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'disease_code': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'experimental_strategy': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'file_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'file_name_key': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'index_file_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'index_file_name_key': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'platform': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'program_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'project_short_name': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_barcode': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'sample_gdc_id': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'type': { + 'type': ['array', 'string'], + 'items': { 'type': 'string' } + }, + 'file_size': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'file_size_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size_lte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'file_size_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size_gte': { + 'type': ['array', 'string', 'integer'], + 'items': { 'type': ['string', 'integer'] } + }, + 'file_size_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'index_file_size_btw': { + 'type': 'array', + 'items': { 'type': ['string', 'integer'] } + }, + 'fetch_count': { + 'type': 'integer' + }, + 'offset': { + 'type': 'integer' + }, + 'genomic_build': { + 'type': 'string' + }, + } + } + } +} \ No newline at end of file diff --git a/apiv4/templates/swagger/index.html b/apiv4/templates/swagger/index.html new file mode 100644 index 00000000..6c91aca2 --- /dev/null +++ b/apiv4/templates/swagger/index.html @@ -0,0 +1,101 @@ + + + + + + ISB-CGC API - Swagger UI + + + + + + + +
+ + + + + + diff --git a/apiv4/templates/swagger/oauth2-redirect.html b/apiv4/templates/swagger/oauth2-redirect.html new file mode 100644 index 00000000..12286708 --- /dev/null +++ b/apiv4/templates/swagger/oauth2-redirect.html @@ -0,0 +1,68 @@ + + + + + + diff --git a/apiv4/user_routes.py b/apiv4/user_routes.py new file mode 100644 index 00000000..c264ac03 --- /dev/null +++ b/apiv4/user_routes.py @@ -0,0 +1,346 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +from flask import jsonify, request +from apiv4 import app +from auth import auth_info, UserValidationException, validate_user, get_user +from user_views import get_user_acls, get_account_details, gcp_validation, gcp_registration, gcp_unregistration, gcp_info +from django.conf import settings +from django.db import close_old_connections + +logger = logging.getLogger(settings.LOGGER_NAME) + + +@app.route('/v4/users/account_details/', methods=['GET'], strict_slashes=False) +def account_details(): + """ + GET: Retrieve extended information for a specific user + """ + + try: + user_info = auth_info() + user = get_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + + account_info = get_account_details(user) + + if account_info: + response_obj = {} + code = None + + if 'message' in account_info: + code = 400 + else: + code = 200 + response_obj['data'] = account_info + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + else: + response = jsonify({ + 'code': 404, + 'message': "Unable to retrieve information for {}.".format(str(user_info['email']))}) + response.status_code = 404 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to retrieve user information.' + }) + response.status_code = 500 + finally: + close_old_connections() + + return response + + +@app.route('/v4/users/gcp/validate//', methods=['GET'], strict_slashes=False) +def validate_gcp(gcp_id): + """ + GET: Validate a Google Cloud Project for registration and return the results to the user + """ + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + response = None + + if not user: + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to identify this user.' + }) + response.status_code = 500 + else: + validation = gcp_validation(user, gcp_id) + + if validation: + response_obj = {} + code = None + + if 'message' in validation: + response_obj['message'] = validation['message'] + if 'notes' in validation: + response_obj['notes'] = validation['notes'] + + if 'roles' not in validation: + code = 400 + else: + code = 200 + response_obj['gcp_project_id'] = validation['gcp_id'] + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + # Lack of a valid object means something went wrong on the server + else: + response = jsonify({ + 'code': 500, + 'message': "Encountered an error while attempting to validate Google Cloud Platform project ID {}.".format(gcp_id) + }) + response.status_code = 500 + + except UserValidationException as e: + response = jsonify({ + 'code': 403, + 'message': str(e) + }) + response.status_code = 403 + except Exception as e: + logger.exception(e) + response = jsonify({ + 'code': 500, + 'message': 'Encountered an error while attempting to validate Google Cloud Platform project ID {}.'.format(gcp_id) + }) + response.status_code = 500 + finally: + close_old_connections() + + return response + + +@app.route('/v4/users/gcp//', methods=['POST', 'DELETE', 'PATCH', 'GET'], strict_slashes=False) +def user_gcp(gcp_id): + """ + POST: Register a Google Cloud Project with ISB-CGC + PATCH: Update the Google Cloud Project's user list with ISB-CGC + DELETE: Unregister the Google Cloud Project with ISB-CGC + GET: Fetch details about the Google Cloud Project + """ + + response_obj = {} + code = None + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + if not user: + response_obj = { + 'message': 'Encountered an error while attempting to identify this user.' + } + code = 500 + else: + action = None + result = None + success = False + + if request.method == 'POST' or request.method == 'PATCH': + action, success = gcp_registration(user, gcp_id, (request.method == 'PATCH')) + elif request.method == 'GET': + result, success = gcp_info(user, gcp_id) + elif request.method == 'DELETE': + action, success = gcp_unregistration(user, gcp_id) + else: + raise Exception("Method not recognized: {}".format(request.method)) + + code = 200 + + if action is not None: + response_obj['gcp_project_id'] = gcp_id + if 'message' in action: + response_obj['message'] = action['message'] + if 'notes' in action: + response_obj['notes'] = action['notes'] + if not success: + code = 400 + elif result is not None: + # The case of an empty result set is handled above + if success: + response_obj['data'] = result + else: + code = 404 + response_obj['message'] = 'A Google Cloud Platform project with ID {} was not found for user {}'.format( + gcp_id, user.email + ) + + # Lack of a valid object means something went wrong on the server + else: + code = 500 + act = "fetch" + if request.method == 'POST': + act = "register" + if request.method == 'DELETE': + act = "unregister" + if request.method == "PATCH": + act = "refresh" + response_obj = { + 'message': "Encountered an error while attempting to {} Google Cloud Platform project ID {}.".format( + act, + gcp_id + ) + } + + except UserValidationException as e: + code = 403 + response_obj = { + 'message': str(e) + } + + except Exception as e: + logger.error("[ERROR] For route /v4/users/gcp/ method {}:".format(request.method)) + logger.exception(e) + code = 500 + response_obj = { + 'message': 'Encountered an error while attempting to register Google Cloud Platform project ID {}.'.format( + gcp_id) + } + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response + +@app.route('/v4/users/gcp/', methods=['POST', 'GET'], strict_slashes=False) +def user_gcps(): + """ + POST: Register a Google Cloud Project with ISB-CGC + PATCH: Update the Google Cloud Project's user list with ISB-CGC + DELETE: Unregister the Google Cloud Project with ISB-CGC + GET: Fetch details about the Google Cloud Project + """ + + response_obj = {} + code = None + gcp_id = None + + try: + user_info = auth_info() + user = validate_user(user_info['email']) + + if not user: + response_obj = { + 'message': 'Encountered an error while attempting to identify this user.' + } + code = 500 + else: + action = None + result = None + success = None + + if request.method == 'POST': + gcp_id = request.args.get('project_id', default=None, type=string) + if gcp_id: + action, success = gcp_registration(user, gcp_id, False) + elif request.method == 'GET': + result, success = gcp_info(user) + else: + raise Exception("Method not recognized: {}".format(request.method)) + + if not success: + if result is not None: + code = 404 + response_obj['message'] = 'No Google Cloud Platform projects found for user {}'.format( + gcp_id, user.email + ) + else: + code = 200 + + if action: + if 'message' in action: + response_obj['message'] = action['message'] + if 'notes' in action: + response_obj['notes'] = action['notes'] + if success: + response_obj['gcp_project_id'] = action['gcp_id'] + elif result: + response_obj['data'] = result + + # Lack of a valid object means something went wrong on the server + else: + code = 500 + act = "fetch the registered GCP project list for user {}".format(user.email) + if request.method == 'POST': + if gcp_id: + act = "register Google Cloud Platform project ID {}".format(gcp_id) + else: + act = "register a Google Cloud Platform project" + response_obj = { + 'message': "Encountered an error while attempting to {}".format( + act + ) + } + + except UserValidationException as e: + code = 403 + response_obj = { + 'message': str(e) + } + + except Exception as e: + logger.error("[ERROR] For route /v4/users/gcp/{gcp_id} method {}:".format(request.method)) + logger.exception(e) + code = 500 + response_obj = { + 'message': 'Encountered an error while attempting to register Google Cloud Platform project ID {}.'.format( + gcp_id) + } + finally: + close_old_connections() + + response_obj['code'] = code + response = jsonify(response_obj) + response.status_code = code + + return response + + + + + diff --git a/apiv4/user_views.py b/apiv4/user_views.py new file mode 100644 index 00000000..32a18190 --- /dev/null +++ b/apiv4/user_views.py @@ -0,0 +1,164 @@ +# +# Copyright 2019, Institute for Systems Biology +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import json +import django +import re + +from flask import request +from werkzeug.exceptions import BadRequest + +from django.contrib.auth.models import User as Django_User +from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned +from django.conf import settings + +from accounts.sa_utils import auth_dataset_whitelists_for_user +from accounts.utils import register_or_refresh_gcp, verify_gcp_for_reg, api_gcp_delete, get_user_gcps +from accounts.models import AuthorizedDataset +from projects.models import Program +from auth import get_user_acls, UserValidationException + +from jsonschema import validate as schema_validate, ValidationError + +BLACKLIST_RE = settings.BLACKLIST_RE + +logger = logging.getLogger(settings.LOGGER_NAME) + + +def get_account_details(user): + accounts_details = None + + try: + whitelists = get_user_acls(user) + + if whitelists: + uads = AuthorizedDataset.objects.filter(whitelist_id__in=whitelists) + accounts_details = {'dataset_access': [{'name': uad.name, 'whitelist_id': uad.whitelist_id} for uad in uads]} + + except UserValidationException as u: + logger.warn(u) + accounts_details = {'message': str(u)} + + except Exception as e: + logger.error("[ERROR] Encountered an error while retrieving user account details:") + logger.exception(e) + accounts_details = {'message': "Encountered an error while retrieving account details for {}.".format(user.email)} + + return accounts_details + + +def gcp_info(user, gcp_id=None): + gcps = None + success = False + + try: + gcps = get_user_gcps(user, gcp_id) + success = bool(gcps is not None) and (len(gcps) > 0) + + except Exception as e: + logger.error("[ERROR] Encountered an error while retrieving GCP project details:") + logger.exception(e) + gcps = {'message': "Encountered an error while retrieving GCP project details for {}.".format(user.email if not gcp_id else gcp_id)} + + return gcps, success + + +def gcp_validation(user, gcp_id, refresh=False): + validation = None + + try: + validation, status = verify_gcp_for_reg(user, gcp_id, refresh) + + logger.info("Validation result: {}".format(str(validation))) + + if validation: + if 'roles' in validation: + unregs = [x for x in validation['roles'] if not validation['roles'][x]['registered_user']] + + if len(unregs): + validation['notes'] = "The following users are not registered in our system. Please note that if GCP {} ".format(gcp_id) + \ + "is intended for use with controlled access data, all users must log in to the ISB-CGC " + \ + "web application at and link their Google Account to their eRA " + \ + "Commons ID. The link to do so is found in Account Settings. Unregistered users: " + \ + "{}".format("; ".join(unregs)) + + if 'message' not in validation: + validation['message'] = "Google Cloud Platform project ID {} was successfully validated for registration.".format(gcp_id) + + else: + logger.warn("[WARNING] Validation of {} by user {} was unsuccessful!".format(gcp_id, user.email)) + + except Exception as e: + logger.error("[ERROR] While attempting to validate a project for registration:") + logger.exception(e) + + return validation + + +def gcp_registration(user, gcp_id, refresh): + + registration = None + success = False + try: + validation = gcp_validation(user, gcp_id, refresh) + + if validation: + if 'roles' in validation: + + registered_users = [x for x, y in validation['roles'].items() if y['registered_user']] + registration, status = register_or_refresh_gcp(user, gcp_id, registered_users, refresh) + logger.info("Registration: {}".format(str(registration))) + + if status == 200: + success = True + if 'notes' in validation: + registration['notes'] = validation['notes'] + if 'message' not in registration: + registration['message'] = "Google Cloud Platform project ID {} was successfully {}.".format(gcp_id, 'refreshed' if refresh else 'registered') + else: + registration = validation + logger.warn("[WARNING] Validation of {} by user {} was unsuccessful! This project was not {}".format(gcp_id, user.email, 'refreshed' if refresh else 'registered')) + logger.warn("[WARNING] Reason given: {}".format(validation['message'])) + else: + logger.warn("[WARNING] Validation of {} by user {} was unsuccessful!".format(gcp_id, user.email)) + + except Exception as e: + logger.error("[ERROR] While registering a GCP:") + logger.exception(e) + + return registration, success + + +def gcp_unregistration(user, gcp_id): + unreg = None + success = False + try: + + unreg, status = api_gcp_delete(user, gcp_id) + + if status == 200: + success = True + if 'message' not in unreg: + unreg['message'] = "Google Cloud Platform project ID {} was successfully unregistered.".format(gcp_id) + else: + logger.warn("[WARNING] Unregistration of {} by user {} was unsuccessful!".format(gcp_id, user.email)) + + except Exception as e: + logger.error("[ERROR] While unregistering a GCP:") + logger.exception(e) + + return unreg, success diff --git a/appengine_config.py b/appengine_config.py deleted file mode 100644 index 565aaa4f..00000000 --- a/appengine_config.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import sys -from google.appengine.ext import vendor - -# Per https://github.com/GoogleCloudPlatform/google-cloud-python/issues/1705#issuecomment-209721632 we have to unload -# some GAE-installed libs to make sure our newer versions are used -def unload_module(module_name): - target_modules = [m for m in sys.modules if m.startswith(module_name)] - for m in target_modules: - if m in sys.modules: - del sys.modules[m] - -# Add any libraries installed in the "lib" folder. -vendor.add('lib') -if bool(os.environ.get('LOCAL_DEV', 'False') == 'True'): - print "Local development detected, vendoring in endpoints_lib..." - vendor.add('endpoints_lib') - -# The default endpoints/GAE oauth2 is way too old. -unload_module('oauth2client') - -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep - -SHARED_SOURCE_DIRECTORIES = [ - os.path.abspath('./ISB-CGC-Common') -] - -# Add the shared Django application subdirectory to the Python module search path -for path in SHARED_SOURCE_DIRECTORIES: - sys.path.append(path) - -# Initialize Django (when running ISB-CGC-API as standalone using dev_appserver.py) -import django -django.setup() diff --git a/bq_data_access/__init__.py b/bq_data_access/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/bq_data_access/clinical_data.py b/bq_data_access/clinical_data.py deleted file mode 100755 index 6a6afba6..00000000 --- a/bq_data_access/clinical_data.py +++ /dev/null @@ -1,190 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import sys -from re import compile as re_compile - -from api.schema.tcga_clinical import schema as clinical_schema - -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import DataTypes, BigQuerySchemaToValueTypeConverter -from bq_data_access.utils import DurationLogged - -CLINICAL_FEATURE_TYPE = 'CLIN' - -BSP_TABLE_NAME = 'Biospecimen_data' - -class InvalidClinicalFeatureIDException(Exception): - def __init__(self, feature_id, reason): - self.feature_id = feature_id - self.reason = reason - - def __str__(self): - return "Invalid internal clinical feature ID '{feature_id}', reason '{reason}'".format( - feature_id=self.feature_id, - reason=self.reason - ) - - -class ClinicalFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: CLIN:vital_status - regex = re_compile("^CLIN:" - # column name - "([a-zA-Z0-9_\-]+)$") - - def __init__(self, table_field, value_type): - self.table_field = table_field - self.value_type = value_type - - @classmethod - def get_table_field_and_value_type(cls, column_id): - table_field = None - value_type = None - for clinical_field in clinical_schema: - name, schema_field_type = clinical_field['name'], clinical_field['type'] - if name == column_id: - table_field= name - value_type = BigQuerySchemaToValueTypeConverter.get_value_type(schema_field_type) - - return table_field, value_type - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - column_name = feature_fields[0] - - table_field, value_type = cls.get_table_field_and_value_type(column_name) - if table_field is None: - raise FeatureNotFoundException(feature_id) - - return cls(table_field, value_type) - - -class ClinicalFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Clinical_data', - 'info': 'Clinical', - 'id': 'tcga_clinical' - } - ] - - def __init__(self, feature_id, **kwargs): - self.table_name = '' - self.feature_def = None - self.parse_internal_feature_id(feature_id) - super(ClinicalFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return self.feature_def.value_type - - def get_feature_type(self): - return DataTypes.CLIN - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT clin.ParticipantBarcode, biospec.sample_id, clin.{column_name} " - "FROM ( " - " SELECT ParticipantBarcode, {column_name} " - " FROM [{project_name}:{dataset_name}.{table_name}] " - " ) AS clin " - " JOIN ( " - " SELECT ParticipantBarcode, SampleBarcode as sample_id " - " FROM [{project_name}:{dataset_name}.{bsp_table_name}] " - " ) AS biospec " - " ON clin.ParticipantBarcode = biospec.ParticipantBarcode " - "WHERE biospec.sample_id IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - query_template += " GROUP BY clin.ParticipantBarcode, biospec.sample_id, clin.{column_name}" - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - column_name=feature_def.table_field, bsp_table_name=BSP_TABLE_NAME, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_CLIN: " + query) - return query - - @DurationLogged('CLIN', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Always None - - 'value': Value of the selected column from the clinical data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'case_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': None, - 'value': row['f'][2]['v'] - }) - - return result - - def get_table_name(self): - return self.TABLES[0]['name'] - - def parse_internal_feature_id(self, feature_id): - self.feature_def = ClinicalFeatureDef.from_feature_id(feature_id) - self.table_name = self.get_table_name() - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - ClinicalFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # ClinicalFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/cohort_bigquery.py b/bq_data_access/cohort_bigquery.py deleted file mode 100755 index f5ba29fe..00000000 --- a/bq_data_access/cohort_bigquery.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from copy import deepcopy -import sys - -from api.api_helpers import authorize_credentials_with_Google - -COHORT_DATASETS = { - 'prod': 'cloud_deployment_cohorts', - 'staging': 'cloud_deployment_cohorts', - 'dev': 'dev_deployment_cohorts' -} - -COHORT_TABLES = { - 'prod': 'prod_cohorts', - 'staging': 'staging_cohorts' -} - -class BigQueryCohortSupport(object): - cohort_schema = [ - { - "name": "cohort_id", - "type": "INTEGER", - "mode": "REQUIRED" - }, - { - "name": "case_barcode", - "type": "STRING" - }, - { - "name": "sample_barcode", - "type": "STRING" - }, - { - "name": "aliquot_barcode", - "type": "STRING" - }, - { - "name": "project_id", - "type": "INTEGER" - } - ] - - patient_type = 'patient' - sample_type = 'sample' - aliquot_type = 'aliquot' - - @classmethod - def get_schema(cls): - return deepcopy(cls.cohort_schema) - - def __init__(self, project_id, dataset_id, table_id): - self.project_id = project_id - self.dataset_id = dataset_id - self.table_id = table_id - - def _build_request_body_from_rows(self, rows): - insertable_rows = [] - for row in rows: - insertable_rows.append({ - 'json': row - }) - - return { - "rows": insertable_rows - } - - def _streaming_insert(self, rows): - bigquery_service = authorize_credentials_with_Google() - table_data = bigquery_service.tabledata() - - body = self._build_request_body_from_rows(rows) - - print >> sys.stdout, self.project_id+":"+self.dataset_id+":"+self.table_id - - response = table_data.insertAll(projectId=self.project_id, - datasetId=self.dataset_id, - tableId=self.table_id, - body=body).execute() - - return response - - def _build_cohort_row(self, cohort_id, - case_barcode=None, sample_barcode=None, aliquot_barcode=None, project_id=None): - return { - 'cohort_id': cohort_id, - 'case_barcode': case_barcode, - 'sample_barcode': sample_barcode, - 'aliquot_barcode': aliquot_barcode, - 'project_id': project_id - } - - # Create a cohort based on a dictionary of sample, patient/case/participant, and project IDs - def add_cohort_to_bq(self, cohort_id, samples): - rows = [] - for sample in samples: - rows.append(self._build_cohort_row(cohort_id, case_barcode=sample['case_barcode'], sample_barcode=sample['sample_barcode'], project_id=sample['project'].id)) - - response = self._streaming_insert(rows) - - print >> sys.stdout, response.__str__() - - return response - - # Create a cohort based only on sample and optionally project IDs (patient/participant/case ID is NOT added) - def add_cohort_with_sample_barcodes(self, cohort_id, samples): - rows = [] - for sample in samples: - # TODO This is REALLY specific to TCGA. This needs to be changed - # patient_barcode = sample_barcode[:12] - barcode = sample - project_id = None - if isinstance(sample, tuple): - barcode = sample[0] - if len(sample) > 1: - project_id = sample[1] - elif isinstance(sample, dict): - barcode = sample['sample_id'] - if 'project_id' in sample: - project_id = sample['project_id'] - - rows.append(self._build_cohort_row(cohort_id, sample_barcode=barcode, project_id=project_id)) - - response = self._streaming_insert(rows) - return response diff --git a/bq_data_access/cohort_cloudsql.py b/bq_data_access/cohort_cloudsql.py deleted file mode 100755 index ef097704..00000000 --- a/bq_data_access/cohort_cloudsql.py +++ /dev/null @@ -1,152 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import sys -import traceback - -from api.api_helpers import sql_connection - -from MySQLdb.cursors import DictCursor - -DJANGO_COHORT_TABLE = 'cohorts_samples' -DJANGO_COHORT_INFO_TABLE = 'cohorts_cohort' -DJANGO_COHORT_SAMPLES_TABLE = 'cohorts_samples' - -logger = logging - - -class CohortException(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return 'Cohort error: ' + self.message - -class CloudSQLCohortAccess(object): - @classmethod - def parse_barcodes(cls, barcode_string): - codes = barcode_string.replace('[', '').replace(']', '').replace('\'', '').replace(' ', '').split(',') - return codes - - @classmethod - def get_cohort_barcodes(cls, cohort_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join(['%s' for x in xrange(len(cohort_id_array))]) - query = 'SELECT sample_barcode AS barcode FROM {cohort_table} WHERE cohort_id IN ({cohort_id_stmt})'.format( - cohort_table=DJANGO_COHORT_TABLE, - cohort_id_stmt=cohort_id_stmt) - values = cohort_id_array - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - result = cursor.fetchall() - barcodes = [] - for row in result: - barcodes.append(row['barcode']) - cursor.close() - db.close() - - # Return only unique barcodes - return list(set(barcodes)) - - except Exception as e: - raise CohortException('get_cohort_barcodes CloudSQL error, cohort IDs {cohort_ids}: {message}'.format( - cohort_ids=cohort_id_array, - message=str(e.message))) - raise CohortException('bad cohort: ' + str(cohort_id_array)) - - @classmethod - def get_cohorts_for_datapoints(cls, cohort_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join(['%s' for x in xrange(len(cohort_id_array))]) - - query = 'SELECT sample_barcode, cohort_id FROM {cohort_samples_table} WHERE cohort_id IN ({cohort_id_stmt})'.format( - cohort_samples_table=DJANGO_COHORT_SAMPLES_TABLE, - cohort_id_stmt=cohort_id_stmt) - - values = cohort_id_array - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(values)) - - result = cursor.fetchall() - cohort_per_samples = {} - - for row in result: - cohort_id, sample_barcode = row['cohort_id'], row['sample_barcode'] - if sample_barcode not in cohort_per_samples: - cohort_per_samples[sample_barcode] = [] - cohort_per_samples[sample_barcode].append(cohort_id) - - cursor.close() - db.close() - - return cohort_per_samples - - except Exception as e: - logger.exception(e) - raise CohortException('get_cohorts_for_datapoints CloudSQL error, cohort IDs {cohort_ids}: {message}'.format( - cohort_ids=cohort_id_array, - message=str(e.message))) - - @classmethod - def get_cohort_info(cls, cohort_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join(['%s' for x in xrange(len(cohort_id_array))]) - - query_template = ("SELECT ti.id AS cohort_id, ti.name, COUNT(ts.sample_barcode) AS size " - "FROM {cohort_info_table} ti " - " LEFT JOIN {cohort_samples_table} ts ON ts.cohort_id = ti.id " - "WHERE ti.id IN ({cohort_id_stmt}) " - "GROUP BY ti.id, ti.name") - - query = query_template.format( - cohort_info_table=DJANGO_COHORT_INFO_TABLE, - cohort_samples_table=DJANGO_COHORT_SAMPLES_TABLE, - cohort_id_stmt=cohort_id_stmt) - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(cohort_id_array)) - - result = [] - - for row in cursor.fetchall(): - result.append({ - 'id': row['cohort_id'], - 'name': row['name'], - 'size': row['size'] - }) - - cursor.close() - db.close() - - return result - - except Exception as e: - print >> sys.stdout, "[ERROR] In get_cohort_info: " - print >> sys.stdout, e - print >> sys.stdout, traceback.format_exc() - raise CohortException('get_cohort_info CloudSQL error, cohort IDs {cohort_ids}: {message}'.format( - cohort_ids=cohort_id_array, - message=str(e.message))) diff --git a/bq_data_access/cohort_utils.py b/bq_data_access/cohort_utils.py deleted file mode 100755 index 051d2640..00000000 --- a/bq_data_access/cohort_utils.py +++ /dev/null @@ -1,61 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from bq_data_access.feature_value_types import DataTypes, IdentifierTypes - -class BarcodeAdapter(object): - identifier_length = { - IdentifierTypes.PATIENT: 12, - IdentifierTypes.SAMPLE: 16, - IdentifierTypes.ALIQUOT: 28 - } - - supported_identifers = { - DataTypes.CLIN: IdentifierTypes.PATIENT, - DataTypes.GEXP: IdentifierTypes.PATIENT, - DataTypes.METH: IdentifierTypes.PATIENT, - DataTypes.CNVR: IdentifierTypes.PATIENT, - DataTypes.RPPA: IdentifierTypes.PATIENT, - DataTypes.MIRN: IdentifierTypes.PATIENT, - DataTypes.GNAB: IdentifierTypes.PATIENT - } - - @classmethod - def get_identifier_type_for_data_type(cls, data_type): - return cls.supported_identifers[data_type] - - @classmethod - def convert(cls, data_type, barcode_list): - identifier_type = cls.get_identifier_type_for_data_type(data_type) - truncate_length = cls.identifier_length[identifier_type] - result = [] - for barcode in barcode_list: - truncated_barcode = barcode[:truncate_length] - result.append(truncated_barcode) - - return result - -class CohortQueryBuilder(object): - @classmethod - def build(cls, field_name, barcodes_list, delimiter=',', barcode_quote='\''): - barcodes_quoted = [] - for barcode in barcodes_list: - barcodes_quoted.append('{quote}{barcode}{quote}'.format(quote=barcode_quote, barcode=barcode)) - result = field_name + ' IN ({identifiers})' - result = result.format(identifiers=delimiter.join(barcodes_quoted)) - return result diff --git a/bq_data_access/copynumber_data.py b/bq_data_access/copynumber_data.py deleted file mode 100755 index ac885e8a..00000000 --- a/bq_data_access/copynumber_data.py +++ /dev/null @@ -1,164 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -CNVR_FEATURE_TYPE = 'CNVR' -IDENTIFIER_COLUMN_NAME = 'sample_id' - - -def get_feature_type(): - return CNVR_FEATURE_TYPE - - -class CNVRFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: CNVR:max_segment_mean:X:133276258:133276370 - regex = re_compile("^CNVR:" - # value - "(avg_segment_mean|std_dev_segment_mean|min_segment_mean|max_segment_mean|num_segments):" - # validate outside - chromosome 1-23, X, Y, M - "(\d|\d\d|X|Y|M):" - # coordinates start:end - "(\d+):(\d+)$") - - def __init__(self, value_field, chromosome, start, end): - self.value_field = value_field - self.chromosome = chromosome - self.start = start - self.end = end - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - logging.debug(feature_fields) - value_field, chromosome, start, end = feature_fields[0] - - valid_chr_set = frozenset([str(x) for x in xrange(1, 24)] + ['X', 'Y', 'M']) - if chromosome not in valid_chr_set: - raise FeatureNotFoundException(feature_id) - - return cls(value_field, chromosome, start, end) - - -class CNVRFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Copy_Number_segments', - 'info': 'CNV', - 'id': 'cnv' - } - ] - - def __init__(self, feature_id, **kwargs): - self.table_name = '' - self.feature_def = None - self.parse_internal_feature_id(feature_id) - super(CNVRFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.CNVR - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, study_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - study_id_stmt = '' - if study_id_array is not None: - study_id_stmt = ', '.join([str(study_id) for study_id in study_id_array]) - - value_field_bqsql = { - 'avg_segment_mean': 'AVG(Segment_Mean)', - 'std_dev_segment_mean': 'STDDEV(Segment_Mean)', - 'min_segment_mean': 'MIN(Segment_Mean)', - 'max_segment_mean': 'MAX(Segment_Mean)', - 'num_segments': 'COUNT(*)' - } - - query_template = "SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, {value_field} AS value " \ - "FROM [{project_name}:{dataset_name}.{table_name}] " \ - "WHERE ( Chromosome='{chr}' AND ( " \ - " ( Start<{start} AND End>{start} ) OR " \ - " ( Start>{start}-1 AND Start<{end}+1 ) ) ) " \ - "AND SampleBarcode IN ( " \ - " SELECT sample_barcode " \ - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " \ - " WHERE cohort_id IN ({cohort_id_list})" \ - " AND (study_id IS NULL" - - query_template += (" OR study_id IN ({study_id_list})))" if study_id_array is not None else "))") - query_template += " GROUP BY ParticipantBarcode, SampleBarcode, AliquotBarcode" - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - value_field=value_field_bqsql[feature_def.value_field], - chr=feature_def.chromosome, - start=feature_def.start, end=feature_def.end, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, study_id_list=study_id_stmt) - - logging.debug("BQ_QUERY_CNVR: " + query) - return query - - @DurationLogged('CNVR', 'UNPACK') - def unpack_query_response(self, query_result_array): - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': row['f'][3]['v'] - }) - - return result - - def get_table_name(self): - return self.TABLES[0]['name'] - - def parse_internal_feature_id(self, feature_id): - self.feature_def = CNVRFeatureDef.from_feature_id(feature_id) - self.table_name = self.get_table_name() - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - CNVRFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # CNVRFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/data_access.py b/bq_data_access/data_access.py deleted file mode 100755 index b0bfa265..00000000 --- a/bq_data_access/data_access.py +++ /dev/null @@ -1,436 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile -from time import sleep - -from django.conf import settings - -from api.api_helpers import authorize_credentials_with_Google -from api.api_helpers import sql_connection, MySQLdb - -from bq_data_access.feature_value_types import ValueType - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.gexp_data import GEXPFeatureProvider, GEXP_FEATURE_TYPE -from bq_data_access.methylation_data import METHFeatureProvider, METH_FEATURE_TYPE -from bq_data_access.copynumber_data import CNVRFeatureProvider, CNVR_FEATURE_TYPE -from bq_data_access.protein_data import RPPAFeatureProvider, RPPA_FEATURE_TYPE -from bq_data_access.mirna_data import MIRNFeatureProvider, MIRN_FEATURE_TYPE -from bq_data_access.clinical_data import ClinicalFeatureProvider, CLINICAL_FEATURE_TYPE -from bq_data_access.gnab_data import GNABFeatureProvider, GNAB_FEATURE_TYPE -from bq_data_access.user_data import UserFeatureProvider, USER_FEATURE_TYPE - - -class FeatureProviderFactory(object): - @classmethod - def get_feature_type_string(cls, feature_id): - regex = re_compile("^(CLIN|GEXP|METH|CNVR|RPPA|MIRN|GNAB|USER):") - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - return None - - feature_type = feature_fields[0] - return feature_type - - @classmethod - def get_provider_class_from_feature_id(cls, feature_id): - """ - Args: - feature_id: Feature identifier - - Returns: - Feature data provider class for the datatype defined in the - feature identifier. - - Raises: - FeatureNotFoundException: If the datatype part of the feature - identifier is unknown. - - """ - feature_type = cls.get_feature_type_string(feature_id) - if feature_type is None: - logging.debug("FeatureProviderFactory.from_feature_id: invalid feature ID: " + str(feature_id)) - raise FeatureNotFoundException(feature_id) - - if feature_type == CLINICAL_FEATURE_TYPE: - return ClinicalFeatureProvider - elif feature_type == GEXP_FEATURE_TYPE: - return GEXPFeatureProvider - elif feature_type == METH_FEATURE_TYPE: - return METHFeatureProvider - elif feature_type == CNVR_FEATURE_TYPE: - return CNVRFeatureProvider - elif feature_type == RPPA_FEATURE_TYPE: - return RPPAFeatureProvider - elif feature_type == MIRN_FEATURE_TYPE: - return MIRNFeatureProvider - elif feature_type == GNAB_FEATURE_TYPE: - return GNABFeatureProvider - elif feature_type == USER_FEATURE_TYPE: - return UserFeatureProvider - - @classmethod - def from_feature_id(cls, feature_id, **kwargs): - provider_class = cls.get_provider_class_from_feature_id(feature_id) - return provider_class(feature_id, **kwargs) - - @classmethod - def from_parameters(cls, parameters_obj, **kwargs): - if isinstance(parameters_obj, FeatureIdQueryDescription): - return cls.from_feature_id(parameters_obj.feature_id, **kwargs) - elif isinstance(parameters_obj, ProviderClassQueryDescription): - class_type = parameters_obj.feature_data_provider_class - feature_id = parameters_obj.feature_id - return class_type(feature_id, **kwargs) - - -class FeatureIdQueryDescription(object): - def __init__(self, feature_id, cohort_id_array, project_id_array): - self.feature_id = feature_id - self.cohort_id_array = cohort_id_array - self.project_id_array = project_id_array - - -class ProviderClassQueryDescription(object): - def __init__(self, feature_data_provider_class, feature_id, cohort_id_array, project_id_array): - self.feature_data_provider_class = feature_data_provider_class - self.feature_id = feature_id - self.cohort_id_array = cohort_id_array - self.project_id_array = project_id_array - - -def is_valid_feature_identifier(feature_id): - """ - Answers if given internal feature identifier is valid. - - Args: - feature_id: Internal feature identifier - - Returns: - True if feature id is valid, otherwise False. - """ - is_valid = False - try: - provider_class = FeatureProviderFactory.get_provider_class_from_feature_id(feature_id) - is_valid = provider_class.is_valid_feature_id(feature_id) - except FeatureNotFoundException: - # FeatureProviderFactory.get_provider_class_from_feature_id raises FeatureNotFoundException - # if the feature identifier does not look valid. Nothing needs to be done here, - # since is_valid is already False. - pass - finally: - return is_valid - - -def get_feature_vector(feature_id, cohort_id_array): - """ - Fetches the data from BigQuery tables for a given feature identifier and - one or more stored cohorts. Returns the intersection of the samples defined - by the feature identifier and the stored cohort. - - Each returned data point is represented as a dict containing patient, sample and - aliquot barcodes, and the value as defined by the feature identifier. - - Args: - feature_id: Feature identifier - cohort_id_array: Array of cohort identifiers (integers) - - Returns: - Data as an array of dicts. - """ - provider = FeatureProviderFactory.from_feature_id(feature_id) - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - - result = provider.get_data(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - - items = [] - for data_point in result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = provider.process_data_point(data_point) - # TODO refactor missing value logic - if value is None: - value = 'NA' - data_item['value'] = value - items.append(data_item) - - return provider.get_value_type(), items - - -def submit_tcga_job(param_obj, bigquery_service, cohort_settings): - provider = FeatureProviderFactory.from_parameters(param_obj, bigquery_service=bigquery_service) - feature_id = param_obj.feature_id - cohort_id_array = param_obj.cohort_id_array - project_id_array = param_obj.project_id_array - job_reference = provider.get_data_job_reference(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id, project_id_array) - - logging.info("Submitted TCGA {job_id}: {fid} - {cohorts}".format(job_id=job_reference['jobId'], fid=feature_id, - cohorts=str(cohort_id_array))) - job_item = { - 'feature_id': feature_id, - 'provider': provider, - 'ready': False, - 'job_reference': job_reference - } - - return job_item - - -def submit_jobs_with_user_data(params_array): - bigquery_service = authorize_credentials_with_Google() - provider_array = [] - - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - - # Submit jobs - for parameter_object in params_array: - feature_id = parameter_object.feature_id - cohort_id_array = parameter_object.cohort_id_array - - user_data = user_feature_handler(feature_id, cohort_id_array) - - if user_data['include_tcga']: - job_item = submit_tcga_job(parameter_object, bigquery_service, cohort_settings) - provider_array.append(job_item) - - if len(user_data['user_studies']) > 0: - converted_feature_id = user_data['converted_feature_id'] - user_feature_id = user_data['user_feature_id'] - logging.debug("user_feature_id: {0}".format(user_feature_id)) - provider = UserFeatureProvider(converted_feature_id, user_feature_id=user_feature_id) - - # The UserFeatureProvider instance might not generate a BigQuery query and job at all given the combination - # of cohort(s) and feature identifiers. The provider is not added to the array, and therefore to the - # polling loop below, if it would not submit a BigQuery job. - if provider.is_queryable(cohort_id_array): - job_reference = provider.get_data_job_reference(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - - logging.info("Submitted USER {job_id}: {fid} - {cohorts}".format(job_id=job_reference['jobId'], fid=feature_id, - cohorts=str(cohort_id_array))) - provider_array.append({ - 'feature_id': feature_id, - 'provider': provider, - 'ready': False, - 'job_reference': job_reference - }) - else: - logging.debug("No UserFeatureDefs for '{0}'".format(converted_feature_id)) - - return provider_array - - -def get_submitted_job_results(provider_array, project_id, poll_retry_limit, skip_formatting_for_plot): - result = {} - all_done = False - total_retries = 0 - poll_count = 0 - - # Poll for completion - while all_done is False and total_retries < poll_retry_limit: - poll_count += 1 - total_retries += 1 - - for item in provider_array: - provider = item['provider'] - feature_id = item['feature_id'] - is_finished = provider.is_bigquery_job_finished(project_id) - logging.info("Status {job_id}: {status}".format(job_id=item['job_reference']['jobId'], - status=str(is_finished))) - - if item['ready'] is False and is_finished: - item['ready'] = True - query_result = provider.download_and_unpack_query_result() - - if not skip_formatting_for_plot: - data = format_query_result_for_plot(provider, query_result) - - value_type = provider.get_value_type() - if feature_id not in result: - result[feature_id] = { - 'type': value_type, - 'data': data - } - else: - # TODO fix possible bug: - # The ValueType of the data from the user feature provider may not match that of the TCGA - # provider above. - result[feature_id]['data'].extend(data) - else: - result[feature_id] = { - 'data': query_result, - 'type': None - } - - sleep(1) - - all_done = all([j['ready'] for j in provider_array]) - logging.debug("Done: {done} retry: {retry}".format(done=str(all_done), retry=total_retries)) - - return result - - -def format_query_result_for_plot(provider_instance, query_result): - data = [] - - for data_point in query_result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = str(provider_instance.process_data_point(data_point)) - - if value is None: - value = 'NA' - data_item['value'] = value - data.append(data_item) - - return data - - -def get_feature_vectors_with_user_data(params_array, poll_retry_limit=20, skip_formatting_for_plot=False): - provider_array = submit_jobs_with_user_data(params_array) - - project_id = settings.BQ_PROJECT_ID - result = get_submitted_job_results(provider_array, project_id, poll_retry_limit, skip_formatting_for_plot) - - return result - - -def get_feature_vectors_tcga_only(params_array, poll_retry_limit=20, skip_formatting_for_plot=False): - bigquery_service = authorize_credentials_with_Google() - provider_array = [] - - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - - # Submit jobs - for parameter_object in params_array: - job_item = submit_tcga_job(parameter_object, bigquery_service, cohort_settings) - provider_array.append(job_item) - - project_id = settings.BQ_PROJECT_ID - result = get_submitted_job_results(provider_array, project_id, poll_retry_limit, skip_formatting_for_plot) - - return result - - -def user_feature_handler(feature_id, cohort_id_array): - include_tcga = False - user_studies = () - for cohort_id in cohort_id_array: - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - cursor.execute("SELECT project_id FROM cohorts_samples WHERE cohort_id = %s GROUP BY project_id", (cohort_id,)) - for row in cursor.fetchall(): - if row['project_id'] is None: - include_tcga = True - else: - user_studies += (row['project_id'],) - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - user_feature_id = None - if feature_id.startswith('USER:'): - # Try and convert it with a shared ID to a TCGA queryable id - user_feature_id = feature_id - feature_id = UserFeatureProvider.convert_user_feature_id(feature_id) - if feature_id is None: - # Querying user specific data, don't include TCGA - include_tcga = False - - return { - 'converted_feature_id': feature_id, - 'include_tcga': include_tcga, - 'user_studies': user_studies, - 'user_feature_id': user_feature_id - } - - -def get_feature_vector(feature_id, cohort_id_array): - include_tcga = False - user_studies = () - for cohort_id in cohort_id_array: - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - cursor.execute("SELECT project_id FROM cohorts_samples WHERE cohort_id = %s GROUP BY project_id", (cohort_id,)) - for row in cursor.fetchall(): - if row['project_id'] is None: - include_tcga = True - else: - user_studies += (row['project_id'],) - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - # ex: feature_id 'CLIN:Disease_Code' - user_feature_id = None - if feature_id.startswith('USER:'): - # Try and convert it with a shared ID to a TCGA queryable id - user_feature_id = feature_id - feature_id = UserFeatureProvider.convert_user_feature_id(feature_id) - if feature_id is None: - # Querying user specific data, don't include TCGA - include_tcga = False - - items = [] - type = None - result = [] - cohort_settings = settings.GET_BQ_COHORT_SETTINGS() - if include_tcga: - provider = FeatureProviderFactory.from_feature_id(feature_id) - result = provider.get_data(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - - # ex: result[0] - # {'aliquot_id': None, 'case_id': u'TCGA-BH-A0B1', 'sample_id': u'TCGA-BH-A0B1-10A', 'value': u'BRCA'} - for data_point in result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = provider.process_data_point(data_point) - # TODO refactor missing value logic - if value is None: - value = 'NA' - data_item['value'] = value - items.append(data_item) - - type = provider.get_value_type() - - if len(user_studies) > 0: - # Query User Data - user_provider = UserFeatureProvider(feature_id, user_feature_id=user_feature_id) - user_result = user_provider.get_data(cohort_id_array, cohort_settings.dataset_id, cohort_settings.table_id) - result.extend(user_result) - - for data_point in user_result: - data_item = {key: data_point[key] for key in ['case_id', 'sample_id', 'aliquot_id']} - value = provider.process_data_point(data_point) - # TODO refactor missing value logic - if value is None: - value = 'NA' - data_item['value'] = value - items.append(data_item) - - if not type: - type = user_provider.get_value_type() - - return type, items diff --git a/bq_data_access/errors.py b/bq_data_access/errors.py deleted file mode 100644 index 6369cac4..00000000 --- a/bq_data_access/errors.py +++ /dev/null @@ -1,28 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -class FeatureAccessError(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return self.message - -class FeatureNotFoundException(FeatureAccessError): - """Exception raised when the queried internal feature identifier is invalid""" - diff --git a/bq_data_access/feature_data_provider.py b/bq_data_access/feature_data_provider.py deleted file mode 100644 index 26dac2d5..00000000 --- a/bq_data_access/feature_data_provider.py +++ /dev/null @@ -1,143 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -import sys -from uuid import uuid4 -from time import sleep - -from django.conf import settings -from api.api_helpers import authorize_credentials_with_Google -from bq_data_access.utils import DurationLogged - - -class FeatureDataProvider(object): - """ - Class for building data access modules for different datatypes. - - TODO: Document interface - """ - def __init__(self, bigquery_service=None): - self.job_reference = None - self.bigquery_service = bigquery_service - - def get_bq_service(self): - if self.bigquery_service is None: - self.bigquery_service = authorize_credentials_with_Google() - - return self.bigquery_service - - @DurationLogged('FEATURE', 'BQ_SUBMIT') - def submit_bigquery_job(self, bigquery, project_id, query_body, batch=False): - job_data = { - 'jobReference': { - 'projectId': project_id, - 'job_id': str(uuid4()) - }, - 'configuration': { - 'query': { - 'query': query_body, - 'priority': 'BATCH' if batch else 'INTERACTIVE' - } - } - } - - return bigquery.jobs().insert( - projectId=project_id, - body=job_data).execute(num_retries=5) - - @DurationLogged('FEATURE', 'BQ_POLL') - def poll_async_job(self, bigquery_service, project_id, job_id, poll_interval=5): - job_collection = bigquery_service.jobs() - - poll = True - - while poll: - sleep(poll_interval) - job = job_collection.get(projectId=project_id, - jobId=job_id).execute() - - if job['status']['state'] == 'DONE': - poll = False - - if 'errorResult' in job['status']: - raise Exception(job['status']) - - @DurationLogged('FEATURE', 'BQ_FETCH') - def download_query_result(self, bigquery, job_reference): - result = [] - page_token = None - total_rows = 0 - - while True: - page = bigquery.jobs().getQueryResults( - pageToken=page_token, - **job_reference).execute(num_retries=2) - - if int(page['totalRows']) == 0: - break - - rows = page['rows'] - result.extend(rows) - total_rows += len(rows) - - page_token = page.get('pageToken') - if not page_token: - break - - return result - - def is_bigquery_job_finished(self, project_id): - job_collection = self.get_bq_service().jobs() - bigquery_job_id = self.job_reference['jobId'] - - job = job_collection.get(projectId=project_id, - jobId=bigquery_job_id).execute() - - return job['status']['state'] == 'DONE' - - def download_and_unpack_query_result(self): - bigquery_service = self.get_bq_service() - query_result_array = self.download_query_result(bigquery_service, self.job_reference) - - result = self.unpack_query_response(query_result_array) - return result - - def submit_query_and_get_job_ref(self, project_id, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - bigquery_service = self.get_bq_service() - - query_body = self.build_query(project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array) - query_job = self.submit_bigquery_job(bigquery_service, project_id, query_body) - - # Poll for completion of the query - self.job_reference = query_job['jobReference'] - job_id = query_job['jobReference']['jobId'] - logging.debug("JOBID {id}".format(id=job_id)) - - return self.job_reference - - def get_data_job_reference(self, cohort_id_array, cohort_dataset, cohort_table, project_id_array): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - - result = self.submit_query_and_get_job_ref(project_id, project_name, dataset_name, self.table_name, - self.feature_def, cohort_dataset, cohort_table, cohort_id_array, - project_id_array) - return result - diff --git a/bq_data_access/feature_search/__init__.py b/bq_data_access/feature_search/__init__.py deleted file mode 100755 index e69de29b..00000000 diff --git a/bq_data_access/feature_search/clinical_searcher.py b/bq_data_access/feature_search/clinical_searcher.py deleted file mode 100755 index c8dfd7b0..00000000 --- a/bq_data_access/feature_search/clinical_searcher.py +++ /dev/null @@ -1,92 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from api.schema.tcga_clinical import schema as clinical_schema - -from bq_data_access.clinical_data import CLINICAL_FEATURE_TYPE -from bq_data_access.feature_search.common import InvalidFieldException, EmptyQueryException - -class ClinicalSearcher(object): - feature_search_valid_fields = ['keyword'] - - @classmethod - def get_datatype_identifier(cls): - return CLINICAL_FEATURE_TYPE - - @classmethod - def get_searchable_fields(cls): - return [ - {'name': 'keyword', - 'label': 'Keyword', - 'static': False}, - ] - - @classmethod - def build_feature_label(cls, column_name): - name_parts = [] - for part in column_name.split('_'): - name_parts.append(part.capitalize()) - human_readable_name = ' '.join(name_parts) - return human_readable_name - - def filter_by_name(self, keyword): - result = [] - - for item in clinical_schema: - name = item['name'] - if name.find(keyword.lower()) != -1: - result.append(item) - - return result - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(self.get_datatype_identifier(), keyword, field) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def search(self, parameters): - self.validate_feature_search_input(parameters) - search_result = self.filter_by_name(parameters['keyword']) - - found_features = [] - for feature_item in search_result: - column_name = feature_item['name'] - human_readable_name = self.build_feature_label(column_name) - internal_id = 'CLIN:' + column_name - type = "N" - if feature_item['type'] == "STRING" : - type = "C" - found_features.append({ - 'feature_type': 'CLIN', - 'internal_feature_id': internal_id, - 'label': human_readable_name, - 'type' : type - }) - - return found_features diff --git a/bq_data_access/feature_search/common.py b/bq_data_access/feature_search/common.py deleted file mode 100755 index d9b66f51..00000000 --- a/bq_data_access/feature_search/common.py +++ /dev/null @@ -1,41 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -class FeatureSearchError(Exception): - def __init__(self, message): - self.message = message - - def __str__(self): - return self.message - -class InvalidDataTypeException(FeatureSearchError): - """Exception raised when requested data type does not exist""" - -class InvalidQueryException(FeatureSearchError): - """Exception raised when a query for a datatype is invalid""" - -class InvalidFieldException(InvalidQueryException): - """Exception raised when the requested search field does not exist for a datatype""" - -class EmptyQueryException(InvalidQueryException): - """Exception raised when a query contain only empty keywords""" - -class BackendException(FeatureSearchError): - """Exception raised when feature search fails because of backend datasource""" - -FOUND_FEATURE_LIMIT = 20 diff --git a/bq_data_access/feature_search/copynumber_search.py b/bq_data_access/feature_search/copynumber_search.py deleted file mode 100644 index 7d6c4966..00000000 --- a/bq_data_access/feature_search/copynumber_search.py +++ /dev/null @@ -1,145 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.copynumber_data import CNVR_FEATURE_TYPE - - -class CNVRSearcher(object): - feature_search_valid_fields = set(['gene_name', 'value_field']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, - 'values': ['avg_segment_mean', 'std_dev_segment_mean', 'min_segment_mean', 'max_segment_mean', 'num_segments']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return CNVR_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_cnvr" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("CNVR", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Copy Number | Gene:EGFR, Value:avg_segment_mean' - label = "Copy Number | Gene:" + row['gene_name'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, value_field, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name=%s'\ - ' AND value_field LIKE %s' \ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = [input['gene_name'], - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = CNVR_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError as mse: - raise BackendException('database error') - diff --git a/bq_data_access/feature_search/gexp_searcher.py b/bq_data_access/feature_search/gexp_searcher.py deleted file mode 100755 index e39fe52a..00000000 --- a/bq_data_access/feature_search/gexp_searcher.py +++ /dev/null @@ -1,155 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -import logging - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.gexp_data import GEXP_FEATURE_TYPE - - -class GEXPSearcher(object): - feature_search_valid_fields = set(['gene_name', 'platform', 'center']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'platform', - 'label': 'Platform', - 'static': True, - 'values': ['Illumina GA', 'Illumina HiSeq']}, - {'name': 'center', - 'label': 'Center', - 'static': True, 'values': ['UNC']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return GEXP_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_gexp" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("GEXP: '%s', '%s'", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - logging.debug("CLOUDSQL_QUERY_GEXP_FIELDS: {}".format(query)) - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException("MySQLError: {}".format(str(mse))) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, gene, info): - # print info - # Example: 'EGFR mRNA (Illumina HiSeq, UNC RSEM)' - label = gene + " mRNA (" + info['platform'] + ", " + info['generating_center'] + " " + info['value_label'] + ")" - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, platform, generating_center, value_label, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name=%s'\ - ' AND platform LIKE %s' \ - ' AND generating_center LIKE %s'\ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - logging.debug("CLOUDSQL_QUERY_GEXP_SEARCH: {}".format(query)) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = [input['gene_name'], - '%' + input['platform'] + '%', - '%' + input['center'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = GEXP_FEATURE_TYPE - item['label'] = self.build_feature_label(item['gene_name'], item) - - return items - - except MySQLError as mse: - raise BackendException("MySQLError: {}".format(str(mse))) - diff --git a/bq_data_access/feature_search/gnab_searcher.py b/bq_data_access/feature_search/gnab_searcher.py deleted file mode 100644 index f9193e86..00000000 --- a/bq_data_access/feature_search/gnab_searcher.py +++ /dev/null @@ -1,145 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.gnab_data import GNAB_FEATURE_TYPE - - -class GNABSearcher(object): - feature_search_valid_fields = set(['gene_name', 'value_field']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, - 'values': ['variant_classification', 'variant_type', 'sequence_source', 'num_mutations']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return GNAB_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_gnab" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("GNAB", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Mutation | Gene:EGFR, Value:variant_classification' - label = "Mutation | Gene:" + row['gene_name'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, value_field, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name=%s'\ - ' AND value_field LIKE %s' \ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = [input['gene_name'], - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = GNAB_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError as mse: - raise BackendException('database error') - diff --git a/bq_data_access/feature_search/methylation_searcher.py b/bq_data_access/feature_search/methylation_searcher.py deleted file mode 100644 index 62fc8093..00000000 --- a/bq_data_access/feature_search/methylation_searcher.py +++ /dev/null @@ -1,167 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.methylation_data import METH_FEATURE_TYPE - -class METHSearcher(object): - feature_search_valid_fields = set(['gene_name', 'probe_name', 'platform', 'relation_to_gene', 'relation_to_island']) - field_search_valid_fields = set(['gene_name', 'probe_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'probe_name', - 'label': 'CpG Probe', - 'static': False}, - {'name': 'platform', - 'label': 'Platform', - 'static': True, - 'values': ['HumanMethylation27', 'HumanMethylation450']}, - {'name': 'relation_to_gene', - 'label': 'Gene region', - 'static': True, 'values': ['Body', '5\'UTR', '1stExon', 'TSS200', 'TSS1500', '3\'UTR']}, - {'name': 'relation_to_island', - 'label': 'CpG Island region', - 'static': True, 'values': ['Island', 'N_Shelf', 'N_Shore', 'S_Shore', 'S_Shelf']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return METH_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_meth" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("METH", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Methylation | Probe:cg07311521, Gene:EGFR, Gene Region:TSS1500, Relation to CpG Island:Island, Platform:HumanMethylation450, Value:beta_value' - # If value is not present, display '-' - if row['gene_name'] is '': - row['gene_name'] = "-" - row['relation_to_gene'] = "-" - if row['relation_to_island'] is '': - row['relation_to_island'] = "-" - - label = "Methylation | Probe:" + row['probe_name'] + ", Gene:" + row['gene_name'] + \ - ", Gene Region:" + row['relation_to_gene'] + ", CpG Island Region:" + row['relation_to_island'] + \ - ", Platform:" + row['platform'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, probe_name, platform, relation_to_gene, relation_to_island, ' \ - 'value_field, internal_feature_id ' \ - 'FROM {table_name} ' \ - 'WHERE gene_name=%s ' \ - 'AND probe_name LIKE %s ' \ - 'AND platform LIKE %s ' \ - 'AND relation_to_gene LIKE %s ' \ - 'AND relation_to_island LIKE %s ' \ - 'LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - query_args = [input['gene_name'], - '%' + input['probe_name'] + '%', - '%' + input['platform'] + '%', - '%' + input['relation_to_gene'] + '%', - '%' + input['relation_to_island'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = METH_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error') \ No newline at end of file diff --git a/bq_data_access/feature_search/microrna_searcher.py b/bq_data_access/feature_search/microrna_searcher.py deleted file mode 100644 index 2834de04..00000000 --- a/bq_data_access/feature_search/microrna_searcher.py +++ /dev/null @@ -1,149 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict -import logging -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.mirna_data import MIRN_FEATURE_TYPE - - -class MIRNSearcher(object): - feature_search_valid_fields = set(['mirna_name', 'platform', 'value_field']) - field_search_valid_fields = set(['mirna_name']) - - searchable_fields = [ - {'name': 'mirna_name', - 'label': 'miRNA Name', - 'static': False}, - {'name': 'platform', - 'label': 'Platform', - 'static': True, - 'values': ['IlluminaGA', 'IlluminaHiSeq']}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, 'values': ['RPM', 'normalized_count']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return MIRN_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_mirna" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("MIRN", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'MicroRNA | miRNA Name:hsa-mir-126, Platform:IlluminaGA, Value:RPM' - label = "MicroRNA | miRNA Name:" + row['mirna_name'] + ", Platform:" + row['platform'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT mirna_name, platform, value_field, internal_feature_id ' \ - 'FROM {table_name} ' \ - 'WHERE mirna_name LIKE %s ' \ - 'AND platform LIKE %s ' \ - 'AND value_field LIKE %s ' \ - 'LIMIT %s'.format(table_name=self.get_table_name() - ) - logging.debug(query) - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - query_args = ['%' + input['mirna_name'] + '%', - '%' + input['platform'] + '%', - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = MIRN_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error') \ No newline at end of file diff --git a/bq_data_access/feature_search/mirna.py b/bq_data_access/feature_search/mirna.py deleted file mode 100644 index 51d70e05..00000000 --- a/bq_data_access/feature_search/mirna.py +++ /dev/null @@ -1,74 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException - -from bq_data_access.mirna_data import build_feature_label, MIRN_FEATURE_TYPE - - -class MIRNSearcher(object): - search_fields = set(['mirna_name', 'platform', 'value_field']) - - @classmethod - def get_table_name(cls): - return "feature_defs_mirna" - - def validate_search_field(self, keyword, field): - if field not in self.search_fields: - raise InvalidFieldException("MIRN", keyword, field) - - def search(self, keyword, field): - self.validate_search_field(keyword, field) - - query = 'SELECT mirna_name, platform, value_field, internal_feature_id ' \ - 'FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = MIRN_FEATURE_TYPE - item['label'] = build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error', keyword, field) - - - diff --git a/bq_data_access/feature_search/mutation.py b/bq_data_access/feature_search/mutation.py deleted file mode 100644 index f78e08f0..00000000 --- a/bq_data_access/feature_search/mutation.py +++ /dev/null @@ -1,139 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict - -from api.api_helpers import sql_connection -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.maf_data import build_feature_label, GNAB_FEATURE_TYPE - -class GNABSearcher(object): - feature_search_valid_fields = set(['gene_name', 'value_field']) - field_search_valid_fields = set(['gene_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'value_field', - 'label': 'Value', - 'static': True, - 'values': ['variant_classification', 'variant_type', 'sequence_source', 'num_mutations']} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return GNAB_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_gnab" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("GNAB", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, value_field, internal_feature_id' \ - ' FROM {table_name}' \ - ' WHERE gene_name LIKE %s'\ - ' AND value_field LIKE %s' \ - ' LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - # sql_keyword = '%' + keyword + '%' - query_args = ['%' + input['gene_name'] + '%', - '%' + input['value_field'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = GNAB_FEATURE_TYPE - item['label'] = build_feature_label(item) - - return items - - except MySQLError as mse: - raise BackendException('database error') - diff --git a/bq_data_access/feature_search/protein.py b/bq_data_access/feature_search/protein.py deleted file mode 100644 index d07f8b24..00000000 --- a/bq_data_access/feature_search/protein.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from MySQLdb.cursors import DictCursor -from _mysql_exceptions import MySQLError - -from copy import deepcopy -from collections import defaultdict -from api.api_helpers import sql_connection - -from bq_data_access.feature_search.common import FOUND_FEATURE_LIMIT -from bq_data_access.feature_search.common import BackendException, InvalidFieldException, EmptyQueryException - -from bq_data_access.protein_data import RPPA_FEATURE_TYPE - -class RPPASearcher(object): - feature_search_valid_fields = set(['gene_name', 'protein_name']) - field_search_valid_fields = set(['gene_name', 'probe_name', 'protein_name']) - - searchable_fields = [ - {'name': 'gene_name', - 'label': 'Gene', - 'static': False}, - {'name': 'protein_name', - 'label': 'Protein', - 'static': False} - ] - - @classmethod - def get_searchable_fields(cls): - return deepcopy(cls.searchable_fields) - - @classmethod - def get_datatype_identifier(cls): - return RPPA_FEATURE_TYPE - - @classmethod - def get_table_name(cls): - return "feature_defs_rppa" - - def validate_field_search_input(self, keyword, field): - if field not in self.field_search_valid_fields: - raise InvalidFieldException("RPPA", keyword, field) - - def field_value_search(self, keyword, field): - self.validate_field_search_input(keyword, field) - - query = 'SELECT DISTINCT {search_field} FROM {table_name} WHERE {search_field} LIKE %s LIMIT %s'.format( - table_name=self.get_table_name(), - search_field=field - ) - # Format the keyword for MySQL string matching - sql_keyword = '%' + keyword + '%' - query_args = [sql_keyword, FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row[field]) - - return items - - except MySQLError as mse: - raise BackendException('database error: ' + str(mse)) - - def validate_feature_search_input(self, parameters): - # Check that the input contains only allowed fields - for field, keyword in parameters.iteritems(): - if field not in self.feature_search_valid_fields: - raise InvalidFieldException(", ".join([self.get_datatype_identifier(), field, keyword])) - - # At least one field has to have a non-empty keyword - found_field = False - for field, keyword in parameters.iteritems(): - if len(keyword) > 0: - found_field = True - continue - - if not found_field: - raise EmptyQueryException(self.get_datatype_identifier()) - - def build_feature_label(self, row): - # Example: 'Protein | Gene:EGFR, Protein:EGFR_pY1068, Value:protein_expression' - label = "Protein | Gene:" + row['gene_name'] + ", Protein:" + row['protein_name'] + ", Value:" + row['value_field'] - return label - - def search(self, parameters): - self.validate_feature_search_input(parameters) - - query = 'SELECT gene_name, protein_name, value_field, internal_feature_id ' \ - 'FROM {table_name} ' \ - 'WHERE gene_name=%s ' \ - 'AND protein_name LIKE %s ' \ - 'LIMIT %s'.format(table_name=self.get_table_name() - ) - - # Fills in '' for fields that were not specified in the parameters - input = defaultdict(lambda: '', parameters) - - # Format the keyword for MySQL string matching - query_args = [input['gene_name'], - '%' + input['protein_name'] + '%', - FOUND_FEATURE_LIMIT] - - try: - db = sql_connection() - cursor = db.cursor(DictCursor) - cursor.execute(query, tuple(query_args)) - items = [] - - for row in cursor.fetchall(): - items.append(row) - - # Generate human readable labels - for item in items: - item['feature_type'] = RPPA_FEATURE_TYPE - item['label'] = self.build_feature_label(item) - - return items - - except MySQLError: - raise BackendException('database error') \ No newline at end of file diff --git a/bq_data_access/feature_search/util.py b/bq_data_access/feature_search/util.py deleted file mode 100755 index bd3c2ed7..00000000 --- a/bq_data_access/feature_search/util.py +++ /dev/null @@ -1,49 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from bq_data_access.feature_search.gexp_searcher import GEXPSearcher -from bq_data_access.feature_search.clinical_searcher import ClinicalSearcher -from bq_data_access.feature_search.methylation_searcher import METHSearcher -from bq_data_access.feature_search.copynumber_search import CNVRSearcher -from bq_data_access.feature_search.protein import RPPASearcher -from bq_data_access.feature_search.microrna_searcher import MIRNSearcher -from bq_data_access.feature_search.gnab_searcher import GNABSearcher - -class SearchableFieldHelper(object): - datatype_handlers = [ - GEXPSearcher, - ClinicalSearcher, - METHSearcher, - CNVRSearcher, - RPPASearcher, - MIRNSearcher, - GNABSearcher - ] - - @classmethod - def get_fields_for_all_datatypes(cls): - result = [] - for searcher in cls.datatype_handlers: - datatype = searcher.get_datatype_identifier() - searchable_fields = searcher.get_searchable_fields() - result.append({ - 'datatype': datatype, - 'fields': searchable_fields - }) - - return result diff --git a/bq_data_access/feature_value_types.py b/bq_data_access/feature_value_types.py deleted file mode 100755 index a0b1bd19..00000000 --- a/bq_data_access/feature_value_types.py +++ /dev/null @@ -1,101 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from protorpc.messages import Enum - - -def enum(**enums): - return type('Enum', (), enums) - -# TODO decouple from protorpc.messages -class ValueType(Enum): - STRING = 1 - INTEGER = 2 - FLOAT = 3 - BOOLEAN = 4 - UNKNOWN = 5 # We can get queries that return no data, which may be of an unknown type - - -def is_log_transformable(attr_type): - return isinstance(attr_type, ValueType) and (attr_type == ValueType.FLOAT or attr_type == ValueType.INTEGER) - -IdentifierTypes = enum(PATIENT=1, SAMPLE=2, ALIQUOT=3) -DataTypes = enum(CLIN=1, GEXP=2, METH=3, CNVR=4, RPPA=5, MIRN=6, GNAB=7, USER=8) - -IDENTIER_FIELDS_FOR_DATA_TYPES = { - #TODO: change clin to match new BQ clin table in tcga_data_open - DataTypes.CLIN: { - IdentifierTypes.PATIENT: 'ParticipantBarcode' - }, - #TODO: change gexp to match new BQ gexp table in tcga_data_open; not yet uploaded yet - DataTypes.GEXP: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.METH: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.CNVR: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.RPPA: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.MIRN: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'SampleBarcode', - IdentifierTypes.ALIQUOT: 'AliquotBarcode' - }, - DataTypes.GNAB: { - IdentifierTypes.PATIENT: 'ParticipantBarcode', - IdentifierTypes.SAMPLE: 'Tumor_SampleBarcode', - IdentifierTypes.ALIQUOT: 'Tumor_AliquotBarcode' - }, - DataTypes.USER: { - IdentifierTypes.SAMPLE: 'sample_barcode' - } -} - -class DataPointIdentifierTools(object): - @classmethod - def get_id_field_name_for_data_type(cls, data_type, identifier_type): - return IDENTIER_FIELDS_FOR_DATA_TYPES[data_type][identifier_type] - -class BigQuerySchemaToValueTypeConverter(object): - field_to_value_types = { - 'STRING': ValueType.STRING, - 'INTEGER': ValueType.INTEGER, - 'FLOAT': ValueType.FLOAT, - 'BOOLEAN': ValueType.BOOLEAN - } - - @classmethod - def get_value_type(cls, schema_field): - return cls.field_to_value_types[schema_field] - -class StringToDataTypeConverter(object): - @classmethod - def get_datatype(cls, x): - pass \ No newline at end of file diff --git a/bq_data_access/gexp_data.py b/bq_data_access/gexp_data.py deleted file mode 100755 index 70129881..00000000 --- a/bq_data_access/gexp_data.py +++ /dev/null @@ -1,192 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.utils import DurationLogged - -import sys - -TABLES = [ - { - 'table_id': 'mRNA_UNC_GA_RSEM', - 'platform': 'Illumina GA', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_ga', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - }, - { - 'table_id': 'mRNA_UNC_HiSeq_RSEM', - 'platform': 'Illumina HiSeq', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_hiseq', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - } -] - -GEXP_FEATURE_TYPE = 'GEXP' - -GENE_LABEL_FIELD = 'HGNC_gene_symbol' - - -def get_feature_type(): - return GEXP_FEATURE_TYPE - - -class GEXPFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: GEXP:TP53:mrna_bcgsc_illumina_hiseq - regex = re_compile("^GEXP:" - # gene - "([a-zA-Z0-9\-]+):" - # table - "(" + "|".join([table['id'] for table in TABLES]) + - ")$") - - def __init__(self, gene, value_field, table_id): - self.gene = gene - self.value_field = value_field - self.table_id = table_id - - @classmethod - def get_table_info(cls, table_id): - table_info = None - for table_entry in TABLES: - if table_id == table_entry['id']: - table_info = table_entry - - return table_info - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - gene_label, table_id = feature_fields[0] - value_field = cls.get_table_info(table_id)['value_field'] - return cls(gene_label, value_field, table_id) - - -class GEXPFeatureProvider(FeatureDataProvider): - TABLES = TABLES - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(GEXPFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.GEXP - - def process_data_point(self, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = "SELECT ParticipantBarcode AS case_id, SampleBarcode AS sample_id, AliquotBarcode AS aliquot_id, {value_field} AS value " \ - "FROM [{project_name}:{dataset_name}.{table_name}] AS gexp " \ - "WHERE {gene_label_field}='{gene_symbol}' " \ - "AND SampleBarcode IN ( " \ - " SELECT sample_barcode " \ - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " \ - " WHERE cohort_id IN ({cohort_id_list}) " \ - " AND (project_id IS NULL" - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene_label_field=GENE_LABEL_FIELD, - gene_symbol=feature_def.gene, value_field=feature_def.value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GEXP: " + query) - return query - - @DurationLogged('GEXP', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the clinical data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'case_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - - # TODO refactor, duplicate code shared with GEXPFeatureDef - def get_table_info(self, table_id): - table_info = None - for table_entry in self.TABLES: - if table_id == table_entry['id']: - table_info = table_entry - - return table_info - - def parse_internal_feature_id(self, feature_id): - self.feature_def = GEXPFeatureDef.from_feature_id(feature_id) - - table_info = self.get_table_info(self.feature_def.table_id) - self.table_name = table_info['table_id'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - GEXPFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # GEXPFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/gnab_data.py b/bq_data_access/gnab_data.py deleted file mode 100755 index a3eb2088..00000000 --- a/bq_data_access/gnab_data.py +++ /dev/null @@ -1,176 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -GNAB_FEATURE_TYPE = 'GNAB' -IDENTIFIER_COLUMN_NAME = 'sample_id' - - -def get_feature_type(): - return GNAB_FEATURE_TYPE - - -class GNABFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: GNAB:SMYD3:sequence_source - regex = re_compile("^GNAB:" - # gene - "([a-zA-Z0-9_.\-]+):" - # value field - "(variant_classification|variant_type|sequence_source|num_mutations)$") - - def __init__(self, gene, value_field): - self.gene = gene - self.value_field = value_field - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - gene_label, value_field = feature_fields[0] - - return cls(gene_label, value_field) - - -class GNABFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Somatic_Mutation_calls', - 'info': 'MAF', - 'id': 'maf' - } - ] - - VALUE_FIELD_NUM_MUTATIONS = 'num_mutations' - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_info = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(GNABFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - if self.feature_def.value_field == self.VALUE_FIELD_NUM_MUTATIONS: - return ValueType.FLOAT - else: - return ValueType.STRING - - def get_feature_type(self): - return DataTypes.GNAB - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = "SELECT ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " \ - "{value_field} AS value " \ - "FROM [{project_name}:{dataset_name}.{table_name}] " \ - "WHERE Hugo_Symbol='{gene}' " \ - "AND Tumor_SampleBarcode IN ( " \ - " SELECT sample_barcode " \ - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " \ - " WHERE cohort_id IN ({cohort_id_list})" \ - " AND (project_id IS NULL" - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - value_field_bqsql = self.feature_def.value_field - - if self.feature_def.value_field == self.VALUE_FIELD_NUM_MUTATIONS: - value_field_bqsql = 'count(*)' - query_template += ("GROUP BY ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - "Normal_SampleBarcode, Normal_AliquotBarcode") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=feature_def.gene, value_field=value_field_bqsql, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GNAB: " + query) - return query - - @DurationLogged('GNAB', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the MAF data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': row['f'][3]['v'], - }) - - return result - - def get_table_info(self): - return self.TABLES[0] - - def parse_internal_feature_id(self, feature_id): - self.feature_def = GNABFeatureDef.from_feature_id(feature_id) - self.table_info = self.get_table_info() - - if self.table_info is None: - raise FeatureNotFoundException(feature_id) - - self.table_name = self.table_info['name'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - GNABFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # GNABFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/maf_data.py b/bq_data_access/maf_data.py deleted file mode 100755 index f5d280a7..00000000 --- a/bq_data_access/maf_data.py +++ /dev/null @@ -1,233 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from api.api_helpers import authorize_credentials_with_Google - -from django.conf import settings - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes - -GNAB_FEATURE_TYPE = 'GNAB' -IDENTIFIER_COLUMN_NAME = 'sample_id' - -TABLES = [ - { - 'name': 'Somatic_Mutation_calls', - 'info': 'MAF', - 'id': 'maf' - } -] - -VALUE_FIELD_NUM_MUTATIONS = 'num_mutations' -VALUES = frozenset(['variant_classification', 'variant_type', 'sequence_source', VALUE_FIELD_NUM_MUTATIONS]) - -def get_feature_type(): - return GNAB_FEATURE_TYPE - -def get_table_info(): - return TABLES[0] - -def get_table_id(): - return get_table_info()['id'] - -def build_feature_label(row): - # Example: 'Mutation | Gene:EGFR, Value:variant_classification' - label = "Mutation | Gene:" + row['gene_name'] + ", Value:" + row['value_field'] - return label - -def build_internal_feature_id(gene, value_field): - return '{feature_type}:{gene}:{table}:{value}'.format( - feature_type=get_feature_type(), - gene=gene, - table=get_table_id(), - value=value_field) - -def build_internal_feature_id(gene, value_field): - return '{feature_type}:{gene}:{value}'.format( - feature_type=get_feature_type(), - gene=gene, - value=value_field - ) - -def validate_input(query_table_id): - valid_tables = set([x['id'] for x in TABLES]) - if query_table_id not in valid_tables: - raise Exception("Invalid table ID for maf") - -def build_query(project_name, dataset_name, table_name, gene, value_field, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - "Normal_SampleBarcode, Normal_AliquotBarcode, {value_field} AS value " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE Hugo_Symbol='{gene}' " - "AND Tumor_SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - if value_field == 'num_mutations': - value_field = 'count(*)' - query_template += ("GROUP BY ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - "Normal_SampleBarcode, Normal_AliquotBarcode") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=gene, value_field=value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GNAB: " + query) - return query - -def do_query(project_id, project_name, dataset_name, table_name, gene_label, value_field, cohort_dataset, cohort_table, cohort_id_array): - bigquery_service = authorize_credentials_with_Google() - - query = build_query(project_name, dataset_name, table_name, gene_label, value_field, cohort_dataset, cohort_table, cohort_id_array) - query_body = { - 'query': query - } - - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=project_id, body=query_body).execute() - - result = [] - num_result_rows = int(query_response['totalRows']) - if num_result_rows == 0: - return result - - for row in query_response['rows']: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': row['f'][5]['v'], - }) - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][3]['v'], - 'aliquot_id': row['f'][4]['v'], - 'value': row['f'][5]['v'], - }) - - return result - -def build_feature_query(): - query_template = ("SELECT Hugo_Symbol \ - FROM [{project_name}:{dataset_name}.{table_name}] \ - GROUP BY Hugo_Symbol") - - query_str = query_template.format(dataset_name=settings.BIGQUERY_DATASET, - project_name=settings.BIGQUERY_PROJECT_NAME, table_name='MAF') - - return [query_str] - -def build_feature_table_stmt(): - stmt = ("CREATE TABLE IF NOT EXISTS {table_name} ( " - "id int(11) unsigned NOT NULL AUTO_INCREMENT, " - "gene_name tinytext, " - "num_search_hits tinytext, " - "value_field tinytext, " - "internal_feature_id tinytext, " - "PRIMARY KEY (id))").format(table_name='feature_defs_gnab') - - fieldnames = ['gene_name', 'num_search_hits', 'value_field', 'internal_feature_id'] - - return fieldnames, stmt - -def insert_features_stmt(): - stmt = ("INSERT INTO {table_name} " - "(gene_name, num_search_hits, value_field, internal_feature_id) " - "VALUES (%s, %s, %s, %s)").format(table_name='feature_defs_gnab') - - return stmt - -def parse_response(row): - result = [] - - gene = row[0]['v'] - - for value in VALUES: - result.append({ - 'gene_name': gene, - 'num_search_hits': 0, - 'value_field': value, - 'internal_feature_id': build_internal_feature_id(gene, value) - }) - - return len(VALUES), result - -class GNABFeatureProvider(object): - def __init__(self, feature_id): - self.feature_type = '' - self.gene_label = '' - self.table_info = None - self.value_field = '' - self.table_name = '' - self.parse_internal_feature_id(feature_id) - - def get_value_type(self): - if self.value_field == VALUE_FIELD_NUM_MUTATIONS: - return ValueType.FLOAT - else: - return ValueType.STRING - - def get_feature_type(self): - return DataTypes.GNAB - - @classmethod - def process_data_point(cls, data_point): - return str(data_point['value']) - - def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - result = do_query(project_id, project_name, dataset_name, - self.table_name, self.gene_label, self.value_field, - cohort_dataset, cohort_table, cohort_id_array) - return result - - def get_data(self, cohort_id_array, cohort_dataset, cohort_table): - result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table) - return result - - def parse_internal_feature_id(self, feature_id): - # TODO Better input validation - feature_type, gene_label, value_field = feature_id.split(':') - if value_field not in VALUES: - raise FeatureNotFoundException(feature_id) - - self.feature_type = feature_type - self.gene_label = gene_label - self.value_field = value_field - self.table_info = get_table_info() - - if self.table_info is None: - raise FeatureNotFoundException(feature_id) - - self.table_name = self.table_info['name'] diff --git a/bq_data_access/metadata.py b/bq_data_access/metadata.py deleted file mode 100755 index dd982498..00000000 --- a/bq_data_access/metadata.py +++ /dev/null @@ -1,50 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -DISEASE_CODES = [ - 'ACC', - 'BLCA', - 'BRCA', - 'CESC', - 'COAD', - 'DLBC', - 'ESCA', - 'GBM', - 'HNSC', - 'KICH', - 'KIRC', - 'KIRP', - 'LAML', - 'LGG', - 'LIHC', - 'LUAD', - 'LUSC', - 'MESO', - 'OV', - 'PAAD', - 'PCPG', - 'PRAD', - 'READ', - 'SARC', - 'SKCM', - 'STAD', - 'THCA', - 'UCEC', - 'UCS', - 'UVM' -] diff --git a/bq_data_access/methylation_data.py b/bq_data_access/methylation_data.py deleted file mode 100755 index 5a210b52..00000000 --- a/bq_data_access/methylation_data.py +++ /dev/null @@ -1,295 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.utils import DurationLogged - -METH_FEATURE_TYPE = 'METH' -IDENTIFIER_COLUMN_NAME = 'sample_id' - -TABLES = [ - { - 'name': 'DNA_Methylation_chr1', - 'info': 'Methylation chr1', - 'id': 'methylation_chr1' - }, - { - 'name': 'DNA_Methylation_chr2', - 'info': 'Methylation chr2', - 'id': 'methylation_chr2' - }, - { - 'name': 'DNA_Methylation_chr3', - 'info': 'Methylation chr3', - 'id': 'methylation_chr3' - }, - { - 'name': 'DNA_Methylation_chr4', - 'info': 'Methylation chr4', - 'id': 'methylation_chr4' - }, - { - 'name': 'DNA_Methylation_chr5', - 'info': 'Methylation chr5', - 'id': 'methylation_chr5' - }, - { - 'name': 'DNA_Methylation_chr6', - 'info': 'Methylation chr6', - 'id': 'methylation_chr6' - }, - { - 'name': 'DNA_Methylation_chr7', - 'info': 'Methylation chr7', - 'id': 'methylation_chr7' - }, - { - 'name': 'DNA_Methylation_chr8', - 'info': 'Methylation chr8', - 'id': 'methylation_chr8' - }, - { - 'name': 'DNA_Methylation_chr9', - 'info': 'Methylation chr9', - 'id': 'methylation_chr9' - }, - { - 'name': 'DNA_Methylation_chr10', - 'info': 'Methylation chr10', - 'id': 'methylation_chr10' - }, - { - 'name': 'DNA_Methylation_chr11', - 'info': 'Methylation chr11', - 'id': 'methylation_chr11' - }, - { - 'name': 'DNA_Methylation_chr12', - 'info': 'Methylation chr12', - 'id': 'methylation_chr12' - }, - { - 'name': 'DNA_Methylation_chr13', - 'info': 'Methylation chr13', - 'id': 'methylation_chr13' - }, - { - 'name': 'DNA_Methylation_chr14', - 'info': 'Methylation chr14', - 'id': 'methylation_chr14' - }, - { - 'name': 'DNA_Methylation_chr15', - 'info': 'Methylation chr15', - 'id': 'methylation_chr15' - }, - { - 'name': 'DNA_Methylation_chr16', - 'info': 'Methylation chr16', - 'id': 'methylation_chr16' - }, - { - 'name': 'DNA_Methylation_chr17', - 'info': 'Methylation chr17', - 'id': 'methylation_chr17' - }, - { - 'name': 'DNA_Methylation_chr18', - 'info': 'Methylation chr18', - 'id': 'methylation_chr18' - }, - { - 'name': 'DNA_Methylation_chr19', - 'info': 'Methylation chr19', - 'id': 'methylation_chr19' - }, - { - 'name': 'DNA_Methylation_chr20', - 'info': 'Methylation chr20', - 'id': 'methylation_chr20' - }, - { - 'name': 'DNA_Methylation_chr21', - 'info': 'Methylation chr21', - 'id': 'methylation_chr21' - }, - { - 'name': 'DNA_Methylation_chr22', - 'info': 'Methylation chr22', - 'id': 'methylation_chr22' - }, - { - 'name': 'DNA_Methylation_chrX', - 'info': 'Methylation chrX', - 'id': 'methylation_chrX' - }, - { - 'name': 'DNA_Methylation_chrY', - 'info': 'Methylation chrY', - 'id': 'methylation_chrY' - } -] - -VALUES = ['beta_value'] - - -def get_feature_type(): - return METH_FEATURE_TYPE - - -class METHFeatureDef(object): - def __init__(self, probe, platform, chromosome): - self.probe = probe - self.platform = platform - self.chromosome = chromosome - - @classmethod - def from_feature_id(cls, feature_id): - # Example ID: METH:cg08246323:HumanMethylation450:methylation_chr16 - regex = re_compile("^METH:" - # TODO better validation for probe name - "([a-zA-Z0-9_.\-]+):" - # platform - "(HumanMethylation27|HumanMethylation450):" - # validate outside - chromosome 1-23, X, Y, M - "methylation_chr(\d|\d\d|X|Y|M)$") - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - probe, platform, chromosome = feature_fields[0] - - valid_chr_set = frozenset([str(x) for x in xrange(1, 24)] + ['X', 'Y', 'M']) - if chromosome not in valid_chr_set: - raise FeatureNotFoundException(feature_id) - - return cls(probe, platform, chromosome) - - def __str__(self): - return "METH:{probe}:{platform}:methylation_chr{chr}".format( - probe=self.probe, - platform=self.platform, - chr=self.chromosome - ) - - -class METHFeatureProvider(FeatureDataProvider): - TABLES = TABLES - - def __init__(self, feature_id, **kwargs): - self.feature_type = '' - self.cpg_probe = '' - self.feature_def = None - self.table_name = '' - self.platform = '' - self.parse_internal_feature_id(feature_id) - super(METHFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.METH - - @classmethod - def process_data_point(cls, data_point): - return data_point['beta_value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, beta_value " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE ( Probe_Id='{probe_id}' AND Platform='{platform}') " - "AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - probe_id=feature_def.probe, platform=feature_def.platform, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_METH: " + query) - return query - - @DurationLogged('METH', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the clinical data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'beta_value': float(row['f'][3]['v']) - }) - - return result - - def get_table_name_from_feature_def(self, feature_def): - for table_info in self.TABLES: - if table_info['id'].endswith(feature_def.chromosome): - return table_info['name'] - - raise Exception("Table not found for " + str(feature_def)) - - def parse_internal_feature_id(self, feature_id): - self.feature_def = METHFeatureDef.from_feature_id(feature_id) - self.table_name = self.get_table_name_from_feature_def(self.feature_def) - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - METHFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # METHFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/mirna_data.py b/bq_data_access/mirna_data.py deleted file mode 100755 index 65da0b00..00000000 --- a/bq_data_access/mirna_data.py +++ /dev/null @@ -1,228 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -TABLES = [ - { - 'name': 'miRNA_BCGSC_GA_mirna', - 'info': 'miRNA (GA, BCGSC RPM)', - 'platform': 'IlluminaGA', - 'feature_id': 'mirna_illumina_ga_rpm', - 'value_field': 'reads_per_million_miRNA_mapped' - }, - { - 'name': 'miRNA_BCGSC_HiSeq_mirna', - 'info': 'miRNA (HiSeq, BCGSC RPM)', - 'platform': 'IlluminaHiSeq', - 'feature_id': 'mirna_illumina_hiseq_rpm', - 'value_field': 'reads_per_million_miRNA_mapped' - }, - { - 'name': 'miRNA_Expression', - 'platform': 'both', - 'info': 'miRNA', - 'feature_id': 'expression', - 'value_field': 'normalized_count' - } -] - -TABLE_IDX_MIRNA_EXPRESSION = 2 - -VALUE_READS_PER_MILLION = 'RPM' -VALUE_NORMALIZED_COUNT = 'normalized_count' - -MIRN_FEATURE_TYPE = 'MIRN' -COHORT_FIELD_NAME = 'sample_id' - - -def get_feature_type(): - return MIRN_FEATURE_TYPE - - -def get_mirna_expression_table_info(): - return TABLES[TABLE_IDX_MIRNA_EXPRESSION] - - -def get_table_info(platform, value): - table_info = None - if value == VALUE_NORMALIZED_COUNT: - table_info = get_mirna_expression_table_info() - else: - for table_entry in TABLES: - if platform == table_entry['platform']: - table_info = table_entry - return table_info - - -class MIRNFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: MIRN:hsa-mir-1244-1:mirna_illumina_ga_rpm - regex = re_compile("^MIRN:" - # mirna name - "([a-zA-Z0-9._\-]+):" - # table - "(" + "|".join([table['feature_id'] for table in TABLES]) + - ")$") - - def __init__(self, mirna_name, platform, value_field, table_id): - self.mirna_name = mirna_name - self.platform = platform - self.value_field = value_field - self.table_id = table_id - - @classmethod - def get_table_info(cls, table_id): - table_info = None - for table_entry in TABLES: - if table_id == table_entry['feature_id']: - table_info = table_entry - - return table_info - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - mirna_name, table_id = feature_fields[0] - table_info = cls.get_table_info(table_id) - platform = table_info['platform'] - value_field = table_info['value_field'] - - return cls(mirna_name, platform, value_field, table_id) - - -class MIRNFeatureProvider(FeatureDataProvider): - TABLES = TABLES - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_info = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(MIRNFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.MIRN - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, {value_field} AS value, {mirna_name_field} " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE {mirna_name_field}='{mirna_name}' ") - - if table_name == get_mirna_expression_table_info()['name']: - mirna_name_field = 'mirna_id' - query_template += " AND Platform='{platform}' " - else: - mirna_name_field = 'miRNA_ID' - - query_template += \ - ("AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - mirna_name_field=mirna_name_field, mirna_name=feature_def.mirna_name, - platform=feature_def.platform, - value_field=feature_def.value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_MIRN: " + query) - return query - - @DurationLogged('MIRN', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the miRNA data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - - def get_table_info(self, table_id): - table_info = None - for table_entry in self.TABLES: - if table_id == table_entry['feature_id']: - table_info = table_entry - - return table_info - - def parse_internal_feature_id(self, feature_id): - self.feature_def = MIRNFeatureDef.from_feature_id(feature_id) - self.table_info = self.get_table_info(self.feature_def.table_id) - self.table_name = self.table_info['name'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - MIRNFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # MIRNFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/mrna_data.py b/bq_data_access/mrna_data.py deleted file mode 100755 index d609d64f..00000000 --- a/bq_data_access/mrna_data.py +++ /dev/null @@ -1,186 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from api.api_helpers import authorize_credentials_with_Google -from django.conf import settings - -import logging - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes - -TABLES = [ - { - 'table_id': 'mRNA_BCGSC_GA_RPKM', - 'platform': 'Illumina GA', - 'center': 'BCGSC', - 'id': 'mrna_bcgsc_illumina_ga', - 'value_label': 'RPKM', - 'value_field': 'RPKM' - }, - { - 'table_id': 'mRNA_BCGSC_HiSeq_RPKM', - 'platform': 'Illumina HiSeq', - 'center': 'BCGSC', - 'id': 'mrna_bcgsc_illumina_hiseq', - 'value_label': 'RPKM', - 'value_field': 'RPKM' - }, - { - 'table_id': 'mRNA_UNC_GA_RSEM', - 'platform': 'Illumina GA', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_ga', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - }, - { - 'table_id': 'mRNA_UNC_HiSeq_RSEM', - 'platform': 'Illumina HiSeq', - 'center': 'UNC', - 'id': 'mrna_unc_illumina_hiseq', - 'value_label': 'RSEM', - 'value_field': 'normalized_count' - } -] - -GEXP_FEATURE_TYPE = 'GEXP' - -def get_feature_type(): - return GEXP_FEATURE_TYPE - -def build_feature_label(gene, info): - # print info - # Example: 'EGFR mRNA (Illumina HiSeq, UNC RSEM)' - label = gene + " mRNA (" + info['platform'] + ", " + info['center'] + " " + info['value_label'] + ")" - return label - -def build_internal_feature_id(gene, table_id): - return '{feature_type}:{gene}:{table}'.format( - feature_type=get_feature_type(), - gene=gene, - table=table_id - ) - -def get_table_info(table_id): - table_info = None - for table_entry in TABLES: - if table_id == table_entry['id']: - table_info = table_entry - - return table_info - -def build_query(project_name, dataset_name, table_name, gene_symbol, value_field, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode AS patient_id, SampleBarcode AS sample_id, AliquotBarcode AS aliquot_id, {value_field} AS value " - "FROM [{project_name}:{dataset_name}.{table_name}] AS gexp " - "WHERE original_gene_symbol='{gene_symbol}' " - "AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene_symbol=gene_symbol, value_field=value_field, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_GEXP: " + query) - return query - -def do_query(project_id, project_name, dataset_name, table_name, gene_symbol, value_field, - cohort_dataset, cohort_table, cohort_id_array): - bigquery_service = authorize_credentials_with_Google() - - query = build_query(project_name, dataset_name, table_name, gene_symbol, value_field, - cohort_dataset, cohort_table, cohort_id_array) - query_body = { - 'query': query - } - - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=project_id, body=query_body).execute() - result = [] - num_result_rows = int(query_response['totalRows']) - if num_result_rows == 0: - return result - - for row in query_response['rows']: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - -class MRNAFeatureProvider(object): - def __init__(self, feature_id): - self.feature_type = '' - self.gene_label = '' - self.table_id = '' - self.table_info = None - self.value_field = '' - self.table_name = '' - self.parse_internal_feature_id(feature_id) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.GEXP - - def process_data_point(self, data_point): - return str(data_point['value']) - - def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - result = do_query(project_id, project_name, dataset_name, self.table_name, self.gene_label, self.value_field, - cohort_dataset, cohort_table, cohort_id_array) - return result - - def get_data(self, cohort_id_array, cohort_dataset, cohort_table): - result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table) - return result - - def parse_internal_feature_id(self, feature_id): - # TODO better feature ID input validation - feature_type, gene_label, table_id = feature_id.split(':') - self.feature_type = feature_type - self.gene_label = gene_label - self.table_id = table_id - self.table_info = get_table_info(table_id) - - if self.table_info is None: - raise FeatureNotFoundException(feature_id) - - self.table_name = self.table_info['table_id'] - self.value_field = self.table_info['value_field'] - diff --git a/bq_data_access/protein_data.py b/bq_data_access/protein_data.py deleted file mode 100755 index ec76f805..00000000 --- a/bq_data_access/protein_data.py +++ /dev/null @@ -1,168 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from django.conf import settings - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import ValueType, DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -RPPA_FEATURE_TYPE = 'RPPA' - - -def get_feature_type(): - return RPPA_FEATURE_TYPE - - -class RPPAFeatureDef(object): - # Regular expression for parsing the feature definition. - # - # Example ID: RPPA:GYG1:GYG-Glycogenin1 - regex = re_compile("^RPPA:" - # gene - "([a-zA-Z0-9\-]+):" - # protein name - "([a-zA-Z0-9._\-]+)$") - - def __init__(self, gene, protein_name): - self.gene = gene - self.protein_name = protein_name - - @classmethod - def from_feature_id(cls, feature_id): - feature_fields = cls.regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - - gene_label, protein_name = feature_fields[0] - return cls(gene_label, protein_name) - - -class RPPAFeatureProvider(FeatureDataProvider): - TABLES = [ - { - 'name': 'Protein_RPPA_data', - 'info': 'Protein', - 'id': 'protein' - } - ] - - def __init__(self, feature_id, **kwargs): - self.feature_def = None - self.table_info = None - self.table_name = '' - self.parse_internal_feature_id(feature_id) - super(RPPAFeatureProvider, self).__init__(**kwargs) - - def get_value_type(self): - return ValueType.FLOAT - - def get_feature_type(self): - return DataTypes.RPPA - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, SampleBarcode, AliquotBarcode, protein_expression AS value " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE ( gene_name='{gene}' AND protein_name='{protein}' ) " - "AND SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=feature_def.gene, protein=feature_def.protein_name, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_RPPA: " + query) - return query - - @DurationLogged('RPPA', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. The array will contain dicts with - the following fields: - - 'patient_id': Patient barcode - - 'sample_id': Sample barcode - - 'aliquot_id': Aliquot barcode - - 'value': Value of the selected column from the protein data table - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'value': float(row['f'][3]['v']) - }) - - return result - - def get_data_from_bigquery(self, cohort_id_array, cohort_dataset, cohort_table): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - result = self.do_query(project_id, project_name, dataset_name, self.table_name, self.feature_def, - cohort_dataset, cohort_table, cohort_id_array) - return result - - def get_data(self, cohort_id_array, cohort_dataset, cohort_table): - result = self.get_data_from_bigquery(cohort_id_array, cohort_dataset, cohort_table) - return result - - def parse_internal_feature_id(self, feature_id): - self.feature_def = RPPAFeatureDef.from_feature_id(feature_id) - self.table_name = self.TABLES[0]['name'] - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - RPPAFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # RPPAFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/seqpeek/__init__.py b/bq_data_access/seqpeek/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/bq_data_access/seqpeek/seqpeek_interpro.py b/bq_data_access/seqpeek/seqpeek_interpro.py deleted file mode 100644 index 18dfb77c..00000000 --- a/bq_data_access/seqpeek/seqpeek_interpro.py +++ /dev/null @@ -1,72 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from json import loads as json_loads -import logging -from api.api_helpers import authorize_credentials_with_Google - -from django.conf import settings - - -class SeqPeekMAFWithCohorts(object): - def __init__(self, maf_vector, cohort_info): - self.maf_vector = maf_vector - self.cohort_info = cohort_info - - -class InterProDataProvider(object): - def __init__(self): - logging.debug(__name__ + ".__init__") - - def build_query(self, project_name, uniprot_id): - query_template = "SELECT * FROM [{project_name}:test.interpro_filtered] WHERE uniprot_id=\'{uniprot_id}\'" - - query = query_template.format(project_name=project_name, uniprot_id=uniprot_id) - logging.debug("INTERPRO SQL: " + query) - return query - - def do_query(self, project_id, project_name, uniprot_id): - bigquery_service = authorize_credentials_with_Google() - - query = self.build_query(project_name, uniprot_id) - query_body = { - 'query': query - } - - table_data = bigquery_service.jobs() - query_response = table_data.query(projectId=project_id, body=query_body).execute() - - num_result_rows = int(query_response['totalRows']) - if num_result_rows == 0: - return None - - row = query_response['rows'][0] - interpro_literal = row['f'][1]['v'] - interpro_literal = interpro_literal.replace('\'', '"') - interpro_literal = json_loads(interpro_literal) - - return interpro_literal - - def get_data_from_bigquery(self, uniprot_id): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - result = self.do_query(project_id, project_name, uniprot_id) - return result - - def get_data(self, uniprot_id): - return self.get_data_from_bigquery(uniprot_id) diff --git a/bq_data_access/seqpeek/seqpeek_maf_formatter.py b/bq_data_access/seqpeek/seqpeek_maf_formatter.py deleted file mode 100644 index 1159f6bc..00000000 --- a/bq_data_access/seqpeek/seqpeek_maf_formatter.py +++ /dev/null @@ -1,85 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from collections import defaultdict -from copy import deepcopy -import logging -from re import compile as re_compile - -from bq_data_access.cohort_cloudsql import CloudSQLCohortAccess - -COORDINATE_FIELD_NAME = 'uniprot_aapos' -TYPE_FIELD_NAME = 'variant_classification' - -DIGIT_FINDER_RE = re_compile('^\d+$') - - -class SeqPeekMAFWithCohorts(object): - def __init__(self, maf_vector, cohort_info, removed_row_stats): - self.maf_vector = maf_vector - self.cohort_info = cohort_info - self.removed_row_statistics = removed_row_stats - - -class SeqPeekMAFDataFormatter(object): - def annotate_vector_with_cohorts(self, cohort_id_array, result): - # Resolve which (requested) cohorts each datapoint belongs to. - cohort_set_dict = CloudSQLCohortAccess.get_cohorts_for_datapoints(cohort_id_array) - - for row in result: - sample_id = row['sample_id'] - - # Add an array of cohort - # only if the number of containing cohort exceeds the configured threshold. - cohort_set = [] - # TODO FIX - this check shouldn't be needed - if sample_id in cohort_set_dict: - cohort_set = cohort_set_dict[sample_id] - row['cohort'] = cohort_set - - def remove_rows_with_no_aa_position(self, data): - result = [] - removed_stats = defaultdict(int) - - count = 0 - for row in data: - aapos = row[COORDINATE_FIELD_NAME] - if aapos is not None and len(DIGIT_FINDER_RE.findall(aapos)) == 1: - count += 1 - item = deepcopy(row) - item[COORDINATE_FIELD_NAME] = int(aapos) - result.append(item) - # Include removed row in statistics - else: - removed_stats[row[TYPE_FIELD_NAME]] += 1 - - logging.debug("SeqPeek MAF filtered rows: {0}, total: {1}".format(len(result), len(data))) - return result, removed_stats - - def get_cohort_information(self, cohort_id_array): - # Get the name, size and ID for every requested cohort. - cohort_info_array = CloudSQLCohortAccess.get_cohort_info(cohort_id_array) - - return cohort_info_array - - def format_maf_vector_for_view(self, maf_vector, cohort_id_array): - filtered_maf_vector, removed_stats = self.remove_rows_with_no_aa_position(maf_vector) - self.annotate_vector_with_cohorts(cohort_id_array, filtered_maf_vector) - cohort_info = self.get_cohort_information(cohort_id_array) - - return SeqPeekMAFWithCohorts(filtered_maf_vector, cohort_info, removed_stats) diff --git a/bq_data_access/seqpeek/seqpeek_view.py b/bq_data_access/seqpeek/seqpeek_view.py deleted file mode 100644 index 675949af..00000000 --- a/bq_data_access/seqpeek/seqpeek_view.py +++ /dev/null @@ -1,268 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from copy import deepcopy -import logging - -from bq_data_access.seqpeek.seqpeek_interpro import InterProDataProvider - -SAMPLE_ID_FIELD_NAME = 'sample_id' -TRACK_ID_FIELD = "tumor" -COORDINATE_FIELD_NAME = 'uniprot_aapos' -PROTEIN_ID_FIELD = 'uniprot_id' - -PROTEIN_DOMAIN_DB = 'PFAM' - -SEQPEEK_VIEW_DEBUG_MODE = False - - -def build_gnab_feature_id(gene): - return "GNAB:{gene_label}:variant_classification".format(gene_label=gene) - - -def get_number_of_unique_samples(track): - sample_ids = set() - for mutation in track['mutations']: - sample_ids.add(mutation[SAMPLE_ID_FIELD_NAME]) - - return len(sample_ids) - - -def get_number_of_mutated_positions(track): - sample_locations = set() - for mutation in track['mutations']: - sample_locations.add(mutation[COORDINATE_FIELD_NAME]) - - return len(sample_locations) - - -# TODO remove if not needed -def clean_track_mutations(mutations_array): - retval = [] - for mutation in mutations_array: - cleaned = deepcopy(mutation) - cleaned[COORDINATE_FIELD_NAME] = int(mutation[COORDINATE_FIELD_NAME]) - retval.append(cleaned) - - return retval - - -def sort_track_mutations(mutations_array): - return sorted(mutations_array, key=lambda k: k[COORDINATE_FIELD_NAME]) - - -def get_track_statistics_by_track_type(track, cohort_info_map): - track_id = track[TRACK_ID_FIELD] - - result = { - 'samples': { - 'numberOf': get_number_of_unique_samples(track), - 'mutated_positions': get_number_of_mutated_positions(track) - } - } - - if track['type'] == 'tumor': - cohort_info = cohort_info_map[track_id] - result['cohort_size'] = cohort_info['size'] - else: - # Do not assign cohort size for the 'COMBINED' track. - result['cohort_size'] = None - - return result - - -def filter_protein_domains(match_array): - return [m for m in match_array if m['dbname'] == PROTEIN_DOMAIN_DB] - - -def get_table_row_id(tumor_type): - return "seqpeek_row_{0}".format(tumor_type) - - -def build_seqpeek_regions(protein_data): - return [{ - 'type': 'exon', - 'start': 0, - 'end': protein_data['length'] - }] - - -def build_summary_track(tracks): - all = [] - for track in tracks: - all.extend(track["mutations"]) - - return { - 'mutations': all, - 'label': 'COMBINED', - 'tumor': 'none-combined', - 'type': 'summary' - } - - -def get_track_label_and_cohort_information(track_id_value, cohort_info_map): - cohort_info = cohort_info_map[track_id_value] - - label = cohort_info['name'] - cohort_size = cohort_info['size'] - return label, cohort_size - - -def get_track_label(track, cohort_info_array): - # The IDs in cohort_info_array are integers, whereas the track IDs are strings. - cohort_map = {str(item['id']): item['name'] for item in cohort_info_array} - return cohort_map[track[TRACK_ID_FIELD]] - - -def get_protein_domains(uniprot_id): - protein = InterProDataProvider().get_data(uniprot_id) - return protein - - -class MAFData(object): - def __init__(self, cohort_info, data): - self.cohort_info = cohort_info - self.data = data - - @classmethod - def from_dict(cls, param): - return cls(param['cohort_set'], param['items']) - - -def build_track_data(track_id_list, all_tumor_mutations): - tracks = [] - for track_id in track_id_list: - tracks.append({ - TRACK_ID_FIELD: track_id, - 'mutations': filter(lambda m: int(track_id) in set(m['cohort']), all_tumor_mutations) - }) - - return tracks - - -def find_uniprot_id(mutations): - uniprot_id = None - for m in mutations: - if PROTEIN_ID_FIELD in m: - uniprot_id = m[PROTEIN_ID_FIELD] - break - - return uniprot_id - - -def get_genes_tumors_lists_debug(): - return { - 'symbol_list': ['EGFR', 'TP53', 'PTEN'], - 'disease_codes': ['ACC', 'BRCA', 'GBM'] - } - - -def get_genes_tumors_lists_remote(): - context = { - 'symbol_list': [], - 'track_id_list': [] - } - - return context - - -def get_genes_tumors_lists(): - if SEQPEEK_VIEW_DEBUG_MODE: - return get_genes_tumors_lists_debug() - else: - return get_genes_tumors_lists_remote() - - -def get_track_id_list(param): - return map(str, param) - - -def format_removed_row_statistics_to_list(stats_dict): - result = [] - for key, value in stats_dict.items(): - result.append({ - 'name': key, - 'num': value - }) - - return result - - -class SeqPeekViewDataBuilder(object): - def build_view_data(self, hugo_symbol, filtered_maf_vector, seqpeek_cohort_info, cohort_id_list, removed_row_statistics): - context = get_genes_tumors_lists() - - cohort_info_map = {str(item['id']): item for item in seqpeek_cohort_info} - track_id_list = get_track_id_list(cohort_id_list) - - # Since the gene (hugo_symbol) parameter is part of the GNAB feature ID, - # it will be sanity-checked in the SeqPeekMAFDataAccess instance. - uniprot_id = find_uniprot_id(filtered_maf_vector) - - logging.info("UniProt ID: " + str(uniprot_id)) - protein_data = get_protein_domains(uniprot_id) - track_data = build_track_data(track_id_list, filtered_maf_vector) - - plot_data = { - 'gene_label': hugo_symbol, - 'tracks': track_data, - 'protein': protein_data - } - - # Pre-processing - # - Sort mutations by chromosomal coordinate - for track in plot_data['tracks']: - track['mutations'] = sort_track_mutations(track['mutations']) - - # Annotations - # - Add label, possibly human readable - # - Add type that indicates whether the track is driven by data from search or - # if the track is aggregate - for track in plot_data['tracks']: - track['type'] = 'tumor' - - label, cohort_size = get_track_label_and_cohort_information(track[TRACK_ID_FIELD], cohort_info_map) - track['label'] = label - - # Display the "combined" track only if more than one cohort is visualized - if len(cohort_id_list) >= 2: - plot_data['tracks'].append(build_summary_track(plot_data['tracks'])) - - for track in plot_data['tracks']: - # Calculate statistics - track['statistics'] = get_track_statistics_by_track_type(track, cohort_info_map) - # Unique ID for each row - track['render_info'] = { - 'row_id': get_table_row_id(track[TRACK_ID_FIELD]) - } - - plot_data['regions'] = build_seqpeek_regions(plot_data['protein']) - plot_data['protein']['matches'] = filter_protein_domains(plot_data['protein']['matches']) - - tumor_list = ','.join(track_id_list) - - context.update({ - 'plot_data': plot_data, - 'hugo_symbol': hugo_symbol, - 'tumor_list': tumor_list, - 'cohort_id_list': track_id_list, - 'removed_row_statistics': format_removed_row_statistics_to_list(removed_row_statistics) - }) - - return context - diff --git a/bq_data_access/seqpeek_maf_data.py b/bq_data_access/seqpeek_maf_data.py deleted file mode 100755 index 10437638..00000000 --- a/bq_data_access/seqpeek_maf_data.py +++ /dev/null @@ -1,85 +0,0 @@ -""" - -Copyright 2016, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging - -from bq_data_access.gnab_data import GNABFeatureProvider -from bq_data_access.utils import DurationLogged - -SEQPEEK_FEATURE_TYPE = 'SEQPEEK' - - -class SeqPeekDataProvider(GNABFeatureProvider): - def __init__(self, feature_id, **kwargs): - super(SeqPeekDataProvider, self).__init__(feature_id, **kwargs) - - @classmethod - def process_data_point(cls, data_point): - return str(data_point['value']) - - def build_query(self, project_name, dataset_name, table_name, feature_def, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - # Generate the 'IN' statement string: (%s, %s, ..., %s) - cohort_id_stmt = ', '.join([str(cohort_id) for cohort_id in cohort_id_array]) - project_id_stmt = '' - if project_id_array is not None: - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = \ - ("SELECT ParticipantBarcode, Tumor_SampleBarcode, Tumor_AliquotBarcode, " - " Hugo_symbol, " - " UniProt_AApos, " - " variant_classification, " - " HGNC_UniProt_ID_Supplied_By_UniProt as uniprot_id " - "FROM [{project_name}:{dataset_name}.{table_name}] " - "WHERE Hugo_Symbol='{gene}' " - "AND Tumor_SampleBarcode IN ( " - " SELECT sample_barcode " - " FROM [{project_name}:{cohort_dataset}.{cohort_table}] " - " WHERE cohort_id IN ({cohort_id_list})" - " AND (project_id IS NULL") - - query_template += (" OR project_id IN ({project_id_list})))" if project_id_array is not None else "))") - - query = query_template.format(dataset_name=dataset_name, project_name=project_name, table_name=table_name, - gene=feature_def.gene, - cohort_dataset=cohort_dataset, cohort_table=cohort_table, - cohort_id_list=cohort_id_stmt, project_id_list=project_id_stmt) - - logging.debug("BQ_QUERY_SEQPEEK: " + query) - return query - - @DurationLogged('SEQPEEK_GNAB', 'UNPACK') - def unpack_query_response(self, query_result_array): - result = [] - - skip_count = 0 - for row in query_result_array: - result.append({ - 'patient_id': row['f'][0]['v'], - 'sample_id': row['f'][1]['v'], - 'aliquot_id': row['f'][2]['v'], - 'hugo_symbol': row['f'][3]['v'], - 'uniprot_aapos': row['f'][4]['v'], - 'variant_classification': row['f'][5]['v'], - 'uniprot_id': row['f'][6]['v'], - }) - - logging.debug("Query result is {qrows} rows, skipped {skipped} rows".format(qrows=len(query_result_array), - skipped=skip_count)) - return result - diff --git a/bq_data_access/user_data.py b/bq_data_access/user_data.py deleted file mode 100644 index c1607150..00000000 --- a/bq_data_access/user_data.py +++ /dev/null @@ -1,456 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -import logging -from re import compile as re_compile - -from api.api_helpers import sql_connection - -import MySQLdb - -from django.conf import settings - -from bq_data_access.feature_value_types import ValueType - -from bq_data_access.errors import FeatureNotFoundException -from bq_data_access.feature_value_types import DataTypes -from bq_data_access.feature_data_provider import FeatureDataProvider -from bq_data_access.utils import DurationLogged - -USER_FEATURE_TYPE = 'USER' - - -class InvalidUserFeatureIDException(Exception): - def __init__(self, feature_id, reason): - self.feature_id = feature_id - self.reason = reason - - def __str__(self): - return "Invalid feature ID '{feature_id}', reason '{reason}'".format( - feature_id=self.feature_id, - reason=self.reason - ) - - -class UserFeatureDef(object): - def __init__(self, bq_table, column_name, project_id, is_numeric, filters): - logging.debug(str([bq_table, column_name, project_id, is_numeric, filters])) - self.bq_table = bq_table - self.column_name = column_name - self.project_id = project_id - self.filters = filters - self.bq_row_id = None - self.type = "STRING" if is_numeric else "FLOAT" - - def get_value_type(self): - if self.type == "STRING": - return ValueType.STRING - else: - return ValueType.FLOAT - - @classmethod - def get_table_and_field(cls, bq_id): - split = bq_id.split(':') - # First pieces are the project:dataset:table:Data_type:symbol:column_name - bq_table = split[0] + ':' + split[1] + '.' + split[2] - # Last piece is the column name - column_name = bq_id.split(':')[-1] - - # Is only symbol at the moment - symbol = None - if len(split) > 5: - symbol = split[4] - - return bq_table, column_name, symbol - - @classmethod - def from_user_feature_id(cls, feature_id): - logging.debug("UserFeatureDef.from_user_feature_id {0}".format(str([feature_id]))) - # ID breakdown: project ID:Feature ID - # Example ID: USER:1:6 - regex = re_compile("^USER:" - # project ID - "([0-9]+):" - # Feature ID - "([0-9]+)$" - ) - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - project_id, user_feature_id = feature_fields[0] - bq_id = None - shared_id = None - is_numeric = False - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(""" - SELECT feature_name, bq_map_id, shared_map_id, is_numeric - FROM projects_user_feature_definitions - WHERE id = %s - """, (user_feature_id,)) - for row in cursor.fetchall(): - if row['shared_map_id']: - shared_id = row['shared_map_id'] - bq_id = row["bq_map_id"] - is_numeric = row['is_numeric'] == 1 - - cursor.close() - db.close() - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - if shared_id is not None: - return cls.from_feature_id(bq_id, project_id) - - if bq_id is None: - raise FeatureNotFoundException(feature_id) - - # Else we're querying a very specific feature from a specific project - bq_table, column_name, symbol = cls.get_table_and_field(bq_id) - if bq_table is None or column_name is None: - raise FeatureNotFoundException(feature_id) - - logging.debug("{0} {1} {2}".format(bq_table, column_name, symbol)) - - filters = None - if symbol is not None: - filters = { - 'Symbol': symbol - } - - return [cls(bq_table, column_name, project_id, is_numeric, filters)] - - @classmethod - def from_feature_id(cls, feature_id, project_id=None): - logging.debug("UserFeatureDef.from_feature_id: {0}".format(str([feature_id, project_id]))) - if feature_id is None: - raise FeatureNotFoundException(feature_id) - # ID breakdown: project ID:Feature ID - # Example ID: USER:1:6 - regex = re_compile("^USER:" - # project ID - "([0-9]+):" - # Feature ID - "([0-9]+)$" - ) - - feature_fields = regex.findall(feature_id) - if len(feature_fields) == 0: - raise FeatureNotFoundException(feature_id) - project_id, user_feature_id = feature_fields[0] - - - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(""" - SELECT bq_map_id, project_id, is_numeric - FROM projects_user_feature_definitions - WHERE id = %s - """, (user_feature_id,)) - - results = [] - for row in cursor.fetchall(): - bq_table, column_name, symbol = cls.get_table_and_field(row['bq_map_id']) - filters = None - if symbol is not None: - filters = { - 'Symbol': symbol - } - results.append(cls(bq_table, column_name, row['project_id'], row['is_numeric'] == 1, filters)) - - cursor.close() - db.close() - - return results - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - def unpack_value_from_bigquery_row(self, bq_row): - return bq_row['f'][self.bq_row_id + 2]['v'] - - def build_query(self, cohort_table, cohort_ids, project_id_array): - - cohort_str = ",".join([str(cohort_id) for cohort_id in cohort_ids]) - project_id_stmt = '' - if project_id_array is not None and len(project_id_array): - project_id_stmt = ', '.join([str(project_id) for project_id in project_id_array]) - - query_template = "SELECT {fdef_id} AS fdef_id, t.sample_barcode, t.{column_name} FROM [{table_name}] AS t " \ - "JOIN [{cohort_table}] AS c ON c.sample_barcode = t.sample_barcode " \ - "WHERE c.cohort_id IN ({cohort_list}) AND (c.project_id IS NULL" - - - query_template += (" OR c.project_id IN ({project_id_list}))" if project_id_array is not None and len(project_id_array) else ")") - query = query_template.format(fdef_id=self.bq_row_id, - column_name=self.column_name, - table_name=self.bq_table, - cohort_table=cohort_table, - cohort_list=cohort_str, - project_id_list=project_id_stmt) - - if self.filters is not None: - for key, val in self.filters.items(): - query += ' AND t.{filter_key} = "{value}" '.format(filter_key=key, value=val) - - query += " GROUP BY t.sample_barcode, t.{column_name} ".format(column_name=self.column_name) # To prevent duplicates from multiple cohorts - return query - - -class UserFeatureProvider(FeatureDataProvider): - """ - Feature data provider for user data. - """ - def __init__(self, feature_id, user_feature_id=None, **kwargs): - self.feature_defs = None - self.parse_internal_feature_id(feature_id, user_feature_id=user_feature_id) - self._project_ids = None - super(UserFeatureProvider, self).__init__(**kwargs) - - def get_feature_type(self): - return DataTypes.USER - - @classmethod - def convert_user_feature_id(cls, feature_id): - bq_id = None - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - cursor.execute(""" - SELECT feature_name, bq_map_id, shared_map_id - FROM projects_user_feature_definitions - WHERE id = %s - """, (int(feature_id.split(':')[-1]),)) - for row in cursor.fetchall(): - bq_id = row["shared_map_id"] - - cursor.close() - db.close() - - logging.debug("UserFeatureProvider.convert_user_feature_id {0} -> {1}".format(feature_id, bq_id)) - return bq_id - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - @classmethod - def process_data_point(cls, data_point): - return data_point['value'] - - def get_value_type(self): - return self.feature_defs[0].get_value_type() - - def build_query(self, project_ids, cohort_id_array, cohort_dataset, cohort_table, project_id_array): - """ - Builds the BigQuery query string for USER data. The query string is constructed from one or more data sources - (queries), such that each associated UserFeatureDef instance constructs one data source. Each data source - selects one column from a user data table, and therefore maps to one column in the query result. - - The query result table contains a column ("fdef_id") that identifies which data source (UserFeatureDef instance) - that produced each row. - - When unpacking the query result table, the decoding of each row is delegated to the a UserFeatureDef instance - identified by the "fdef_id" column value in that row. - - Example of a query result table: - - |-------+--------------+--------+--------+--------| - |fdef_id|sample_barcode|column_0|column_1|column_n| - |-------+--------------+--------+--------+--------| - |0 |barcode_1 | |null |null | - |0 |barcode_2 | |null |null | - |0 |... | |null |null | - |0 |barcode_m | |null |null | - |-------+--------------+--------+--------+--------| - |1 |barcode_1 |null | |null | - |1 |barcode_2 |null | |null | - |1 |... |null | |null | - |1 |barcode_m |null | |null | - |-------+--------------+--------+--------+--------| - |n |barcode_1 |null |null | | - |n |barcode_2 |null |null | | - |n |... |null |null | | - |n |barcode_m |null |null | | - |-------+--------------+--------+--------+--------| - - Returns: BigQuery query string. - - """ - queries = [] - cohort_table_full = settings.BIGQUERY_PROJECT_NAME + ':' + cohort_dataset + '.' + cohort_table - # TODO: this is a hack to append project_ids to the tcga project id list. project_id_array is actually empty. - project_id_array += project_ids - for feature_def in self.feature_defs: - if int(feature_def.project_id) in project_ids: - # Build our query - queries.append(feature_def.build_query(cohort_table_full, cohort_id_array, project_id_array)) - - # Create a combination query using the UNION ALL operator. Each data source defined above (query1, query2, ...) - # will be combined as follows: - # - # (query 1) - # , - # (query 2) - # , - # ... - # , - # (query n) - # - query = ' , '.join(['(' + query + ')' for query in queries]) - logging.info("BQ_QUERY_USER: " + query) - return query - - def unpack_value_from_row_with_feature_def(self, row): - """ - Decodes the value from a query result. - - Delegates the selection and decoding of the correct column in the row to the UserFeatureDef instance - identified by the "fdef_id" column on the row. - - Args: - row: BigQuery query result row. - - Returns: The value for the row. - - """ - feature_def_index = int(row['f'][0]['v']) # fdef_id - feature_def = self.feature_defs[feature_def_index] - - return feature_def.unpack_value_from_bigquery_row(row) - - @DurationLogged('USER', 'UNPACK') - def unpack_query_response(self, query_result_array): - """ - Unpacks values from a BigQuery response object into a flat array. - - Args: - query_result_array: A BigQuery query response object - - Returns: - Array of dict objects. - """ - result = [] - - for row in query_result_array: - result.append({ - 'patient_id': None, - 'sample_id': row['f'][1]['v'], - 'aliquot_id': None, - 'value': self.unpack_value_from_row_with_feature_def(row) - }) - - return result - - def get_project_ids(self, cohort_id_array): - """ - Returns: The user project identifiers associated with the samples in all given cohorts. - """ - if self._project_ids is not None: - return self._project_ids - - project_ids = () - for cohort_id in cohort_id_array: - try: - db = sql_connection() - cursor = db.cursor(MySQLdb.cursors.DictCursor) - - cursor.execute("SELECT project_id FROM cohorts_samples WHERE cohort_id = %s GROUP BY project_id", (cohort_id,)) - for row in cursor.fetchall(): - if row['project_id'] is not None: - project_ids += (row['project_id'],) - - except Exception as e: - if db: db.close() - if cursor: cursor.close() - raise e - - self._project_ids = project_ids - return self._project_ids - - def parse_internal_feature_id(self, feature_id, user_feature_id=None): - if user_feature_id is None or user_feature_id is '': - logging.debug("UserFeatureProvider.parse_internal_feature_id - feature_id: {0}".format(feature_id)) - self.feature_defs = UserFeatureDef.from_feature_id(feature_id) - else: - logging.debug("UserFeatureProvider.parse_internal_feature_id - user_feature_id: {0}".format(user_feature_id)) - self.feature_defs = UserFeatureDef.from_user_feature_id(user_feature_id) - - # Assign a unique identifier to all feature defs. - # TODO document - for index, feature_def in enumerate(self.feature_defs): - feature_def.bq_row_id = index - - def _submit_query_and_get_job_ref(self, project_id, project_name, dataset_name, cohort_dataset, cohort_table, cohort_id_array, project_id_array): - project_ids = self.get_project_ids(cohort_id_array) - - bigquery_service = self.get_bq_service() - - query_body = self.build_query(project_ids, cohort_id_array, cohort_dataset, cohort_table, project_id_array) - query_job = self.submit_bigquery_job(bigquery_service, project_id, query_body) - - self.job_reference = query_job['jobReference'] - - return self.job_reference - - def is_queryable(self, cohort_id_array): - """ - Answers if this instance would submit a BigQuery job if the submit_query_and_get_job_ref member function - was called. - """ - project_ids = self.get_project_ids(cohort_id_array) - queryable = False - - for feature_def in self.feature_defs: - if int(feature_def.project_id) in project_ids: - queryable = True - break - - return queryable - - def get_data_job_reference(self, cohort_id_array, cohort_dataset, cohort_table, project_id_array): - project_id = settings.BQ_PROJECT_ID - project_name = settings.BIGQUERY_PROJECT_NAME - dataset_name = settings.BIGQUERY_DATASET - - result = self._submit_query_and_get_job_ref(project_id, project_name, dataset_name, - cohort_dataset, cohort_table, cohort_id_array, project_id_array) - return result - - @classmethod - def is_valid_feature_id(cls, feature_id): - is_valid = False - try: - UserFeatureDef.from_feature_id(feature_id) - is_valid = True - except Exception: - # UserFeatureDef.from_feature_id raises Exception if the feature identifier - # is not valid. Nothing needs to be done here, since is_valid is already False. - pass - finally: - return is_valid diff --git a/bq_data_access/utils.py b/bq_data_access/utils.py deleted file mode 100755 index 3c094a01..00000000 --- a/bq_data_access/utils.py +++ /dev/null @@ -1,141 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" - -from collections import OrderedDict -import logging -from time import time - - -def vector_to_dict(vector, id_key, value_key): - result = {} - for item in vector: - item_id = item[id_key] - value = item[value_key] - result[item_id] = value - - return result - - -def find_all_id_keys(*vectors, **kwargs): - value_key = kwargs['value_key'] - id_key = kwargs['id_key'] - identifiers = set() - for index, vector in enumerate(vectors): - for item in vector: - item_id = item[id_key] - - identifiers.add(item_id) - - return identifiers - - -class VectorMergeSupport(object): - def __init__(self, missing_value, sample_id_key, case_id_key, row_ids=[]): - self.missing_value = missing_value - self.sample_id_key = sample_id_key - self.case_id_key = case_id_key - - # Sparse data goes here - self.data = OrderedDict() - self.sample_ids = OrderedDict() - self.case_ids = {} - self.current_sample_index = 0 - - for row_id in row_ids: - self.data[row_id] = [] - - def _get_index_for_sample_id(self, sample_id): - if sample_id not in self.sample_ids: - self.sample_ids[sample_id] = self.current_sample_index - self.current_sample_index += 1 - - return self.sample_ids[sample_id] - - def _add_data_point(self, row_id, sample_id, value): - if row_id not in self.data: - self.data[row_id] = [] - - self.data[row_id].append((self._get_index_for_sample_id(sample_id), value)) - - def add_dict_array(self, vector, vector_id, value_key): - for item in vector: - sample_id = item[self.sample_id_key] - if item[self.sample_id_key] not in self.case_ids: - self.case_ids[sample_id] = item[self.case_id_key] - value = item[value_key] - self._add_data_point(vector_id, sample_id, value) - - def get_merged_dict(self): - num_samples = self.current_sample_index - result = [{} for _ in xrange(num_samples)] - sample_id_keys = self.sample_ids.keys() - - for row_id, row_samples in self.data.items(): - row_values = [self.missing_value] * len(self.sample_ids) - - for sample_index, value in row_samples: - row_values[sample_index] = value - - counter = 0 - for index, value in enumerate(row_values): - counter += 1 - d = result[index] - d[self.sample_id_key] = sample_id_keys[index] - d[row_id] = value - d[self.case_id_key] = self.case_ids[sample_id_keys[index]] - - return result - - -class DurationLogged(object): - """ - Decorator for logging duration of a function. By default, the messages are logged by calling - "logging.info". The messages have the following format: - " TIME " - - For example, if datatype is CLIN and operation is BQ_QUERY and duration was 1.5 seconds, the message would be - "CLIN BQ_QUERY TIME 1.5". - """ - def __init__(self, datatype, operation): - """ - Constructor. - - Args: - datatype: "Datatype" field for the logged message - operation: "Operation" field for the logged message - """ - self.datatype = datatype - self.operation = operation - - def log(self, msg): - logging.info(msg) - - def __call__(self, f): - def wrapped_function(*args, **kwargs): - time_start = time() - result = f(*args, **kwargs) - time_end = time() - time_elapsed = time_end - time_start - - self.log("{dt} {op} TIME {t}".format(dt=self.datatype, - op=self.operation, - t=str(time_elapsed))) - return result - - return wrapped_function - diff --git a/cgc_api.py b/cgc_api.py deleted file mode 100644 index 8cc0ae98..00000000 --- a/cgc_api.py +++ /dev/null @@ -1,76 +0,0 @@ -""" - -Copyright 2015, Institute for Systems Biology - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -""" -import endpoints - -from api_3.isb_cgc_api.cohorts_delete import CohortsDeleteAPI -from api_3.isb_cgc_api.cohorts_get import CohortsGetAPI -from api_3.isb_cgc_api.cohorts_list import CohortsListAPI -from api_3.isb_cgc_api.cohorts_cloudstoragefilepaths import CohortsCloudStorageFilePathsAPI -from api_3.isb_cgc_api.files_get_file_paths import FilesGetPath -from api_3.isb_cgc_api.cohort_file_manifest import CohortFileManifestAPI - -from api_3.isb_cgc_api_TCGA.cohorts_preview import TCGACohortsPreviewAPI -from api_3.isb_cgc_api_TCGA.cohorts_create import TCGACohortsCreateAPI -from api_3.isb_cgc_api_TCGA.patients_get import TCGACasesGetAPI -from api_3.isb_cgc_api_TCGA.samples_get import TCGASamplesGetAPI -from api_3.isb_cgc_api_TCGA.samples_cloudstoragefilepaths import TCGASamplesCloudStorageFilePathsAPI -from api_3.isb_cgc_api_TCGA.users_get import TCGAUserGetAPI - -from api_3.isb_cgc_api_TARGET.cohorts_preview import TARGETCohortsPreviewAPI -from api_3.isb_cgc_api_TARGET.cohorts_create import TARGETCohortsCreateAPI -from api_3.isb_cgc_api_TARGET.patients_get import TARGETCasesGetAPI -from api_3.isb_cgc_api_TARGET.samples_get import TARGETSamplesGetAPI -from api_3.isb_cgc_api_TARGET.samples_cloudstoragefilepaths import TARGETSamplesCloudStorageFilePathsAPI -from api_3.isb_cgc_api_TARGET.users_get import TARGETUserGetAPI - -from api_3.isb_cgc_api_CCLE.cohorts_preview import CCLECohortsPreviewAPI -from api_3.isb_cgc_api_CCLE.cohorts_create import CCLECohortsCreateAPI -from api_3.isb_cgc_api_CCLE.patients_get import CCLECasesGetAPI -from api_3.isb_cgc_api_CCLE.samples_get import CCLESamplesGetAPI -from api_3.isb_cgc_api_CCLE.samples_cloudstoragefilepaths import CCLESamplesCloudStorageFilePathsAPI - -package = 'isb-cgc-api' - -APPLICATION = endpoints.api_server([ - CohortsDeleteAPI, - CohortsGetAPI, - CohortsListAPI, - CohortsCloudStorageFilePathsAPI, - CohortFileManifestAPI, - FilesGetPath, - - TCGACohortsPreviewAPI, - TCGACohortsCreateAPI, - TCGACasesGetAPI, - TCGASamplesGetAPI, - TCGASamplesCloudStorageFilePathsAPI, - TCGAUserGetAPI, - - TARGETCohortsPreviewAPI, - TARGETCohortsCreateAPI, - TARGETCasesGetAPI, - TARGETSamplesGetAPI, - TARGETSamplesCloudStorageFilePathsAPI, - TARGETUserGetAPI, - - CCLECohortsPreviewAPI, - CCLECohortsCreateAPI, - CCLECasesGetAPI, - CCLESamplesGetAPI, - CCLESamplesCloudStorageFilePathsAPI -]) diff --git a/requirements.txt b/requirements.txt index bacec27b..062d26a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,28 @@ +django==1.11.23 +mysqlclient==1.3.13 requests==2.20.0 ecdsa==0.13 google-api-python-client==1.6.1 httplib2==0.9.2 oauth2client==3.0.0 python-openid==2.2.5 -pyasn1==0.4.1 +pyasn1==0.4.6 requests-oauthlib==0.7.0 rsa==3.2 simplejson==3.8.1 +PyYAML==4.2b1 six==1.12.0 uritemplate==3.0.0 GoogleAppEngineCloudStorageClient==1.9.22.1 -django-dotenv==1.4.2 django-finalware==1.0.0 django-allauth==0.35.0 jsonschema==2.6.0 -enum34==1.1.6 -google-endpoints==4.8.0 +Flask==1.0.2 +flask-cors==3.0.7 +flask-talisman==0.6.0 +pyjwt==1.6.1 +cryptography==2.4.2 +future==0.17.1 +python-dotenv==0.10.0 +ruamel.yaml==0.15.94 +google-cloud-storage==1.16.1 diff --git a/scripts/isb_auth.py b/scripts/isb_auth.py new file mode 100644 index 00000000..25c23c59 --- /dev/null +++ b/scripts/isb_auth.py @@ -0,0 +1,107 @@ +''' +Copyright 2015, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Authenticates user for accessing the ISB-CGC Endpoint APIs. + +May be run from the command line or in scripts/ipython. + +The credentials file can be copied to any machine from which you want +to access the API. + +1. Command Line + python ./isb_auth.py saves the user's credentials; + OPTIONAL: + -v for verbose (returns token!) + -s FILE sets credentials file [default: ~/.isb_credentials] + -u URL-only: for use over terminal connections; + gives user a URL to paste into their browser, + and asks for an auth code in return + +2. Python + import isb_auth + isb_auth.get_credentials() + + # optional: to store credentials in a different location + from oauth2client.file import Storage + import isb_auth + import os + + storage_file = os.path.join(os.path.expanduser("~"), "{USER_CREDENTIALS_FILE_NAME}") + storage = Storage(storage_file) + isb_auth.get_credentials(storage=storage) +''' + +from argparse import ArgumentParser +import os + +from oauth2client.client import OAuth2WebServerFlow +from oauth2client import tools +from oauth2client.file import Storage + +VERBOSE = False +# for native application - same as settings.INSTALLED_APP_CLIENT_ID +CLIENT_ID = '907668440978-0ol0griu70qkeb6k3gnn2vipfa5mgl60.apps.googleusercontent.com' +# NOTE: this is NOT actually a 'secret' -- we're using the 'installed +# application' OAuth pattern here +CLIENT_SECRET = 'To_WJH7-1V-TofhNGcEqmEYi' + +EMAIL_SCOPE = 'https://www.googleapis.com/auth/userinfo.email' +DEFAULT_STORAGE_FILE = os.path.join(os.path.expanduser("~"), '.isb_credentials') + + +def maybe_print(msg): + if VERBOSE: + print(msg) + + +def get_credentials(storage=None, oauth_flow_args=[]): + noweb = '--noauth_local_webserver' + if __name__ != '__main__' and noweb not in oauth_flow_args: + oauth_flow_args.append(noweb) + if storage is None: + storage = Storage(DEFAULT_STORAGE_FILE) + credentials = storage.get() + if not credentials or credentials.invalid: + maybe_print('Credentials file was missing or invalid, kicking off OAuth2 flow...') + flow = OAuth2WebServerFlow(CLIENT_ID, CLIENT_SECRET, EMAIL_SCOPE) + flow.auth_uri = flow.auth_uri.rstrip('/') + '?approval_prompt=force' + credentials = tools.run_flow(flow, storage, tools.argparser.parse_args(oauth_flow_args)) + return credentials + + +def main(): + global VERBOSE + args = parse_args() + oauth_flow_args = [args.noauth_local_webserver] if args.noauth_local_webserver else [] + VERBOSE = args.verbose + maybe_print('--verbose: printing extra information') + storage = Storage(args.storage_file) + credentials = get_credentials(storage, oauth_flow_args) + maybe_print('credentials stored in ' + args.storage_file) + maybe_print('access_token: ' + credentials.access_token) + maybe_print('refresh_token: ' + credentials.refresh_token) + + +def parse_args(): + parser = ArgumentParser() + parser.add_argument('--storage_file', '-s', default=DEFAULT_STORAGE_FILE, help='Storage file to use for the credentials (default is {})'.format(DEFAULT_STORAGE_FILE)) + parser.add_argument('--verbose', '-v', dest='verbose', action='store_true', help='Display credentials storage location, access token, and refresh token') + parser.set_defaults(verbose=False) + parser.add_argument('--noauth_local_webserver','-u', action='store_const', const='--noauth_local_webserver') + return parser.parse_args() + + +if __name__ == '__main__': + main() diff --git a/scripts/isb_curl.py b/scripts/isb_curl.py new file mode 100644 index 00000000..b95a7163 --- /dev/null +++ b/scripts/isb_curl.py @@ -0,0 +1,123 @@ +#! /usr/bin/python2.7 +''' +Copyright 2015, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + + +isb_curl can be called by commandline or used as a library + +URL = https://isb-cgc.appspot.com/_ah/api/{API-NAME}/{VERSION}/{ENDPOINT}?{QUERYSTRING-PARAMS} + e.g. for the "cohorts_list" endpoint: + https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/cohorts_list + + +A. Command Line: + python isb_auth.py # saves the user's credentials to their root directory + python isb_curl.py URL + note: if the endpoint takes a resource in the request body, such as the save_cohort endpoint, use the following: + python isb_curl.py https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/save_cohort?name={YOUR-COHORT-NAME} \ + -d '{"Study": "BRCA"}' -H "Content-Type: application/json" + + +B. Python: + import isb_auth + import isb_curl + import requests + + url = 'https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/cohorts_list' + token = isb_curl.get_access_token() + head = {'Authorization': 'Bearer ' + token} + + # for GET requests + resp = requests.get(url, headers=head) + # querystring parameters can be added to either the url itself... + url += '?cohort_id=1' + resp = requests.get(url, headers=head) + # ... or passed in with the params kwarg + url = 'https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/cohorts_list' + params = {'cohort_id': 1} + resp = requests.get(url, headers=head, params=params) + + # if the endpoint takes a resource in the request body, such as the save_cohort endpoint... + url = https://isb-cgc.appspot.com/_ah/api/cohort_api/v1/save_cohort?name=my-new-cohort' + head.update({'Content-Type': 'application/json'}) + payload = {"SampleBarcode": "TCGA-02-0001-01C,TCGA-02-0001-10A,TCGA-01-0642-11A"} + resp = requests.post(url, headers=head, json=payload) + + # if requests version < 2.4.2 + import json + resp = requests.post(url, headers=head, data=json.dumps(payload)) + +''' + +import httplib2 +import os +import sys +from oauth2client.file import Storage +import json + +CREDENTIALS_LOC_ENV = 'ISB_CREDENTIALS' +DEFAULT_CREDENTIALS_LOC = os.path.join(os.path.expanduser("~"), '.isb_credentials') + + +def check(assertion, msg): + if not assertion: + error(msg) + + +def error(msg): + sys.stderr.write(msg + '\n') + sys.exit(1) + + +def get_credentials_location(): + credentials_location = os.environ.get(CREDENTIALS_LOC_ENV, DEFAULT_CREDENTIALS_LOC) + check(credentials_location, 'Couldn\'t locate the credentials file at {} - run isb_auth.py or check the DEFAULT_CREDENTIALS_LOC at the top of this script.'.format(credentials_location)) + return credentials_location + + +def load_credentials(credentials_location): + storage = Storage(credentials_location) + credentials = storage.get() + check(credentials and not credentials.invalid, + 'Couldn\'t locate the credentials file at {} - run isb_auth.py or check the DEFAULT_CREDENTIALS_LOC at the top of this script.'.format(credentials_location)) + return credentials + + +# Although we can use load_credentials to check the expiration (and re-up if needed), we need the +# encrypted ID token, NOT the access_token, to do a request via the ESP. To this end we load the +# file as JSON and pull the provided encrypted token there (it can also be reconstituted). +def get_id_token(credentials_location=get_credentials_location()): + credentials = load_credentials(credentials_location) + if credentials.access_token_expired: + credentials.refresh(httplib2.Http()) + creds_json = open(credentials_location, "r") + token = json.loads(creds_json.read()) + return token['token_response']['id_token'] + + +def main(): + args = sys.argv[1:] + check(args, 'usage: isb_curl.py ') + id_token = get_id_token() + curl_args = ['curl', '-H', 'Authorization: Bearer ' + id_token] + args + os.execvp('curl', curl_args) + + +# this allows us to call this from command line +if __name__ == '__main__': + main() + + diff --git a/scripts/test_cases.py b/scripts/test_cases.py new file mode 100644 index 00000000..c2b7d093 --- /dev/null +++ b/scripts/test_cases.py @@ -0,0 +1,156 @@ +from datetime import datetime + +TEST_CASES_BY_PATH = { + '/cohorts': { + 'POST': { + 'TCGA-disease-code': { + 'name': 'Test cohort TCGA-disease-code for {}'.format(datetime.now()), + 'filters': { + 'TCGA': {'disease_code': ['BRCA']} + } + }, + 'TARGET-case-barcode': { + 'name': 'Test cohort TARGET-case-barcode for {}'.format(datetime.now()), + 'filters': { + 'TARGET': {'case_barcode': ['TARGET-52-PAKHTL']} + } + }, + 'CCLE-sample-barcode': { + 'name': 'Test cohort CCLE-sample-barcode for {}'.format(datetime.now()), + 'filters': { + 'CCLE': {'sample_barcode': ['CCLE-22Rv1','CCLE-42-MG-BA','CCLE-769-P','CCLE-A-253']} + } + }, + 'TCGA-bmi-btw': { + 'name': 'Test cohort TCGA-bmi-btw for {}'.format(datetime.now()), + 'filters': { + 'TCGA': {'bmi_btw': ['15','25']} + } + }, + 'TARGET-wbc-at-diagnosis-gte': { + 'name': 'Test cohort TARGET-wbc-at-diagnosis-gte for {}'.format(datetime.now()), + 'filters': { + 'TARGET': {'wbc_at_diagnosis': ['500']} + } + } + }, + 'PATCH': { + 'rename': { + 'name': 'Renamed Cohort' + }, + 'filter_expansion': { + 'filters': { + 'disease_code': ['LUAD'] + } + } + }, + }, + '/cohorts/preview': { + 'POST': { + 'multi-program': { + "filters": { + "TCGA": { + "bmi_btw": ['15', '25'] + }, + "TARGET": { + "disease_code": "LAML" + }, + "CCLE": {} + } + } + } + }, + '/samples/{sample_barcode}': { + 'GET': { + 'TCGA-single-sample': {'sample_barcode': 'TCGA-DX-A23U-10A'}, + 'CCLE-single-sample': {'sample_barcode': 'CCLE-253J'}, + 'TARGET-single-sample': {'sample_barcode': 'TARGET-52-PAREWI-01A'} + }, + }, + '/samples': { + 'POST': { + 'TCGA-multi-sample': { + 'sample_barcodes': ['TCGA-DX-A23U-10A', 'TCGA-WK-A8XQ-10A'] + }, + 'CCLE-multi-sample': { + 'sample_barcodes': ['CCLE-8-MG-BA', 'CCLE-253J', 'CCLE-A3/KAW'] + }, + 'TARGET-multi-sample': { + 'sample_barcodes': ['TARGET-52-PAREWI-01A', 'TARGET-52-PATBLF-10A', 'TARGET-52-PATDVL-01A'] + } + } + }, + '/cases/{case_barcode}': { + 'GET': { + 'TCGA-single-case': {'case_barcode': 'TCGA-DX-A23U'}, + 'CCLE-single-case': {'case_barcode': 'A-204'}, + 'TARGET-single-case': {'case_barcode': 'TARGET-52-PAREWI'} + }, + }, + '/cases': { + 'POST': { + 'TCGA-multi-sample': { + 'case_barcodes': ['TCGA-DX-A23U', 'TCGA-WK-A8XQ'] + }, + 'CCLE-multi-sample': { + 'case_barcodes': ['A-204', '769-P', 'A3/KAW'] + }, + 'TARGET-multi-sample': { + 'case_barcodes': ['TARGET-52-PAREWI', 'TARGET-52-PATBLF', 'TARGET-52-PATDVL'] + } + } + }, + '/cohorts/{cohort_id}/file_manifest': { + 'POST': { + 'TCGA-file-size-lte': { + 'filters': { + 'program_name': ['TCGA'], + 'file_size_lte': ['5000000'] + } + }, + 'disease-code': { + 'filters': { + 'disease_code': ['BRCA'] + } + }, + } + }, + '/files/paths/{file_uuid}': { + 'GET': { + 'TARGET-file-uuid': { + 'file_uuid': '20f1cdc2-2900-4f48-9bd4-66a406bf7a61' + }, + 'TCGA-file-uuid': { + 'file_uuid': 'f7863ca3-3297-40c5-8690-f1cedb32f577' + }, + 'CCLE-file-uuid': { + 'file_uuid': '3aa3c169-7945-4ff3-9787-48270f776aa2' + } + } + + }, + '/files/paths': { + 'POST': { + 'TARGET-file-uuids': { + 'uuids': ['20f1cdc2-2900-4f48-9bd4-66a406bf7a61', '27e8a6c4-2ca7-4b7c-8f41-ec53fb4faa66', 'e3e6154c-ac76-4d0d-bd40-8dc213c35197'] + }, + 'TCGA-file-uuids': { + 'uuids': ['f7863ca3-3297-40c5-8690-f1cedb32f577', '3cfdd784-2aae-4e59-9eb6-733736b7ac37', '78b382a4-1d0e-4946-8c22-510d05dccc09'] + }, + 'CCLE-file-uuids': { + 'uuids': ['3aa3c169-7945-4ff3-9787-48270f776aa2', '6daaeb4d-d66f-4efa-b1fe-ada91f1236ba', 'e6f3246e-59f7-4c6b-90fb-88e441b48522'] + } + } + + }, + '/users/gcp/validate/{gcp_id}': { + 'GET': { + 'should-fail': { + 'gcp_id': 'gibberish_nonsense_id_sfgdfgertergdvg34t' + }, + 'should-pass': { + 'gcp_id': 'cgc-05-0016' + } + } + } +} \ No newline at end of file diff --git a/scripts/test_script.py b/scripts/test_script.py new file mode 100644 index 00000000..70cd17ed --- /dev/null +++ b/scripts/test_script.py @@ -0,0 +1,360 @@ +''' +Copyright 2019, Institute for Systems Biology + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import httplib2 +import os +import sys +import requests +import datetime +from google.cloud import storage +from oauth2client.file import Storage +import json +import copy +import ruamel.yaml + +from test_cases import TEST_CASES_BY_PATH + +CREDENTIALS_LOC_ENV = 'ISB_CREDENTIALS' +DEFAULT_CREDENTIALS_LOC = os.path.join(os.path.expanduser("~"), '.isb_credentials') + +INFO_BY_TIER = { + 'prod': { + 'yaml_path': 'web-app-deployment-files', + 'yaml': 'prod/prod.openapi-appengine.yaml', + 'project': 'isb-cgc', + 'base_uri': 'https://api-dot-isb-cgc.appspot.com' + }, + 'dev': { + 'yaml_path': 'web-app-deployment-files', + 'yaml': 'dev/dev.openapi-appengine.yaml', + 'project': 'isb-cgc', + 'base_uri': 'https://mvm-api-dot-isb-cgc.appspot.com' + }, + 'test': { + 'yaml_path': 'webapp-deployment-files-test', + 'yaml': 'test.openapi-appengine.yaml', + 'project': 'isb-cgc-test', + 'base_uri': 'https://api-dot-isb-cgc-test.appspot.com' + } +} + +# LEAVE THIS EMPTY +# This is for holding POST'd test cases which will then be used for deletions/searches +COHORTS_FOR_TESTS = [] + +# Tests which rely on results from previous tests and so must be run in a specific order +# should be listed here +TEST_DEPENDENCIES = { + 'cohorts': [ + {'{cohort_id}': [ + {'get': {'cohort_id': {'path': '/v4/cohorts/', 'method': 'post'}}}, + {'patch': {'cohort_id': {'path': '/v4/cohorts/', 'method': 'post'}}}, + ]}, + {'{cohort_id}/file_manifest': [ + {'post': {'cohort_id': {'path': '/v4/cohorts/', 'method': 'post'}}}, + {'get': {'cohort_id': {'path': '/v4/cohorts/', 'method': 'post'}}}, + ]} + ], + 'users': [ + {'gcp/{gcp_id}': [ + {'patch': {'gcp_id': {'path': '/v4/users/gcp/', 'method': 'post'}}}, + {'get': {'gcp_id': {'path': '/v4/users/gcp/', 'method': 'post'}}}, + {'delete': {'gcp_id': {'path': '/v4/users/gcp/', 'method': 'post'}}} + ]}, + ] +} + +# LEAVE THIS EMPTY +# Test results are written out here +TEST_RESULTS = {} + + +# LEAVE THIS EMPTY +# This is the holding dict into which the test cases will be built based of test case data +# and the desired tier. Structure: + +# API_PATHS = { +# : { +# : { +# : { +# 'path': +# 'methods': { +# : { +# 'auth_req': , +# ['test_data': {}] +# } +# } +# } +# } +# }, [...] +# } + +API_PATHS = { + 'test': {}, + 'dev': {}, + 'prod': {} +} + + +def load_api_paths(tier): + tier_paths = {} + tier_info = INFO_BY_TIER[tier] + client = storage.Client(project=tier_info['project']) + bucket = client.get_bucket(tier_info['yaml_path']) + yaml_blob = bucket.blob(tier_info['yaml']) + + file_name = "{}.api.yaml".format(tier) + + yaml_blob.download_to_filename(file_name) + + with open(file_name, 'r') as fpi: + yaml = ruamel.yaml.YAML(typ='safe') + + data = yaml.load(fpi) + + for path in data['paths']: + path_split = path.split('/') + resource = path_split[1] + if len(path_split) > 2: + subpath = "/".join(path_split[2:]) + else: + subpath = '/' + + if resource not in tier_paths: + tier_paths[resource] = {} + + if subpath not in tier_paths[resource]: + tier_paths[resource][subpath] = { + 'methods': {}, + 'path': path + } + + subpath_set = tier_paths[resource][subpath] + + for method in data['paths'][path]: + method_set = {} + if method == 'post' or method == 'patch' or 'parameters' in data['paths'][path][method]: + if resource in TEST_DEPENDENCIES: + dependent_subpaths = {list(x.keys())[0]: x[y] for x in TEST_DEPENDENCIES[resource] for y in x} + if subpath in dependent_subpaths: + dependent_methods = {list(y.keys())[0]: y[z] for x in dependent_subpaths for y in dependent_subpaths[x] for z in y} + if method in dependent_methods: + method_set['test_data'] = { + 'dependencies': dependent_methods[method] + } + if path in TEST_CASES_BY_PATH: + if method.upper() in TEST_CASES_BY_PATH[path]: + if 'test_data' not in method_set: + method_set['test_data'] = {} + method_set['test_data']['test_cases'] = TEST_CASES_BY_PATH[path][method.upper()] + + if 'parameters' in data['paths'][path][method]: + if 'test_data' not in method_set: + method_set['test_data'] = {} + method_set['test_data']['parameters'] = {} + for param in data['paths'][path][method]['parameters']: + if param['in'] not in method_set['test_data']['parameters']: + method_set['test_data']['parameters'][param['in']] = [] + method_set['test_data']['parameters'][param['in']].append(param['name']) + method_set['auth_req'] = bool('security' in data['paths'][path][method]) + subpath_set['methods'][method.upper()] = method_set + + API_PATHS[tier] = tier_paths + + +def run_tests_and_gather_results(tier, test_set, debug_mode=False): + TEST_RESULTS[tier] = [] + for test in test_set: + try: + path = test['path'] + uri = "{}{}".format(INFO_BY_TIER[tier]['base_uri'], path) + method = test['method'] + payload = None + test_result = None + + if 'test_data' in test: + if test['test_data'] is not None: + if 'path' in test['test_data']['parameters']: + path_params = {x: test['test_data']['values'][x] for x in test['test_data']['parameters']['path']} + uri = uri.format(**path_params) + + if 'query' in test['test_data']['parameters']: + query_string = '&'.join(["{}={}".format(x, test['test_data']['values'][x]) for x in test['test_data']['parameters']['query']]) + uri += "?{}".format(query_string) + + if 'body' in test['test_data']['parameters']: + payload = {x: test['test_data']['values'][x] for x in test['test_data']['parameters']['body']} + else: + test_result = {'result': "FAILED - test case data required but not supplied"} + + if not test_result: + test_result = { + 'path': path, + 'uri': uri, + 'method': method, + 'payload': payload + } + + headers = {'Content-type': 'application/json'} if payload else {'Content-type': 'text/plain'} + if test['auth_req']: + try: + id_token = get_id_token() + headers['Authorization'] = 'Bearer {}'.format(id_token) + except Exception as e: + print("[ERROR] While attempting to obtain login credentials: ") + print(e) + test_result['result'] = "FAILED - Authorization required but no ID token was found." + + if 'result' not in test_result: + # Requests execution + # Parse response + print("Running test {} [{}]...".format(path, method)) + if debug_mode: + print("DEBUG: request: {} <-> {} <-> {} <-> {}".format(method.upper(), uri, payload, headers)) + else: + response = requests.request(method.upper(), uri, headers=headers, json=payload) + if response.headers['Content-type'] == 'application/json': + test_result['result'] = response.json() + else: + test_result['result'] = str(response) + + TEST_RESULTS[tier].append(test_result) + + except Exception as e: + print("[ERROR] During test {}:".format(str(test))) + print(e) + print("Test may not have completed!") + + +def prepare_test_sets(tier): + reliant_tests = {} + unreliant_tests = [] + reliant_paths = None + + tier_paths = API_PATHS[tier] + + for resource in tier_paths: + + # Check to see if this resource has any reliant subpaths + if resource in TEST_DEPENDENCIES: + reliant_paths = set([y for x in TEST_DEPENDENCIES[resource] for y in x]) + + for subpath in tier_paths[resource]: + for method in tier_paths[resource][subpath]['methods']: + method_data = tier_paths[resource][subpath]['methods'][method] + # Build test + test_base = { + 'path': tier_paths[resource][subpath]['path'], + 'auth_req': method_data['auth_req'], + 'method': method.upper() + } + + tests = [] + if 'test_data' in method_data: + if 'dependencies' in method_data: + print("method_data: {}".format(str(method_data))) + else: + print(subpath, resource) + for specific_test in method_data['test_data']['test_cases']: + test = copy.deepcopy(test_base) + test['test_data'] = { + 'parameters': method_data['test_data']['parameters'], + 'values': method_data['test_data']['test_cases'][specific_test] + } + tests.append(test) + else: + tests.append(copy.deepcopy(test_base)) + + if reliant_paths and subpath in reliant_paths: + reliant_tests["{}/{}".format(resource, subpath if subpath != '/' else '')] = tests + else: + unreliant_tests.extend(tests) + + return reliant_tests, unreliant_tests + + +def print_test_run_results(tier): + for test in TEST_RESULTS[tier]: + if type(test) is str: + print(test) + else: + print("Results of test {} [{}]".format(test['path'], test['method'])) + if 'test_data' in test and test['test_data']: + print("Test data: {}".format(str(test['test_data']))) + print("Result: {}".format(str(test['result']))) + print("-----------------------------------------------------------------------------------------------------") + + +def check(assertion, msg): + if not assertion: + error(msg) + + +def error(msg): + sys.stderr.write(msg + '\n') + sys.exit(1) + + +def get_credentials_location(): + credentials_location = os.environ.get(CREDENTIALS_LOC_ENV, DEFAULT_CREDENTIALS_LOC) + check(credentials_location, 'Couldn\'t locate the credentials file at {} - run isb_auth.py or check the DEFAULT_CREDENTIALS_LOC at the top of this script.'.format(credentials_location)) + return credentials_location + + +def load_credentials(credentials_location): + storage = Storage(credentials_location) + credentials = storage.get() + check(credentials and not credentials.invalid, + 'Couldn\'t locate the credentials file at {} - run isb_auth.py or check the DEFAULT_CREDENTIALS_LOC at the top of this script.'.format(credentials_location)) + return credentials + + +# Although we can use load_credentials to check the expiration (and re-up if needed), we need the +# encrypted ID token, NOT the access_token, to do a request to Google Endpoints API. To this end we load the +# file as JSON and pull the provided encrypted token there. +def get_id_token(credentials_location=get_credentials_location()): + credentials = load_credentials(credentials_location) + if credentials.access_token_expired: + credentials.refresh(httplib2.Http()) + creds_json = open(credentials_location, "r") + token = json.loads(creds_json.read()) + return token['token_response']['id_token'] + + +def main(): + tier = 'dev' + load_api_paths(tier) + print(API_PATHS) + print("API paths loaded, ready to prepare tests") + print("----------------------------------------") + reliant, unreliant = prepare_test_sets(tier) + print("Test sets prepared, ready to run tests on {} tier".format(tier)) + print("-------------------------------------------------") + + print("UNRELIANT TESTS") + for test in unreliant: + print(test) + print("-------------------------------------------------") + + run_tests_and_gather_results(tier, unreliant, True) + + print_test_run_results(tier) + + +# this allows us to call this from command line +if __name__ == '__main__': + main() + diff --git a/settings.py b/settings.py index b7e782c7..f57a1d85 100644 --- a/settings.py +++ b/settings.py @@ -1,9 +1,12 @@ """ -Copyright 2017, Institute for Systems Biology +Copyright 2019, Institute for Systems Biology + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -12,115 +15,171 @@ """ import os +from os.path import join, dirname import sys +from pathlib import Path +from dotenv import load_dotenv +from socket import gethostname, gethostbyname -BASE_DIR = os.path.abspath(os.path.dirname(__file__)) -SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY') -DEBUG = os.environ.get('DEBUG') -TEMPLATE_DEBUG = DEBUG +env_path = '' +if os.environ.get('SECURE_LOCAL_PATH', None): + env_path += os.environ.get('SECURE_LOCAL_PATH') -ALLOWED_HOSTS = [ - os.environ.get('ALLOWED_HOST') -] +load_dotenv(dotenv_path=join(dirname(__file__), env_path+'.env')) -### Check what we're running in APP_ENGINE_FLEX = 'aef-' APP_ENGINE = 'Google App Engine/' -IS_DEV = (os.environ.get('IS_DEV', 'False') == 'True') -IS_APP_ENGINE_FLEX = os.getenv('GAE_INSTANCE', '').startswith(APP_ENGINE_FLEX) -IS_APP_ENGINE = os.getenv('SERVER_SOFTWARE', '').startswith(APP_ENGINE) -ADMINS = () -MANAGERS = ADMINS +BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + os.sep -PROJECT_ID = os.environ.get('PROJECT_ID') -BQ_PROJECT_ID = os.environ.get('BQ_PROJECT_ID') -MAX_BQ_INSERT = int(os.environ.get('MAX_BQ_INSERT', '500')) -BQ_MAX_ATTEMPTS = int(os.environ.get('MAX_BQ_ATTEMPTS', '10')) +DEBUG = (os.environ.get('DEBUG', 'False') == 'True') -USER_DATA_ON = bool(os.environ.get('USER_DATA_ON', 'False') == 'True') +print("[STATUS] DEBUG mode is "+str(DEBUG)) -MAX_FILE_LIST_REQUEST = int(os.environ.get('MAX_FILE_LIST_REQUEST', '50000')) -MAX_FILES_IGV = int(os.environ.get('MAX_FILES_IGV', '5')) +SHARED_SOURCE_DIRECTORIES = [ + 'ISB-CGC-Common' +] -BASE_URL = os.environ.get('CLOUD_BASE_URL') -BASE_API_URL = os.environ.get('CLOUD_API_URL') +# The Google AppEngine library and the Google Cloud APIs don't play nice. Teach them to get along. +# This unfortunately requires either hardcoding the path to the SDK, or sorting out a way to +# provide an environment variable indicating where it is. +# From https://github.com/GoogleCloudPlatform/python-repo-tools/blob/master/gcp_devrel/testing/appengine.py#L26 +def setup_sdk_imports(): + """Sets up appengine SDK third-party imports.""" + sdk_path = os.environ.get('GAE_SDK_PATH', '/usr/lib/google-cloud-sdk') -# Compute services -PAIRWISE_SERVICE_URL = os.environ.get('PAIRWISE_SERVICE_URL') + # Trigger loading of the Cloud APIs so they're in sys.modules + import google.cloud -# Data Buckets -OPEN_DATA_BUCKET = os.environ.get('OPEN_DATA_BUCKET') -DCC_CONTROLLED_DATA_BUCKET = os.environ.get('DCC_CONTROLLED_DATA_BUCKET') -CGHUB_CONTROLLED_DATA_BUCKET = os.environ.get('CGHUB_CONTROLLED_DATA_BUCKET') + # The libraries are specifically under platform/google_appengine + if os.path.exists(os.path.join(sdk_path, 'platform/google_appengine')): + sdk_path = os.path.join(sdk_path, 'platform/google_appengine') + + # This sets up libraries packaged with the SDK, but puts them last in + # sys.path to prevent clobbering newer versions + if 'google' in sys.modules: + sys.modules['google'].__path__.append( + os.path.join(sdk_path, 'google')) + + sys.path.append(sdk_path) + + +# Add the shared Django application subdirectory to the Python module search path +for directory_name in SHARED_SOURCE_DIRECTORIES: + sys.path.append(os.path.join(BASE_DIR, directory_name)) + +setup_sdk_imports() -GCLOUD_BUCKET = os.environ.get('GCLOUD_BUCKET') +ALLOWED_HOSTS = list(set(os.environ.get('ALLOWED_HOST', 'localhost').split(',') + ['localhost', '127.0.0.1', '[::1]', gethostname(), gethostbyname(gethostname()),])) +# Testing health checks problem +# ALLOWED_HOSTS = ['*'] + +ADMINS = () +MANAGERS = ADMINS + +LOGGER_NAME = os.environ.get('API_LOGGER_NAME', 'main_logger') + +GCLOUD_PROJECT_ID = os.environ.get('GCLOUD_PROJECT_ID', '') +GCLOUD_PROJECT_NUMBER = os.environ.get('GCLOUD_PROJECT_NUMBER', '') +BIGQUERY_PROJECT_ID = os.environ.get('BIGQUERY_PROJECT_ID', GCLOUD_PROJECT_ID) +BIGQUERY_DATASET_V1 = os.environ.get('BIGQUERY_DATASET_V1', '') +BIGQUERY_DATA_PROJECT_ID = os.environ.get('BIGQUERY_DATA_PROJECT_ID', GCLOUD_PROJECT_ID) + +# Deployment module +CRON_MODULE = os.environ.get('CRON_MODULE') + +# Log Names +SERVICE_ACCOUNT_LOG_NAME = os.environ.get('SERVICE_ACCOUNT_LOG_NAME', 'local_dev_logging') +WEBAPP_LOGIN_LOG_NAME = os.environ.get('WEBAPP_LOGIN_LOG_NAME', 'local_dev_logging') +GCP_ACTIVITY_LOG_NAME = os.environ.get('GCP_ACTIVITY_LOG_NAME', 'local_dev_logging') + +BASE_URL = os.environ.get('BASE_URL', 'https://isb-cgc.appspot.com') +BASE_API_URL = os.environ.get('BASE_API_URL', 'https://api-dot-isb-cgc.appspot.com') + +# Data Buckets +OPEN_DATA_BUCKET = os.environ.get('OPEN_DATA_BUCKET', '') +DCC_CONTROLLED_DATA_BUCKET = os.environ.get('DCC_CONTROLLED_DATA_BUCKET', '') +CGHUB_CONTROLLED_DATA_BUCKET = os.environ.get('CGHUB_CONTROLLED_DATA_BUCKET', '') +GCLOUD_BUCKET = os.environ.get('GOOGLE_STORAGE_BUCKET') # BigQuery cohort storage settings -COHORT_DATASET_ID = os.environ.get('COHORT_DATASET_ID') -DEVELOPER_COHORT_TABLE_ID = os.environ.get('DEVELOPER_COHORT_TABLE_ID') +BIGQUERY_COHORT_DATASET_ID = os.environ.get('BIGQUERY_COHORT_DATASET_ID', 'cohort_dataset') +BIGQUERY_COHORT_TABLE_ID = os.environ.get('BIGQUERY_COHORT_TABLE_ID', 'developer_cohorts') +BIGQUERY_COSMIC_DATASET_ID = os.environ.get('BIGQUERY_COSMIC_DATASET_ID', '') +BIGQUERY_CGC_TABLE_ID = os.environ.get('BIGQUERY_CGC_TABLE_ID', '') +MAX_BQ_INSERT = int(os.environ.get('MAX_BQ_INSERT', '500')) -NIH_AUTH_ON = os.environ.get('NIH_AUTH_ON', False) +USER_DATA_ON = bool(os.environ.get('USER_DATA_ON', False)) DATABASES = { 'default': { 'ENGINE': os.environ.get('DATABASE_ENGINE', 'django.db.backends.mysql'), 'HOST': os.environ.get('DATABASE_HOST', '127.0.0.1'), - 'NAME': os.environ.get('DATABASE_NAME', 'test'), + 'NAME': os.environ.get('DATABASE_NAME', ''), 'USER': os.environ.get('DATABASE_USER'), 'PASSWORD': os.environ.get('DATABASE_PASSWORD') } } -if os.environ.has_key('DB_SSL_CERT'): - DATABASES['default']['OPTIONS'] = { - 'ssl': { - 'ca': os.environ.get('DB_SSL_CA'), - 'cert': os.environ.get('DB_SSL_CERT'), - 'key': os.environ.get('DB_SSL_KEY') - } - } - DB_SOCKET = DATABASES['default']['HOST'] if 'cloudsql' in DATABASES['default']['HOST'] else None +CONN_MAX_AGE = 60 + +IS_DEV = (os.environ.get('IS_DEV', 'False') == 'True') +IS_APP_ENGINE_FLEX = os.getenv('GAE_INSTANCE', '').startswith(APP_ENGINE_FLEX) +IS_APP_ENGINE = os.getenv('SERVER_SOFTWARE', '').startswith(APP_ENGINE) + # Default to localhost for the site ID SITE_ID = 3 if IS_APP_ENGINE_FLEX or IS_APP_ENGINE: - print >> sys.stdout, "[STATUS] AppEngine detected." + print("[STATUS] AppEngine detected - swapping in live site ID.") SITE_ID = 4 -# For running local unit tests for models -if 'test' in sys.argv: - DATABASES = os.environ.get('TEST_DATABASE') def get_project_identifier(): - return BQ_PROJECT_ID - -BIGQUERY_DATASET = os.environ.get('BIGQUERY_DATASET') + return BIGQUERY_PROJECT_ID -def get_bigquery_dataset(): - return BIGQUERY_DATASET - -PROJECT_NAME = os.environ.get('PROJECT_NAME') -BIGQUERY_PROJECT_NAME = os.environ.get('BIGQUERY_PROJECT_NAME') - -def get_bigquery_project_name(): - return BIGQUERY_PROJECT_NAME # Set cohort table here -if DEVELOPER_COHORT_TABLE_ID is None: +if BIGQUERY_COHORT_TABLE_ID is None: raise Exception("Developer-specific cohort table ID is not set.") +BQ_MAX_ATTEMPTS = int(os.environ.get('BQ_MAX_ATTEMPTS', '10')) + + +# TODO Remove duplicate class. +# +# This class is retained here, as it is required by bq_data_access/v1. +# bq_data_access/v2 uses the class from the bq_data_access/bigquery_cohorts module. class BigQueryCohortStorageSettings(object): def __init__(self, dataset_id, table_id): self.dataset_id = dataset_id self.table_id = table_id + def GET_BQ_COHORT_SETTINGS(): - return BigQueryCohortStorageSettings(COHORT_DATASET_ID, DEVELOPER_COHORT_TABLE_ID) + return BigQueryCohortStorageSettings(BIGQUERY_COHORT_DATASET_ID, BIGQUERY_COHORT_TABLE_ID) + +USE_CLOUD_STORAGE = os.environ.get('USE_CLOUD_STORAGE', False) + +PROCESSING_ENABLED = os.environ.get('PROCESSING_ENABLED', False) +PROCESSING_JENKINS_URL = os.environ.get('PROCESSING_JENKINS_URL', 'http://localhost/jenkins') +PROCESSING_JENKINS_PROJECT = os.environ.get('PROCESSING_JENKINS_PROJECT', 'cgc-processing') +PROCESSING_JENKINS_USER = os.environ.get('PROCESSING_JENKINS_USER', 'user') +PROCESSING_JENKINS_PASSWORD = os.environ.get('PROCESSING_JENKINS_PASSWORD', '') -USE_CLOUD_STORAGE = os.environ.get('USE_CLOUD_STORAGE') +SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + +CSRF_COOKIE_SECURE = bool(os.environ.get('CSRF_COOKIE_SECURE', False)) +SESSION_COOKIE_SECURE = bool(os.environ.get('SESSION_COOKIE_SECURE', False)) +SECURE_SSL_REDIRECT = bool(os.environ.get('SECURE_SSL_REDIRECT', False)) + +SECURE_REDIRECT_EXEMPT = [] + +if SECURE_SSL_REDIRECT: + # Exempt the health check so it can go through + SECURE_REDIRECT_EXEMPT = [r'^_ah/(vm_)?health$', ] # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name @@ -146,6 +205,17 @@ def GET_BQ_COHORT_SETTINGS(): # If you set this to False, Django will not use timezone-aware datetimes. USE_TZ = True +# Absolute filesystem path to the directory that will hold user-uploaded files. +# Example: "/home/media/media.lawrence.com/media/" +MEDIA_FOLDER = os.environ.get('MEDIA_FOLDER', 'uploads/') +MEDIA_ROOT = os.path.join(os.path.dirname(__file__), '..', '..', MEDIA_FOLDER) +MEDIA_ROOT = os.path.normpath(MEDIA_ROOT) + +# URL that handles the media served from MEDIA_ROOT. Make sure to use a +# trailing slash. +# Examples: "http://media.lawrence.com/media/", "http://example.com/media/" +MEDIA_URL = '' + # Absolute path to the directory static files should be collected to. # Don't put anything in this directory yourself; store your static files # in apps' "static/" subdirectories and in STATICFILES_DIRS. @@ -156,6 +226,8 @@ def GET_BQ_COHORT_SETTINGS(): # Example: "http://media.lawrence.com/static/" STATIC_URL = os.environ.get('STATIC_URL', '/static/') +GCS_STORAGE_URI = os.environ.get('GCS_STORAGE_URI', 'https://storage.googleapis.com/') + # Additional locations of static files STATICFILES_DIRS = ( # Put strings here, like "/home/html/static" or "C:/www/django/static". @@ -172,22 +244,24 @@ def GET_BQ_COHORT_SETTINGS(): ) # Make this unique, and don't share it with anybody. -# SECRET_KEY = os.environ.get('SECRET_KEY') +SECRET_KEY = os.environ.get('DJANGO_SECRET_KEY', '') + +SECURE_HSTS_INCLUDE_SUBDOMAINS = (os.environ.get('SECURE_HSTS_INCLUDE_SUBDOMAINS','True') == 'True') +SECURE_HSTS_PRELOAD = (os.environ.get('SECURE_HSTS_PRELOAD','True') == 'True') +SECURE_HSTS_SECONDS = int(os.environ.get('SECURE_HSTS_SECONDS','3600')) -MIDDLEWARE_CLASSES = ( - # For using NDB with Django - # documentation: https://cloud.google.com/appengine/docs/python/ndb/#integration - 'google.appengine.ext.ndb.django_middleware.NdbDjangoMiddleware', - 'google.appengine.ext.appstats.recording.AppStatsDjangoMiddleware', +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', + 'GenespotRE.checkreqsize_middleware.CheckReqSize', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'adminrestrict.middleware.AdminPagesRestrictMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', # Uncomment the next line for simple clickjacking protection: - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - -) + 'django.middleware.clickjacking.XFrameOptionsMiddleware' +] ROOT_URLCONF = 'GenespotRE.urls' @@ -201,32 +275,10 @@ def GET_BQ_COHORT_SETTINGS(): 'django.contrib.sites', 'django.contrib.messages', 'django.contrib.staticfiles', - # 'django.contrib.admin', - # 'django.contrib.admindocs', - # 'GenespotRE', - # 'visualizations', - # 'seqpeek', 'sharing', 'cohorts', 'projects', - # 'genes', - # 'variables', - # 'workbooks', - 'data_upload' -) - -############################# -# django-session-security # -############################# - -# testing "session security works at the moment" commit -# INSTALLED_APPS += ('session_security',) -# SESSION_SECURITY_WARN_AFTER = 540 -# SESSION_SECURITY_EXPIRE_AFTER = 600 -SESSION_EXPIRE_AT_BROWSER_CLOSE = True -MIDDLEWARE_CLASSES += ( - # for django-session-security -- must go *after* AuthenticationMiddleware - # 'session_security.middleware.SessionSecurityMiddleware', + 'data_upload', ) ############################### @@ -289,14 +341,29 @@ def GET_BQ_COHORT_SETTINGS(): 'level': 'DEBUG', 'propagate': True, }, - } + 'allauth': { + 'handlers': ['console_dev', 'console_prod'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'google_helpers': { + 'handlers': ['console_dev', 'console_prod'], + 'level': 'DEBUG', + 'propagate': True, + }, + 'data_upload': { + 'handlers': ['console_dev', 'console_prod'], + 'level': 'DEBUG', + 'propagate': True, + }, + }, } ########################## # Start django-allauth # ########################## -LOGIN_REDIRECT_URL = '/dashboard/' +LOGIN_REDIRECT_URL = '/extended_login/' INSTALLED_APPS += ( 'accounts', @@ -318,7 +385,6 @@ def GET_BQ_COHORT_SETTINGS(): 'OPTIONS': { # add any context processors here 'context_processors': ( - 'allauth.socialaccount.context_processors.socialaccount', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', @@ -351,16 +417,24 @@ def GET_BQ_COHORT_SETTINGS(): } } +# Trying to force allauth to only use https +ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'https' +# ...but not if this is a local dev build +if IS_DEV: + ACCOUNT_DEFAULT_HTTP_PROTOCOL = 'http' + ########################## # End django-allauth # ########################## -GOOGLE_APPLICATION_CREDENTIALS = os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') -CLIENT_SECRETS = os.environ.get('CLIENT_SECRETS') -CLIENT_EMAIL = os.environ.get('CLIENT_EMAIL') -WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID') -INSTALLED_APP_CLIENT_ID = os.environ.get('INSTALLED_APP_CLIENT_ID') +GOOGLE_APPLICATION_CREDENTIALS = os.path.join(os.path.dirname(__file__), os.environ.get('GOOGLE_APPLICATION_CREDENTIALS')) if os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') else '' +CLIENT_SECRETS = os.path.join(os.path.dirname(__file__), os.environ.get('CLIENT_SECRETS')) if os.environ.get('CLIENT_SECRETS') else '' +WEB_CLIENT_ID = os.environ.get('WEB_CLIENT_ID', '') # Client ID from client_secrets.json +API_CLIENT_ID = os.environ.get('API_CLIENT_ID', '') # Client ID for the API +IGV_WEB_CLIENT_ID = os.environ.get('IGV_WEB_CLIENT_ID', WEB_CLIENT_ID) +INSTALLED_APP_CLIENT_ID = os.environ.get('INSTALLED_APP_CLIENT_ID', '') # Native Client ID +GCP_REG_CLIENT_EMAIL = os.environ.get('CLIENT_EMAIL','') ################################# # For NIH/eRA Commons login # @@ -370,34 +444,6 @@ def GET_BQ_COHORT_SETTINGS(): OPEN_ACL_GOOGLE_GROUP = os.environ.get('OPEN_ACL_GOOGLE_GROUP', '') GOOGLE_GROUP_ADMIN = os.environ.get('GOOGLE_GROUP_ADMIN', '') SUPERADMIN_FOR_REPORTS = os.environ.get('SUPERADMIN_FOR_REPORTS', '') -ERA_LOGIN_URL = os.environ.get('ERA_LOGIN_URL', '') -SAML_FOLDER = os.environ.get('SAML_FOLDER', '') - -###################################### -# For directory, reports services # -###################################### -GOOGLE_GROUP_ADMIN = os.environ.get('GOOGLE_GROUP_ADMIN', '') -SUPERADMIN_FOR_REPORTS = os.environ.get('SUPERADMIN_FOR_REPORTS', '') - -############################## -# Start django-finalware # -############################## - -INSTALLED_APPS += ( - 'finalware',) - -SITE_SUPERUSER_USERNAME = os.environ.get('SU_USER') -SITE_SUPERUSER_EMAIL = '' -SITE_SUPERUSER_PASSWORD = os.environ.get('SU_PASS') - -############################ -# End django-finalware # -############################ - -CONN_MAX_AGE = 0 - -# Deployment module -CRON_MODULE = os.environ.get('CRON_MODULE') # TaskQueue used when users go through the ERA flow LOGOUT_WORKER_TASKQUEUE = os.environ.get('LOGOUT_WORKER_TASKQUEUE', '') @@ -415,9 +461,6 @@ def GET_BQ_COHORT_SETTINGS(): # Log name for ERA login views LOG_NAME_ERA_LOGIN_VIEW = os.environ.get('LOG_NAME_ERA_LOGIN_VIEW', '') -# Log Names -SERVICE_ACCOUNT_LOG_NAME = os.environ.get('SERVICE_ACCOUNT_LOG_NAME', 'local_dev_logging') - # Service account blacklist file path SERVICE_ACCOUNT_BLACKLIST_PATH = os.environ.get('SERVICE_ACCOUNT_BLACKLIST_PATH', '') @@ -430,9 +473,15 @@ def GET_BQ_COHORT_SETTINGS(): # Dataset configuration file path DATASET_CONFIGURATION_PATH = os.environ.get('DATASET_CONFIGURATION_PATH', '') +# DCF Phase I enable flag +DCF_TEST = bool(os.environ.get('DCF_TEST', 'False') == 'True') + # SA via DCF SA_VIA_DCF = bool(os.environ.get('SA_VIA_DCF', 'False') == 'True') +# DCF Monitoring SA +DCF_MONITORING_SA = os.environ.get('DCF_MONITORING_SA', '') + ################################# # For DCF login # ################################# @@ -451,4 +500,45 @@ def GET_BQ_COHORT_SETTINGS(): DCF_GOOGLE_SA_MONITOR_URL = os.environ.get('DCF_GOOGLE_SA_MONITOR_URL', '') DCF_GOOGLE_SA_URL = os.environ.get('DCF_GOOGLE_SA_URL', '') DCF_TOKEN_REFRESH_WINDOW_SECONDS = int(os.environ.get('DCF_TOKEN_REFRESH_WINDOW_SECONDS', 86400)) -DCF_LOGIN_EXPIRATION_SECONDS = int(os.environ.get('DCF_LOGIN_EXPIRATION_SECONDS', 86400)) \ No newline at end of file +DCF_LOGIN_EXPIRATION_SECONDS = int(os.environ.get('DCF_LOGIN_EXPIRATION_SECONDS', 86400)) + +############################## +# Start django-finalware # +############################## + +INSTALLED_APPS += ( + 'finalware',) + +SITE_SUPERUSER_USERNAME = os.environ.get('SUPERUSER_USERNAME', '') +SITE_SUPERUSER_EMAIL = '' +SITE_SUPERUSER_PASSWORD = os.environ.get('SUPERUSER_PASSWORD', '') + +############################ +# End django-finalware # +############################ + +############################################################## +# MAXes to prevent size-limited events from causing errors +############################################################## + +# Google App Engine has a response size limit of 32M. ~65k entries from the cohort_filelist view will +# equal just under the 32M limit. If each individual listing is ever lengthened or shortened this +# number should be adjusted +MAX_FILE_LIST_REQUEST = 65000 + +# IGV limit to prevent users from trying ot open dozens of files +MAX_FILES_IGV = 5 + +# Rough max file size to allow for eg. barcode list upload, to revent triggering RequestDataTooBig +FILE_SIZE_UPLOAD_MAX = 1950000 + +############################################################## +# MailGun Email Settings +############################################################## + +EMAIL_SERVICE_API_URL = os.environ.get('EMAIL_SERVICE_API_URL', '') +EMAIL_SERVICE_API_KEY = os.environ.get('EMAIL_SERVICE_API_KEY', '') +NOTIFICATION_EMAIL_FROM_ADDRESS = os.environ.get('NOTIFICATOON_EMAIL_FROM_ADDRESS', '') + +# Explicitly check for known items +BLACKLIST_RE = r'((?i)|!\[\]|!!\[\]|\[\]\[\".*\"\]|(?i))' diff --git a/shell/gcloud-pull-staging-files.sh b/shell/gcloud-pull-staging-files.sh index dc70067c..b4698611 100644 --- a/shell/gcloud-pull-staging-files.sh +++ b/shell/gcloud-pull-staging-files.sh @@ -1,24 +1,19 @@ mkdir ./json mkdir ./txt -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${PROD_APP_YAML}" ./app.yaml -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${PROD_SECRETS_FILE}" ./client_secrets.json -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${PROD_JSON_FILE}" ./privatekey.json +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${ENV_FILE}" ./.env +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${API_RUNTIME_SA_KEY}" ./privatekey.json +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${OPEN_API_YAML}" ./openapi-appengine.yaml +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${OPEN_API_YAML}" ./apiv4/api.yaml +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${API_APP_YAML}" ./app.yaml +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${USER_GCP_KEY}" ./ +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${DCF_SECRETS_FILE}" ./dcf_secrets.txt -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${PROD_DATASET_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${SERVICE_ACCOUNT_BLACKLIST_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${GOOGLE_ORG_WHITELIST_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${MANAGED_SERVICE_ACCOUNTS_JSON_FILE}" ./ -./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/${PROD_DATASET_JSON_FILE}" ./ - -if [ -n "${PROD_NIH_AUTH_ON}" ]; then - ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/saml/advanced_settings.json" ./saml/advanced_settings.json - ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/saml/settings.json" ./saml/settings.json - ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/saml/certs/cert.pem" ./saml/certs/cert.pem - ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/saml/certs/key.pem" ./saml/certs/key.pem - ./google-cloud-sdk/bin/gsutil cp "gs://${GCLOUD_BUCKET}/NIH_FTP.txt" ./NIH_FTP.txt -fi +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${SERVICE_ACCOUNT_BLACKLIST_JSON_FILE}" ./ +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${GOOGLE_ORG_WHITELIST_JSON_FILE}" ./ +gsutil cp "gs://${DEPLOYMENT_BUCKET}/${MANAGED_SERVICE_ACCOUNTS_JSON_FILE}" ./ # Pack staged files for caching +echo "Packing JSON and text files for caching into deployment..." cp --verbose *.json ./json -cp --verbose *.txt ./txt +cp --verbose *.txt ./txt \ No newline at end of file diff --git a/shell/gcloud_authenticate.sh b/shell/gcloud_authenticate.sh index 56792252..f405b950 100644 --- a/shell/gcloud_authenticate.sh +++ b/shell/gcloud_authenticate.sh @@ -1,5 +1,11 @@ -echo $DEV_PRIVATE_KEY_FOR_V2 | base64 --decode --ignore-garbage > deployment.key.json +if [ ! -f "deployment.key.json" ]; then + echo ${DEPLOYMENT_KEY} | base64 --decode --ignore-garbage > deployment.key.json +else + echo "Found deployment key JSON file." +fi -/home/circleci/${CIRCLE_PROJECT_REPONAME}/google-cloud-sdk/bin/gcloud auth activate-service-account --key-file deployment.key.json -/home/circleci/${CIRCLE_PROJECT_REPONAME}/google-cloud-sdk/bin/gcloud config set account $GAE_CLIENT_EMAIL -/home/circleci/${CIRCLE_PROJECT_REPONAME}/google-cloud-sdk/bin/gcloud config set project "$GAE_PROJECT_ID" +gcloud auth activate-service-account --key-file deployment.key.json +echo "Setting deployment client email to ${DEPLOYMENT_CLIENT_EMAIL}" +gcloud config set account $DEPLOYMENT_CLIENT_EMAIL +echo "Setting deployment project to ${DEPLOYMENT_PROJECT_ID}" +gcloud config set project "$DEPLOYMENT_PROJECT_ID" \ No newline at end of file diff --git a/shell/install-deps.sh b/shell/install-deps.sh index f2d9c0c0..96e8b50b 100644 --- a/shell/install-deps.sh +++ b/shell/install-deps.sh @@ -2,68 +2,80 @@ if [ -n "$CI" ]; then export HOME=/home/circleci/${CIRCLE_PROJECT_REPONAME} export HOMEROOT=/home/circleci/${CIRCLE_PROJECT_REPONAME} # Clone dependencies - git clone -b isb-cgc-prod https://github.com/isb-cgc/ISB-CGC-Common.git + COMMON_BRANCH=master + if [[ ${CIRCLE_BRANCH} =~ isb-cgc-(prod|uat|test).* ]]; then + COMMON_BRANCH=$(awk -F- '{print $1"-"$2"-"$3}' <<< ${CIRCLE_BRANCH}) + fi + echo "Cloning ISB-CGC-Common branch ${COMMON_BRANCH}..." + git clone -b ${COMMON_BRANCH} https://github.com/isb-cgc/ISB-CGC-Common.git else - export $(cat /home/vagrant/www/.env | grep -v ^# | xargs) 2> /dev/null + export $(cat /home/vagrant/API/.env | grep -v ^# | xargs) 2> /dev/null export HOME=/home/vagrant - export HOMEROOT=/home/vagrant/www + export HOMEROOT=/home/vagrant/API fi +# Remove .pyc files; these can sometimes stick around and if a +# model has changed names it will cause various load failures +find . -type f -name '*.pyc' -delete + +export DEBIAN_FRONTEND=noninteractive + # Install and update apt-get info echo "Preparing System..." apt-get -y install software-properties-common if [ -n "$CI" ]; then - # CI Takes care of Python update - apt-get update -qq -else - # Add apt-get repository to update python from 2.7.6 (default) to latest 2.7.x - add-apt-repository -y ppa:fkrull/deadsnakes-python2.7 - apt-get update -qq - apt-get install -qq -y python2.7 + # Use these next 4 lines to update mysql public build key + echo 'download mysql public build key' + wget -O - -q 'https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x8C718D3B5072E1F5' | grep -v '>' | grep -v '<' | grep -v '{' > mysql_pubkey.asc + apt-key add mysql_pubkey.asc || exit 1 + echo 'mysql build key update done.' + wget https://dev.mysql.com/get/mysql-apt-config_0.8.9-1_all.deb + apt-get install -y lsb-release + dpkg -i mysql-apt-config_0.8.9-1_all.deb fi +apt-get update -qq + # Install apt-get dependencies echo "Installing Dependencies..." -apt-get install -qq -y unzip libffi-dev libssl-dev mysql-client libmysqlclient-dev python2.7-dev git -echo "Dependencies Installed " +if [ -n "$CI" ]; then +apt-get install -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl dos2unix python3.5 +apt-get install -y --force-yes mysql-client +else + apt-get install -qq -y --force-yes unzip libffi-dev libssl-dev libmysqlclient-dev python3-mysqldb python3-dev libpython3-dev git ruby g++ curl dos2unix python3.5 mysql-client-5.7 +fi +echo "Dependencies Installed" + +# If this is local development, clean out lib for a re-structuring +if [ -z "${CI}" ]; then + # Clean out lib to prevent confusion over multiple builds in local development + # and prep for local install + echo "Emptying out ${HOMEROOT}/lib/ ..." +fi # Install PIP + Dependencies -echo "Installing PIP..." -curl --silent https://bootstrap.pypa.io/get-pip.py | python -echo "...PIP installed." +echo "Installing pip3..." +curl --silent https://bootstrap.pypa.io/get-pip.py | python3 # Install our primary python libraries # If we're not on CircleCI, or we are but the lib directory isn't there (cache miss), install lib if [ -z "${CI}" ] || [ ! -d "lib" ]; then echo "Installing Python Libraries..." - pip install -q -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all + pip3 install -r ${HOMEROOT}/requirements.txt -t ${HOMEROOT}/lib --upgrade --only-binary all else echo "Using restored cache for Python Libraries" fi -# Install Google App Engine -# If we're not on CircleCI or we are but google_appengine isn't there, install it -if [ -z "${CI}" ] || [ ! -d "google_appengine" ]; then - echo "Installing Google App Engine..." - wget https://storage.googleapis.com/appengine-sdks/featured/google_appengine_1.9.69.zip -O ${HOME}/google_appengine.zip - unzip -n -qq ${HOME}/google_appengine.zip -d $HOME - export PATH=$PATH:${HOME}/google_appengine/ - echo "Google App Engine Installed" -else - echo "Using restored cache for Google App Engine. " -fi - # Install Google Cloud SDK # If we're not on CircleCI or we are but google-cloud-sdk isn't there, install it -if [ -z "${CI}" ] || [ ! -d "google-cloud-sdk" ]; then +if [ -z "${CI}" ] || [ ! -d "/usr/lib/google-cloud-sdk" ]; then echo "Installing Google Cloud SDK..." export CLOUDSDK_CORE_DISABLE_PROMPTS=1 - curl https://sdk.cloud.google.com | bash - export PATH=$PATH:${HOME}/google-cloud-sdk/bin - echo 'export PATH=$PATH:${HOME}/google-cloud-sdk/bin' | tee -a ${HOME}/.bash_profile + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + apt-get install apt-transport-https ca-certificates + curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + apt-get update && apt-get -y --allow-downgrades install google-cloud-sdk=251.0.0-0 + apt-get -y --allow-downgrades install google-cloud-sdk-app-engine-python=251.0.0-0 echo "Google Cloud SDK Installed" -else - echo "Using restored cache for Google Cloud SDK." fi - diff --git a/shell/pull_config.sh b/shell/pull_config.sh new file mode 100644 index 00000000..8980d728 --- /dev/null +++ b/shell/pull_config.sh @@ -0,0 +1,10 @@ +if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt" ]; then + gsutil cp gs://${DEPLOYMENT_BUCKET}/deployment_config.txt /home/circleci/${CIRCLE_PROJECT_REPONAME}/ + chmod ugo+r /home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt + if [ ! -f "/home/circleci/${CIRCLE_PROJECT_REPONAME}/deployment_config.txt" ]; then + echo "[ERROR] Couldn't assign deployment configuration file - exiting." + exit 1 + fi +else + echo "Found deployment configuration file." +fi \ No newline at end of file diff --git a/shell/python-su.sh b/shell/python-su.sh new file mode 100644 index 00000000..8390271a --- /dev/null +++ b/shell/python-su.sh @@ -0,0 +1 @@ +sudo PYTHONPATH="$PYTHONPATH" /usr/bin/python3 "$@" \ No newline at end of file diff --git a/shell/vagrant-set-env.sh b/shell/vagrant-set-env.sh new file mode 100644 index 00000000..a88b6c18 --- /dev/null +++ b/shell/vagrant-set-env.sh @@ -0,0 +1,2 @@ +echo 'export PYTHONPATH=/home/vagrant/API:/home/vagrant/google_appengine:/home/vagrant/google_appengine/lib/protorpc-1.0:/home/vagrant/API/lib' | tee -a /home/vagrant/.bash_profile +chmod +x /home/vagrant/API/shell/python-su.sh \ No newline at end of file diff --git a/shell/vagrant-start-server.sh b/shell/vagrant-start-server.sh new file mode 100644 index 00000000..860fe309 --- /dev/null +++ b/shell/vagrant-start-server.sh @@ -0,0 +1 @@ +export $(cat /home/vagrant/parentDir/secure_files/.env | grep -v ^# | xargs) 2> /dev/null diff --git a/static/favicon-16x16.png b/static/favicon-16x16.png new file mode 100644 index 00000000..8b194e61 Binary files /dev/null and b/static/favicon-16x16.png differ diff --git a/static/favicon-32x32.png b/static/favicon-32x32.png new file mode 100644 index 00000000..249737fe Binary files /dev/null and b/static/favicon-32x32.png differ diff --git a/static/swagger-ui-bundle.js b/static/swagger-ui-bundle.js new file mode 100644 index 00000000..55e2f50c --- /dev/null +++ b/static/swagger-ui-bundle.js @@ -0,0 +1,93 @@ +!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t():"function"==typeof define&&define.amd?define([],t):"object"==typeof exports?exports.SwaggerUIBundle=t():e.SwaggerUIBundle=t()}(this,function(){return function(e){var t={};function n(r){if(t[r])return t[r].exports;var o=t[r]={i:r,l:!1,exports:{}};return e[r].call(o.exports,o,o.exports,n),o.l=!0,o.exports}return n.m=e,n.c=t,n.d=function(e,t,r){n.o(e,t)||Object.defineProperty(e,t,{configurable:!1,enumerable:!0,get:r})},n.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return n.d(t,"a",t),t},n.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},n.p="/dist",n(n.s=446)}([function(e,t,n){"use strict";e.exports=n(75)},function(e,t,n){e.exports=n(854)()},function(e,t,n){"use strict";t.__esModule=!0,t.default=function(e,t){if(!(e instanceof t))throw new TypeError("Cannot call a class as a function")}},function(e,t,n){"use strict";t.__esModule=!0;var r,o=n(263),i=(r=o)&&r.__esModule?r:{default:r};t.default=function(){function e(e,t){for(var n=0;n>>0;if(""+n!==t||4294967295===n)return NaN;t=n}return t<0?C(e)+t:t}function A(){return!0}function O(e,t,n){return(0===e||void 0!==n&&e<=-n)&&(void 0===t||void 0!==n&&t>=n)}function P(e,t){return M(e,t,0)}function T(e,t){return M(e,t,t)}function M(e,t,n){return void 0===e?n:e<0?Math.max(0,t+e):void 0===t?e:Math.min(t,e)}var I=0,j=1,N=2,R="function"==typeof Symbol&&Symbol.iterator,D="@@iterator",L=R||D;function U(e){this.next=e}function q(e,t,n,r){var o=0===e?t:1===e?n:[t,n];return r?r.value=o:r={value:o,done:!1},r}function F(){return{value:void 0,done:!0}}function z(e){return!!H(e)}function B(e){return e&&"function"==typeof e.next}function V(e){var t=H(e);return t&&t.call(e)}function H(e){var t=e&&(R&&e[R]||e[D]);if("function"==typeof t)return t}function W(e){return e&&"number"==typeof e.length}function J(e){return null===e||void 0===e?ie():a(e)?e.toSeq():function(e){var t=se(e)||"object"==typeof e&&new te(e);if(!t)throw new TypeError("Expected Array or iterable object of values, or keyed object: "+e);return t}(e)}function Y(e){return null===e||void 0===e?ie().toKeyedSeq():a(e)?u(e)?e.toSeq():e.fromEntrySeq():ae(e)}function K(e){return null===e||void 0===e?ie():a(e)?u(e)?e.entrySeq():e.toIndexedSeq():ue(e)}function G(e){return(null===e||void 0===e?ie():a(e)?u(e)?e.entrySeq():e:ue(e)).toSetSeq()}U.prototype.toString=function(){return"[Iterator]"},U.KEYS=I,U.VALUES=j,U.ENTRIES=N,U.prototype.inspect=U.prototype.toSource=function(){return this.toString()},U.prototype[L]=function(){return this},t(J,n),J.of=function(){return J(arguments)},J.prototype.toSeq=function(){return this},J.prototype.toString=function(){return this.__toString("Seq {","}")},J.prototype.cacheResult=function(){return!this._cache&&this.__iterateUncached&&(this._cache=this.entrySeq().toArray(),this.size=this._cache.length),this},J.prototype.__iterate=function(e,t){return le(this,e,t,!0)},J.prototype.__iterator=function(e,t){return ce(this,e,t,!0)},t(Y,J),Y.prototype.toKeyedSeq=function(){return this},t(K,J),K.of=function(){return K(arguments)},K.prototype.toIndexedSeq=function(){return this},K.prototype.toString=function(){return this.__toString("Seq [","]")},K.prototype.__iterate=function(e,t){return le(this,e,t,!1)},K.prototype.__iterator=function(e,t){return ce(this,e,t,!1)},t(G,J),G.of=function(){return G(arguments)},G.prototype.toSetSeq=function(){return this},J.isSeq=oe,J.Keyed=Y,J.Set=G,J.Indexed=K;var $,Z,X,Q="@@__IMMUTABLE_SEQ__@@";function ee(e){this._array=e,this.size=e.length}function te(e){var t=Object.keys(e);this._object=e,this._keys=t,this.size=t.length}function ne(e){this._iterable=e,this.size=e.length||e.size}function re(e){this._iterator=e,this._iteratorCache=[]}function oe(e){return!(!e||!e[Q])}function ie(){return $||($=new ee([]))}function ae(e){var t=Array.isArray(e)?new ee(e).fromEntrySeq():B(e)?new re(e).fromEntrySeq():z(e)?new ne(e).fromEntrySeq():"object"==typeof e?new te(e):void 0;if(!t)throw new TypeError("Expected Array or iterable object of [k, v] entries, or keyed object: "+e);return t}function ue(e){var t=se(e);if(!t)throw new TypeError("Expected Array or iterable object of values: "+e);return t}function se(e){return W(e)?new ee(e):B(e)?new re(e):z(e)?new ne(e):void 0}function le(e,t,n,r){var o=e._cache;if(o){for(var i=o.length-1,a=0;a<=i;a++){var u=o[n?i-a:a];if(!1===t(u[1],r?u[0]:a,e))return a+1}return a}return e.__iterateUncached(t,n)}function ce(e,t,n,r){var o=e._cache;if(o){var i=o.length-1,a=0;return new U(function(){var e=o[n?i-a:a];return a++>i?{value:void 0,done:!0}:q(t,r?e[0]:a-1,e[1])})}return e.__iteratorUncached(t,n)}function fe(e,t){return t?function e(t,n,r,o){if(Array.isArray(n))return t.call(o,r,K(n).map(function(r,o){return e(t,r,o,n)}));if(de(n))return t.call(o,r,Y(n).map(function(r,o){return e(t,r,o,n)}));return n}(t,e,"",{"":e}):pe(e)}function pe(e){return Array.isArray(e)?K(e).map(pe).toList():de(e)?Y(e).map(pe).toMap():e}function de(e){return e&&(e.constructor===Object||void 0===e.constructor)}function he(e,t){if(e===t||e!=e&&t!=t)return!0;if(!e||!t)return!1;if("function"==typeof e.valueOf&&"function"==typeof t.valueOf){if((e=e.valueOf())===(t=t.valueOf())||e!=e&&t!=t)return!0;if(!e||!t)return!1}return!("function"!=typeof e.equals||"function"!=typeof t.equals||!e.equals(t))}function ve(e,t){if(e===t)return!0;if(!a(t)||void 0!==e.size&&void 0!==t.size&&e.size!==t.size||void 0!==e.__hash&&void 0!==t.__hash&&e.__hash!==t.__hash||u(e)!==u(t)||s(e)!==s(t)||c(e)!==c(t))return!1;if(0===e.size&&0===t.size)return!0;var n=!l(e);if(c(e)){var r=e.entries();return t.every(function(e,t){var o=r.next().value;return o&&he(o[1],e)&&(n||he(o[0],t))})&&r.next().done}var o=!1;if(void 0===e.size)if(void 0===t.size)"function"==typeof e.cacheResult&&e.cacheResult();else{o=!0;var i=e;e=t,t=i}var f=!0,p=t.__iterate(function(t,r){if(n?!e.has(t):o?!he(t,e.get(r,y)):!he(e.get(r,y),t))return f=!1,!1});return f&&e.size===p}function me(e,t){if(!(this instanceof me))return new me(e,t);if(this._value=e,this.size=void 0===t?1/0:Math.max(0,t),0===this.size){if(Z)return Z;Z=this}}function ge(e,t){if(!e)throw new Error(t)}function ye(e,t,n){if(!(this instanceof ye))return new ye(e,t,n);if(ge(0!==n,"Cannot step a Range by 0"),e=e||0,void 0===t&&(t=1/0),n=void 0===n?1:Math.abs(n),tr?{value:void 0,done:!0}:q(e,o,n[t?r-o++:o++])})},t(te,Y),te.prototype.get=function(e,t){return void 0===t||this.has(e)?this._object[e]:t},te.prototype.has=function(e){return this._object.hasOwnProperty(e)},te.prototype.__iterate=function(e,t){for(var n=this._object,r=this._keys,o=r.length-1,i=0;i<=o;i++){var a=r[t?o-i:i];if(!1===e(n[a],a,this))return i+1}return i},te.prototype.__iterator=function(e,t){var n=this._object,r=this._keys,o=r.length-1,i=0;return new U(function(){var a=r[t?o-i:i];return i++>o?{value:void 0,done:!0}:q(e,a,n[a])})},te.prototype[h]=!0,t(ne,K),ne.prototype.__iterateUncached=function(e,t){if(t)return this.cacheResult().__iterate(e,t);var n=V(this._iterable),r=0;if(B(n))for(var o;!(o=n.next()).done&&!1!==e(o.value,r++,this););return r},ne.prototype.__iteratorUncached=function(e,t){if(t)return this.cacheResult().__iterator(e,t);var n=V(this._iterable);if(!B(n))return new U(F);var r=0;return new U(function(){var t=n.next();return t.done?t:q(e,r++,t.value)})},t(re,K),re.prototype.__iterateUncached=function(e,t){if(t)return this.cacheResult().__iterate(e,t);for(var n,r=this._iterator,o=this._iteratorCache,i=0;i=r.length){var t=n.next();if(t.done)return t;r[o]=t.value}return q(e,o,r[o++])})},t(me,K),me.prototype.toString=function(){return 0===this.size?"Repeat []":"Repeat [ "+this._value+" "+this.size+" times ]"},me.prototype.get=function(e,t){return this.has(e)?this._value:t},me.prototype.includes=function(e){return he(this._value,e)},me.prototype.slice=function(e,t){var n=this.size;return O(e,t,n)?this:new me(this._value,T(t,n)-P(e,n))},me.prototype.reverse=function(){return this},me.prototype.indexOf=function(e){return he(this._value,e)?0:-1},me.prototype.lastIndexOf=function(e){return he(this._value,e)?this.size:-1},me.prototype.__iterate=function(e,t){for(var n=0;n=0&&t=0&&nn?{value:void 0,done:!0}:q(e,i++,a)})},ye.prototype.equals=function(e){return e instanceof ye?this._start===e._start&&this._end===e._end&&this._step===e._step:ve(this,e)},t(be,n),t(_e,be),t(we,be),t(Ee,be),be.Keyed=_e,be.Indexed=we,be.Set=Ee;var xe="function"==typeof Math.imul&&-2===Math.imul(4294967295,2)?Math.imul:function(e,t){var n=65535&(e|=0),r=65535&(t|=0);return n*r+((e>>>16)*r+n*(t>>>16)<<16>>>0)|0};function Se(e){return e>>>1&1073741824|3221225471&e}function Ce(e){if(!1===e||null===e||void 0===e)return 0;if("function"==typeof e.valueOf&&(!1===(e=e.valueOf())||null===e||void 0===e))return 0;if(!0===e)return 1;var t=typeof e;if("number"===t){if(e!=e||e===1/0)return 0;var n=0|e;for(n!==e&&(n^=4294967295*e);e>4294967295;)n^=e/=4294967295;return Se(n)}if("string"===t)return e.length>je?function(e){var t=De[e];void 0===t&&(t=ke(e),Re===Ne&&(Re=0,De={}),Re++,De[e]=t);return t}(e):ke(e);if("function"==typeof e.hashCode)return e.hashCode();if("object"===t)return function(e){var t;if(Te&&void 0!==(t=Pe.get(e)))return t;if(void 0!==(t=e[Ie]))return t;if(!Oe){if(void 0!==(t=e.propertyIsEnumerable&&e.propertyIsEnumerable[Ie]))return t;if(void 0!==(t=function(e){if(e&&e.nodeType>0)switch(e.nodeType){case 1:return e.uniqueID;case 9:return e.documentElement&&e.documentElement.uniqueID}}(e)))return t}t=++Me,1073741824&Me&&(Me=0);if(Te)Pe.set(e,t);else{if(void 0!==Ae&&!1===Ae(e))throw new Error("Non-extensible objects are not allowed as keys.");if(Oe)Object.defineProperty(e,Ie,{enumerable:!1,configurable:!1,writable:!1,value:t});else if(void 0!==e.propertyIsEnumerable&&e.propertyIsEnumerable===e.constructor.prototype.propertyIsEnumerable)e.propertyIsEnumerable=function(){return this.constructor.prototype.propertyIsEnumerable.apply(this,arguments)},e.propertyIsEnumerable[Ie]=t;else{if(void 0===e.nodeType)throw new Error("Unable to set a non-enumerable property on object.");e[Ie]=t}}return t}(e);if("function"==typeof e.toString)return ke(e.toString());throw new Error("Value type "+t+" cannot be hashed.")}function ke(e){for(var t=0,n=0;n=t.length)throw new Error("Missing value for key: "+t[n]);e.set(t[n],t[n+1])}})},Ue.prototype.toString=function(){return this.__toString("Map {","}")},Ue.prototype.get=function(e,t){return this._root?this._root.get(0,void 0,e,t):t},Ue.prototype.set=function(e,t){return Qe(this,e,t)},Ue.prototype.setIn=function(e,t){return this.updateIn(e,y,function(){return t})},Ue.prototype.remove=function(e){return Qe(this,e,y)},Ue.prototype.deleteIn=function(e){return this.updateIn(e,function(){return y})},Ue.prototype.update=function(e,t,n){return 1===arguments.length?e(this):this.updateIn([e],t,n)},Ue.prototype.updateIn=function(e,t,n){n||(n=t,t=void 0);var r=function e(t,n,r,o){var i=t===y;var a=n.next();if(a.done){var u=i?r:t,s=o(u);return s===u?t:s}ge(i||t&&t.set,"invalid keyPath");var l=a.value;var c=i?y:t.get(l,y);var f=e(c,n,r,o);return f===c?t:f===y?t.remove(l):(i?Xe():t).set(l,f)}(this,nn(e),t,n);return r===y?void 0:r},Ue.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=0,this._root=null,this.__hash=void 0,this.__altered=!0,this):Xe()},Ue.prototype.merge=function(){return rt(this,void 0,arguments)},Ue.prototype.mergeWith=function(t){return rt(this,t,e.call(arguments,1))},Ue.prototype.mergeIn=function(t){var n=e.call(arguments,1);return this.updateIn(t,Xe(),function(e){return"function"==typeof e.merge?e.merge.apply(e,n):n[n.length-1]})},Ue.prototype.mergeDeep=function(){return rt(this,ot,arguments)},Ue.prototype.mergeDeepWith=function(t){var n=e.call(arguments,1);return rt(this,it(t),n)},Ue.prototype.mergeDeepIn=function(t){var n=e.call(arguments,1);return this.updateIn(t,Xe(),function(e){return"function"==typeof e.mergeDeep?e.mergeDeep.apply(e,n):n[n.length-1]})},Ue.prototype.sort=function(e){return Pt(Wt(this,e))},Ue.prototype.sortBy=function(e,t){return Pt(Wt(this,t,e))},Ue.prototype.withMutations=function(e){var t=this.asMutable();return e(t),t.wasAltered()?t.__ensureOwner(this.__ownerID):this},Ue.prototype.asMutable=function(){return this.__ownerID?this:this.__ensureOwner(new x)},Ue.prototype.asImmutable=function(){return this.__ensureOwner()},Ue.prototype.wasAltered=function(){return this.__altered},Ue.prototype.__iterator=function(e,t){return new Ke(this,e,t)},Ue.prototype.__iterate=function(e,t){var n=this,r=0;return this._root&&this._root.iterate(function(t){return r++,e(t[1],t[0],n)},t),r},Ue.prototype.__ensureOwner=function(e){return e===this.__ownerID?this:e?Ze(this.size,this._root,e,this.__hash):(this.__ownerID=e,this.__altered=!1,this)},Ue.isMap=qe;var Fe,ze="@@__IMMUTABLE_MAP__@@",Be=Ue.prototype;function Ve(e,t){this.ownerID=e,this.entries=t}function He(e,t,n){this.ownerID=e,this.bitmap=t,this.nodes=n}function We(e,t,n){this.ownerID=e,this.count=t,this.nodes=n}function Je(e,t,n){this.ownerID=e,this.keyHash=t,this.entries=n}function Ye(e,t,n){this.ownerID=e,this.keyHash=t,this.entry=n}function Ke(e,t,n){this._type=t,this._reverse=n,this._stack=e._root&&$e(e._root)}function Ge(e,t){return q(e,t[0],t[1])}function $e(e,t){return{node:e,index:0,__prev:t}}function Ze(e,t,n,r){var o=Object.create(Be);return o.size=e,o._root=t,o.__ownerID=n,o.__hash=r,o.__altered=!1,o}function Xe(){return Fe||(Fe=Ze(0))}function Qe(e,t,n){var r,o;if(e._root){var i=w(b),a=w(_);if(r=et(e._root,e.__ownerID,0,void 0,t,n,i,a),!a.value)return e;o=e.size+(i.value?n===y?-1:1:0)}else{if(n===y)return e;o=1,r=new Ve(e.__ownerID,[[t,n]])}return e.__ownerID?(e.size=o,e._root=r,e.__hash=void 0,e.__altered=!0,e):r?Ze(o,r):Xe()}function et(e,t,n,r,o,i,a,u){return e?e.update(t,n,r,o,i,a,u):i===y?e:(E(u),E(a),new Ye(t,r,[o,i]))}function tt(e){return e.constructor===Ye||e.constructor===Je}function nt(e,t,n,r,o){if(e.keyHash===r)return new Je(t,r,[e.entry,o]);var i,a=(0===n?e.keyHash:e.keyHash>>>n)&g,u=(0===n?r:r>>>n)&g;return new He(t,1<>1&1431655765))+(e>>2&858993459))+(e>>4)&252645135,e+=e>>8,127&(e+=e>>16)}function st(e,t,n,r){var o=r?e:S(e);return o[t]=n,o}Be[ze]=!0,Be.delete=Be.remove,Be.removeIn=Be.deleteIn,Ve.prototype.get=function(e,t,n,r){for(var o=this.entries,i=0,a=o.length;i=lt)return function(e,t,n,r){e||(e=new x);for(var o=new Ye(e,Ce(n),[n,r]),i=0;i>>e)&g),i=this.bitmap;return 0==(i&o)?r:this.nodes[ut(i&o-1)].get(e+v,t,n,r)},He.prototype.update=function(e,t,n,r,o,i,a){void 0===n&&(n=Ce(r));var u=(0===t?n:n>>>t)&g,s=1<=ct)return function(e,t,n,r,o){for(var i=0,a=new Array(m),u=0;0!==n;u++,n>>>=1)a[u]=1&n?t[i++]:void 0;return a[r]=o,new We(e,i+1,a)}(e,p,l,u,h);if(c&&!h&&2===p.length&&tt(p[1^f]))return p[1^f];if(c&&h&&1===p.length&&tt(h))return h;var b=e&&e===this.ownerID,_=c?h?l:l^s:l|s,w=c?h?st(p,f,h,b):function(e,t,n){var r=e.length-1;if(n&&t===r)return e.pop(),e;for(var o=new Array(r),i=0,a=0;a>>e)&g,i=this.nodes[o];return i?i.get(e+v,t,n,r):r},We.prototype.update=function(e,t,n,r,o,i,a){void 0===n&&(n=Ce(r));var u=(0===t?n:n>>>t)&g,s=o===y,l=this.nodes,c=l[u];if(s&&!c)return this;var f=et(c,e,t+v,n,r,o,i,a);if(f===c)return this;var p=this.count;if(c){if(!f&&--p0&&r=0&&e=e.size||t<0)return e.withMutations(function(e){t<0?kt(e,t).set(0,n):kt(e,0,t+1).set(t,n)});t+=e._origin;var r=e._tail,o=e._root,i=w(_);t>=Ot(e._capacity)?r=xt(r,e.__ownerID,0,t,n,i):o=xt(o,e.__ownerID,e._level,t,n,i);if(!i.value)return e;if(e.__ownerID)return e._root=o,e._tail=r,e.__hash=void 0,e.__altered=!0,e;return wt(e._origin,e._capacity,e._level,o,r)}(this,e,t)},pt.prototype.remove=function(e){return this.has(e)?0===e?this.shift():e===this.size-1?this.pop():this.splice(e,1):this},pt.prototype.insert=function(e,t){return this.splice(e,0,t)},pt.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=this._origin=this._capacity=0,this._level=v,this._root=this._tail=null,this.__hash=void 0,this.__altered=!0,this):Et()},pt.prototype.push=function(){var e=arguments,t=this.size;return this.withMutations(function(n){kt(n,0,t+e.length);for(var r=0;r>>t&g;if(r>=this.array.length)return new mt([],e);var o,i=0===r;if(t>0){var a=this.array[r];if((o=a&&a.removeBefore(e,t-v,n))===a&&i)return this}if(i&&!o)return this;var u=St(this,e);if(!i)for(var s=0;s>>t&g;if(o>=this.array.length)return this;if(t>0){var i=this.array[o];if((r=i&&i.removeAfter(e,t-v,n))===i&&o===this.array.length-1)return this}var a=St(this,e);return a.array.splice(o+1),r&&(a.array[o]=r),a};var gt,yt,bt={};function _t(e,t){var n=e._origin,r=e._capacity,o=Ot(r),i=e._tail;return a(e._root,e._level,0);function a(e,u,s){return 0===u?function(e,a){var u=a===o?i&&i.array:e&&e.array,s=a>n?0:n-a,l=r-a;l>m&&(l=m);return function(){if(s===l)return bt;var e=t?--l:s++;return u&&u[e]}}(e,s):function(e,o,i){var u,s=e&&e.array,l=i>n?0:n-i>>o,c=1+(r-i>>o);c>m&&(c=m);return function(){for(;;){if(u){var e=u();if(e!==bt)return e;u=null}if(l===c)return bt;var n=t?--c:l++;u=a(s&&s[n],o-v,i+(n<>>n&g,s=e&&u0){var l=e&&e.array[u],c=xt(l,t,n-v,r,o,i);return c===l?e:((a=St(e,t)).array[u]=c,a)}return s&&e.array[u]===o?e:(E(i),a=St(e,t),void 0===o&&u===a.array.length-1?a.array.pop():a.array[u]=o,a)}function St(e,t){return t&&e&&t===e.ownerID?e:new mt(e?e.array.slice():[],t)}function Ct(e,t){if(t>=Ot(e._capacity))return e._tail;if(t<1<0;)n=n.array[t>>>r&g],r-=v;return n}}function kt(e,t,n){void 0!==t&&(t|=0),void 0!==n&&(n|=0);var r=e.__ownerID||new x,o=e._origin,i=e._capacity,a=o+t,u=void 0===n?i:n<0?i+n:o+n;if(a===o&&u===i)return e;if(a>=u)return e.clear();for(var s=e._level,l=e._root,c=0;a+c<0;)l=new mt(l&&l.array.length?[void 0,l]:[],r),c+=1<<(s+=v);c&&(a+=c,o+=c,u+=c,i+=c);for(var f=Ot(i),p=Ot(u);p>=1<f?new mt([],r):d;if(d&&p>f&&av;y-=v){var b=f>>>y&g;m=m.array[b]=St(m.array[b],r)}m.array[f>>>v&g]=d}if(u=p)a-=p,u-=p,s=v,l=null,h=h&&h.removeBefore(r,0,a);else if(a>o||p>>s&g;if(_!==p>>>s&g)break;_&&(c+=(1<o&&(l=l.removeBefore(r,s,a-c)),l&&pi&&(i=l.size),a(s)||(l=l.map(function(e){return fe(e)})),r.push(l)}return i>e.size&&(e=e.setSize(i)),at(e,t,r)}function Ot(e){return e>>v<=m&&a.size>=2*i.size?(r=(o=a.filter(function(e,t){return void 0!==e&&u!==t})).toKeyedSeq().map(function(e){return e[0]}).flip().toMap(),e.__ownerID&&(r.__ownerID=o.__ownerID=e.__ownerID)):(r=i.remove(t),o=u===a.size-1?a.pop():a.set(u,void 0))}else if(s){if(n===a.get(u)[1])return e;r=i,o=a.set(u,[t,n])}else r=i.set(t,a.size),o=a.set(a.size,[t,n]);return e.__ownerID?(e.size=r.size,e._map=r,e._list=o,e.__hash=void 0,e):Mt(r,o)}function Nt(e,t){this._iter=e,this._useKeys=t,this.size=e.size}function Rt(e){this._iter=e,this.size=e.size}function Dt(e){this._iter=e,this.size=e.size}function Lt(e){this._iter=e,this.size=e.size}function Ut(e){var t=Qt(e);return t._iter=e,t.size=e.size,t.flip=function(){return e},t.reverse=function(){var t=e.reverse.apply(this);return t.flip=function(){return e.reverse()},t},t.has=function(t){return e.includes(t)},t.includes=function(t){return e.has(t)},t.cacheResult=en,t.__iterateUncached=function(t,n){var r=this;return e.__iterate(function(e,n){return!1!==t(n,e,r)},n)},t.__iteratorUncached=function(t,n){if(t===N){var r=e.__iterator(t,n);return new U(function(){var e=r.next();if(!e.done){var t=e.value[0];e.value[0]=e.value[1],e.value[1]=t}return e})}return e.__iterator(t===j?I:j,n)},t}function qt(e,t,n){var r=Qt(e);return r.size=e.size,r.has=function(t){return e.has(t)},r.get=function(r,o){var i=e.get(r,y);return i===y?o:t.call(n,i,r,e)},r.__iterateUncached=function(r,o){var i=this;return e.__iterate(function(e,o,a){return!1!==r(t.call(n,e,o,a),o,i)},o)},r.__iteratorUncached=function(r,o){var i=e.__iterator(N,o);return new U(function(){var o=i.next();if(o.done)return o;var a=o.value,u=a[0];return q(r,u,t.call(n,a[1],u,e),o)})},r}function Ft(e,t){var n=Qt(e);return n._iter=e,n.size=e.size,n.reverse=function(){return e},e.flip&&(n.flip=function(){var t=Ut(e);return t.reverse=function(){return e.flip()},t}),n.get=function(n,r){return e.get(t?n:-1-n,r)},n.has=function(n){return e.has(t?n:-1-n)},n.includes=function(t){return e.includes(t)},n.cacheResult=en,n.__iterate=function(t,n){var r=this;return e.__iterate(function(e,n){return t(e,n,r)},!n)},n.__iterator=function(t,n){return e.__iterator(t,!n)},n}function zt(e,t,n,r){var o=Qt(e);return r&&(o.has=function(r){var o=e.get(r,y);return o!==y&&!!t.call(n,o,r,e)},o.get=function(r,o){var i=e.get(r,y);return i!==y&&t.call(n,i,r,e)?i:o}),o.__iterateUncached=function(o,i){var a=this,u=0;return e.__iterate(function(e,i,s){if(t.call(n,e,i,s))return u++,o(e,r?i:u-1,a)},i),u},o.__iteratorUncached=function(o,i){var a=e.__iterator(N,i),u=0;return new U(function(){for(;;){var i=a.next();if(i.done)return i;var s=i.value,l=s[0],c=s[1];if(t.call(n,c,l,e))return q(o,r?l:u++,c,i)}})},o}function Bt(e,t,n,r){var o=e.size;if(void 0!==t&&(t|=0),void 0!==n&&(n===1/0?n=o:n|=0),O(t,n,o))return e;var i=P(t,o),a=T(n,o);if(i!=i||a!=a)return Bt(e.toSeq().cacheResult(),t,n,r);var u,s=a-i;s==s&&(u=s<0?0:s);var l=Qt(e);return l.size=0===u?u:e.size&&u||void 0,!r&&oe(e)&&u>=0&&(l.get=function(t,n){return(t=k(this,t))>=0&&tu)return{value:void 0,done:!0};var e=o.next();return r||t===j?e:q(t,s-1,t===I?void 0:e.value[1],e)})},l}function Vt(e,t,n,r){var o=Qt(e);return o.__iterateUncached=function(o,i){var a=this;if(i)return this.cacheResult().__iterate(o,i);var u=!0,s=0;return e.__iterate(function(e,i,l){if(!u||!(u=t.call(n,e,i,l)))return s++,o(e,r?i:s-1,a)}),s},o.__iteratorUncached=function(o,i){var a=this;if(i)return this.cacheResult().__iterator(o,i);var u=e.__iterator(N,i),s=!0,l=0;return new U(function(){var e,i,c;do{if((e=u.next()).done)return r||o===j?e:q(o,l++,o===I?void 0:e.value[1],e);var f=e.value;i=f[0],c=f[1],s&&(s=t.call(n,c,i,a))}while(s);return o===N?e:q(o,i,c,e)})},o}function Ht(e,t,n){var r=Qt(e);return r.__iterateUncached=function(r,o){var i=0,u=!1;return function e(s,l){var c=this;s.__iterate(function(o,s){return(!t||l0}function Kt(e,t,r){var o=Qt(e);return o.size=new ee(r).map(function(e){return e.size}).min(),o.__iterate=function(e,t){for(var n,r=this.__iterator(j,t),o=0;!(n=r.next()).done&&!1!==e(n.value,o++,this););return o},o.__iteratorUncached=function(e,o){var i=r.map(function(e){return e=n(e),V(o?e.reverse():e)}),a=0,u=!1;return new U(function(){var n;return u||(n=i.map(function(e){return e.next()}),u=n.some(function(e){return e.done})),u?{value:void 0,done:!0}:q(e,a++,t.apply(null,n.map(function(e){return e.value})))})},o}function Gt(e,t){return oe(e)?t:e.constructor(t)}function $t(e){if(e!==Object(e))throw new TypeError("Expected [K, V] tuple: "+e)}function Zt(e){return Le(e.size),C(e)}function Xt(e){return u(e)?r:s(e)?o:i}function Qt(e){return Object.create((u(e)?Y:s(e)?K:G).prototype)}function en(){return this._iter.cacheResult?(this._iter.cacheResult(),this.size=this._iter.size,this):J.prototype.cacheResult.call(this)}function tn(e,t){return e>t?1:e=0;n--)t={value:arguments[n],next:t};return this.__ownerID?(this.size=e,this._head=t,this.__hash=void 0,this.__altered=!0,this):An(e,t)},En.prototype.pushAll=function(e){if(0===(e=o(e)).size)return this;Le(e.size);var t=this.size,n=this._head;return e.reverse().forEach(function(e){t++,n={value:e,next:n}}),this.__ownerID?(this.size=t,this._head=n,this.__hash=void 0,this.__altered=!0,this):An(t,n)},En.prototype.pop=function(){return this.slice(1)},En.prototype.unshift=function(){return this.push.apply(this,arguments)},En.prototype.unshiftAll=function(e){return this.pushAll(e)},En.prototype.shift=function(){return this.pop.apply(this,arguments)},En.prototype.clear=function(){return 0===this.size?this:this.__ownerID?(this.size=0,this._head=void 0,this.__hash=void 0,this.__altered=!0,this):On()},En.prototype.slice=function(e,t){if(O(e,t,this.size))return this;var n=P(e,this.size);if(T(t,this.size)!==this.size)return we.prototype.slice.call(this,e,t);for(var r=this.size-n,o=this._head;n--;)o=o.next;return this.__ownerID?(this.size=r,this._head=o,this.__hash=void 0,this.__altered=!0,this):An(r,o)},En.prototype.__ensureOwner=function(e){return e===this.__ownerID?this:e?An(this.size,this._head,e,this.__hash):(this.__ownerID=e,this.__altered=!1,this)},En.prototype.__iterate=function(e,t){if(t)return this.reverse().__iterate(e);for(var n=0,r=this._head;r&&!1!==e(r.value,n++,this);)r=r.next;return n},En.prototype.__iterator=function(e,t){if(t)return this.reverse().__iterator(e);var n=0,r=this._head;return new U(function(){if(r){var t=r.value;return r=r.next,q(e,n++,t)}return{value:void 0,done:!0}})},En.isStack=xn;var Sn,Cn="@@__IMMUTABLE_STACK__@@",kn=En.prototype;function An(e,t,n,r){var o=Object.create(kn);return o.size=e,o._head=t,o.__ownerID=n,o.__hash=r,o.__altered=!1,o}function On(){return Sn||(Sn=An(0))}function Pn(e,t){var n=function(n){e.prototype[n]=t[n]};return Object.keys(t).forEach(n),Object.getOwnPropertySymbols&&Object.getOwnPropertySymbols(t).forEach(n),e}kn[Cn]=!0,kn.withMutations=Be.withMutations,kn.asMutable=Be.asMutable,kn.asImmutable=Be.asImmutable,kn.wasAltered=Be.wasAltered,n.Iterator=U,Pn(n,{toArray:function(){Le(this.size);var e=new Array(this.size||0);return this.valueSeq().__iterate(function(t,n){e[n]=t}),e},toIndexedSeq:function(){return new Rt(this)},toJS:function(){return this.toSeq().map(function(e){return e&&"function"==typeof e.toJS?e.toJS():e}).__toJS()},toJSON:function(){return this.toSeq().map(function(e){return e&&"function"==typeof e.toJSON?e.toJSON():e}).__toJS()},toKeyedSeq:function(){return new Nt(this,!0)},toMap:function(){return Ue(this.toKeyedSeq())},toObject:function(){Le(this.size);var e={};return this.__iterate(function(t,n){e[n]=t}),e},toOrderedMap:function(){return Pt(this.toKeyedSeq())},toOrderedSet:function(){return mn(u(this)?this.valueSeq():this)},toSet:function(){return sn(u(this)?this.valueSeq():this)},toSetSeq:function(){return new Dt(this)},toSeq:function(){return s(this)?this.toIndexedSeq():u(this)?this.toKeyedSeq():this.toSetSeq()},toStack:function(){return En(u(this)?this.valueSeq():this)},toList:function(){return pt(u(this)?this.valueSeq():this)},toString:function(){return"[Iterable]"},__toString:function(e,t){return 0===this.size?e+t:e+" "+this.toSeq().map(this.__toStringMapper).join(", ")+" "+t},concat:function(){return Gt(this,function(e,t){var n=u(e),o=[e].concat(t).map(function(e){return a(e)?n&&(e=r(e)):e=n?ae(e):ue(Array.isArray(e)?e:[e]),e}).filter(function(e){return 0!==e.size});if(0===o.length)return e;if(1===o.length){var i=o[0];if(i===e||n&&u(i)||s(e)&&s(i))return i}var l=new ee(o);return n?l=l.toKeyedSeq():s(e)||(l=l.toSetSeq()),(l=l.flatten(!0)).size=o.reduce(function(e,t){if(void 0!==e){var n=t.size;if(void 0!==n)return e+n}},0),l}(this,e.call(arguments,0)))},includes:function(e){return this.some(function(t){return he(t,e)})},entries:function(){return this.__iterator(N)},every:function(e,t){Le(this.size);var n=!0;return this.__iterate(function(r,o,i){if(!e.call(t,r,o,i))return n=!1,!1}),n},filter:function(e,t){return Gt(this,zt(this,e,t,!0))},find:function(e,t,n){var r=this.findEntry(e,t);return r?r[1]:n},forEach:function(e,t){return Le(this.size),this.__iterate(t?e.bind(t):e)},join:function(e){Le(this.size),e=void 0!==e?""+e:",";var t="",n=!0;return this.__iterate(function(r){n?n=!1:t+=e,t+=null!==r&&void 0!==r?r.toString():""}),t},keys:function(){return this.__iterator(I)},map:function(e,t){return Gt(this,qt(this,e,t))},reduce:function(e,t,n){var r,o;return Le(this.size),arguments.length<2?o=!0:r=t,this.__iterate(function(t,i,a){o?(o=!1,r=t):r=e.call(n,r,t,i,a)}),r},reduceRight:function(e,t,n){var r=this.toKeyedSeq().reverse();return r.reduce.apply(r,arguments)},reverse:function(){return Gt(this,Ft(this,!0))},slice:function(e,t){return Gt(this,Bt(this,e,t,!0))},some:function(e,t){return!this.every(Nn(e),t)},sort:function(e){return Gt(this,Wt(this,e))},values:function(){return this.__iterator(j)},butLast:function(){return this.slice(0,-1)},isEmpty:function(){return void 0!==this.size?0===this.size:!this.some(function(){return!0})},count:function(e,t){return C(e?this.toSeq().filter(e,t):this)},countBy:function(e,t){return function(e,t,n){var r=Ue().asMutable();return e.__iterate(function(o,i){r.update(t.call(n,o,i,e),0,function(e){return e+1})}),r.asImmutable()}(this,e,t)},equals:function(e){return ve(this,e)},entrySeq:function(){var e=this;if(e._cache)return new ee(e._cache);var t=e.toSeq().map(jn).toIndexedSeq();return t.fromEntrySeq=function(){return e.toSeq()},t},filterNot:function(e,t){return this.filter(Nn(e),t)},findEntry:function(e,t,n){var r=n;return this.__iterate(function(n,o,i){if(e.call(t,n,o,i))return r=[o,n],!1}),r},findKey:function(e,t){var n=this.findEntry(e,t);return n&&n[0]},findLast:function(e,t,n){return this.toKeyedSeq().reverse().find(e,t,n)},findLastEntry:function(e,t,n){return this.toKeyedSeq().reverse().findEntry(e,t,n)},findLastKey:function(e,t){return this.toKeyedSeq().reverse().findKey(e,t)},first:function(){return this.find(A)},flatMap:function(e,t){return Gt(this,function(e,t,n){var r=Xt(e);return e.toSeq().map(function(o,i){return r(t.call(n,o,i,e))}).flatten(!0)}(this,e,t))},flatten:function(e){return Gt(this,Ht(this,e,!0))},fromEntrySeq:function(){return new Lt(this)},get:function(e,t){return this.find(function(t,n){return he(n,e)},void 0,t)},getIn:function(e,t){for(var n,r=this,o=nn(e);!(n=o.next()).done;){var i=n.value;if((r=r&&r.get?r.get(i,y):y)===y)return t}return r},groupBy:function(e,t){return function(e,t,n){var r=u(e),o=(c(e)?Pt():Ue()).asMutable();e.__iterate(function(i,a){o.update(t.call(n,i,a,e),function(e){return(e=e||[]).push(r?[a,i]:i),e})});var i=Xt(e);return o.map(function(t){return Gt(e,i(t))})}(this,e,t)},has:function(e){return this.get(e,y)!==y},hasIn:function(e){return this.getIn(e,y)!==y},isSubset:function(e){return e="function"==typeof e.includes?e:n(e),this.every(function(t){return e.includes(t)})},isSuperset:function(e){return(e="function"==typeof e.isSubset?e:n(e)).isSubset(this)},keyOf:function(e){return this.findKey(function(t){return he(t,e)})},keySeq:function(){return this.toSeq().map(In).toIndexedSeq()},last:function(){return this.toSeq().reverse().first()},lastKeyOf:function(e){return this.toKeyedSeq().reverse().keyOf(e)},max:function(e){return Jt(this,e)},maxBy:function(e,t){return Jt(this,t,e)},min:function(e){return Jt(this,e?Rn(e):Un)},minBy:function(e,t){return Jt(this,t?Rn(t):Un,e)},rest:function(){return this.slice(1)},skip:function(e){return this.slice(Math.max(0,e))},skipLast:function(e){return Gt(this,this.toSeq().reverse().skip(e).reverse())},skipWhile:function(e,t){return Gt(this,Vt(this,e,t,!0))},skipUntil:function(e,t){return this.skipWhile(Nn(e),t)},sortBy:function(e,t){return Gt(this,Wt(this,t,e))},take:function(e){return this.slice(0,Math.max(0,e))},takeLast:function(e){return Gt(this,this.toSeq().reverse().take(e).reverse())},takeWhile:function(e,t){return Gt(this,function(e,t,n){var r=Qt(e);return r.__iterateUncached=function(r,o){var i=this;if(o)return this.cacheResult().__iterate(r,o);var a=0;return e.__iterate(function(e,o,u){return t.call(n,e,o,u)&&++a&&r(e,o,i)}),a},r.__iteratorUncached=function(r,o){var i=this;if(o)return this.cacheResult().__iterator(r,o);var a=e.__iterator(N,o),u=!0;return new U(function(){if(!u)return{value:void 0,done:!0};var e=a.next();if(e.done)return e;var o=e.value,s=o[0],l=o[1];return t.call(n,l,s,i)?r===N?e:q(r,s,l,e):(u=!1,{value:void 0,done:!0})})},r}(this,e,t))},takeUntil:function(e,t){return this.takeWhile(Nn(e),t)},valueSeq:function(){return this.toIndexedSeq()},hashCode:function(){return this.__hash||(this.__hash=function(e){if(e.size===1/0)return 0;var t=c(e),n=u(e),r=t?1:0;return function(e,t){return t=xe(t,3432918353),t=xe(t<<15|t>>>-15,461845907),t=xe(t<<13|t>>>-13,5),t=xe((t=(t+3864292196|0)^e)^t>>>16,2246822507),t=Se((t=xe(t^t>>>13,3266489909))^t>>>16)}(e.__iterate(n?t?function(e,t){r=31*r+qn(Ce(e),Ce(t))|0}:function(e,t){r=r+qn(Ce(e),Ce(t))|0}:t?function(e){r=31*r+Ce(e)|0}:function(e){r=r+Ce(e)|0}),r)}(this))}});var Tn=n.prototype;Tn[f]=!0,Tn[L]=Tn.values,Tn.__toJS=Tn.toArray,Tn.__toStringMapper=Dn,Tn.inspect=Tn.toSource=function(){return this.toString()},Tn.chain=Tn.flatMap,Tn.contains=Tn.includes,Pn(r,{flip:function(){return Gt(this,Ut(this))},mapEntries:function(e,t){var n=this,r=0;return Gt(this,this.toSeq().map(function(o,i){return e.call(t,[i,o],r++,n)}).fromEntrySeq())},mapKeys:function(e,t){var n=this;return Gt(this,this.toSeq().flip().map(function(r,o){return e.call(t,r,o,n)}).flip())}});var Mn=r.prototype;function In(e,t){return t}function jn(e,t){return[t,e]}function Nn(e){return function(){return!e.apply(this,arguments)}}function Rn(e){return function(){return-e.apply(this,arguments)}}function Dn(e){return"string"==typeof e?JSON.stringify(e):String(e)}function Ln(){return S(arguments)}function Un(e,t){return et?-1:0}function qn(e,t){return e^t+2654435769+(e<<6)+(e>>2)|0}return Mn[p]=!0,Mn[L]=Tn.entries,Mn.__toJS=Tn.toObject,Mn.__toStringMapper=function(e,t){return JSON.stringify(t)+": "+Dn(e)},Pn(o,{toKeyedSeq:function(){return new Nt(this,!1)},filter:function(e,t){return Gt(this,zt(this,e,t,!1))},findIndex:function(e,t){var n=this.findEntry(e,t);return n?n[0]:-1},indexOf:function(e){var t=this.keyOf(e);return void 0===t?-1:t},lastIndexOf:function(e){var t=this.lastKeyOf(e);return void 0===t?-1:t},reverse:function(){return Gt(this,Ft(this,!1))},slice:function(e,t){return Gt(this,Bt(this,e,t,!1))},splice:function(e,t){var n=arguments.length;if(t=Math.max(0|t,0),0===n||2===n&&!t)return this;e=P(e,e<0?this.count():this.size);var r=this.slice(0,e);return Gt(this,1===n?r:r.concat(S(arguments,2),this.slice(e+t)))},findLastIndex:function(e,t){var n=this.findLastEntry(e,t);return n?n[0]:-1},first:function(){return this.get(0)},flatten:function(e){return Gt(this,Ht(this,e,!1))},get:function(e,t){return(e=k(this,e))<0||this.size===1/0||void 0!==this.size&&e>this.size?t:this.find(function(t,n){return n===e},void 0,t)},has:function(e){return(e=k(this,e))>=0&&(void 0!==this.size?this.size===1/0||e5e3)return e.textContent;return function(e){for(var n,r,o,i,a,u=e.textContent,s=0,l=u[0],c=1,f=e.innerHTML="",p=0;r=n,n=p<7&&"\\"==n?1:c;){if(c=l,l=u[++s],i=f.length>1,!c||p>8&&"\n"==c||[/\S/.test(c),1,1,!/[$\w]/.test(c),("/"==n||"\n"==n)&&i,'"'==n&&i,"'"==n&&i,u[s-4]+r+n=="--\x3e",r+n=="*/"][p])for(f&&(e.appendChild(a=t.createElement("span")).setAttribute("style",["color: #555; font-weight: bold;","","","color: #555;",""][p?p<3?2:p>6?4:p>3?3:+/^(a(bstract|lias|nd|rguments|rray|s(m|sert)?|uto)|b(ase|egin|ool(ean)?|reak|yte)|c(ase|atch|har|hecked|lass|lone|ompl|onst|ontinue)|de(bugger|cimal|clare|f(ault|er)?|init|l(egate|ete)?)|do|double|e(cho|ls?if|lse(if)?|nd|nsure|num|vent|x(cept|ec|p(licit|ort)|te(nds|nsion|rn)))|f(allthrough|alse|inal(ly)?|ixed|loat|or(each)?|riend|rom|unc(tion)?)|global|goto|guard|i(f|mp(lements|licit|ort)|n(it|clude(_once)?|line|out|stanceof|t(erface|ernal)?)?|s)|l(ambda|et|ock|ong)|m(icrolight|odule|utable)|NaN|n(amespace|ative|ext|ew|il|ot|ull)|o(bject|perator|r|ut|verride)|p(ackage|arams|rivate|rotected|rotocol|ublic)|r(aise|e(adonly|do|f|gister|peat|quire(_once)?|scue|strict|try|turn))|s(byte|ealed|elf|hort|igned|izeof|tatic|tring|truct|ubscript|uper|ynchronized|witch)|t(emplate|hen|his|hrows?|ransient|rue|ry|ype(alias|def|id|name|of))|u(n(checked|def(ined)?|ion|less|signed|til)|se|sing)|v(ar|irtual|oid|olatile)|w(char_t|hen|here|hile|ith)|xor|yield)$/.test(f):0]),a.appendChild(t.createTextNode(f))),o=p&&p<7?p:o,f="",p=11;![1,/[\/{}[(\-+*=<>:;|\\.,?!&@~]/.test(c),/[\])]/.test(c),/[$\w]/.test(c),"/"==c&&o<2&&"<"!=n,'"'==c,"'"==c,c+l+u[s+1]+u[s+2]=="\x3c!--",c+l=="/*",c+l=="//","#"==c][--p];);f+=c}}(e)},t.mapToList=function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"key";var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:l.default.Map();if(!l.default.Map.isMap(t)||!t.size)return l.default.List();Array.isArray(n)||(n=[n]);if(n.length<1)return t.merge(r);var a=l.default.List();var u=n[0];var s=!0;var c=!1;var f=void 0;try{for(var p,d=(0,i.default)(t.entries());!(s=(p=d.next()).done);s=!0){var h=p.value,v=(0,o.default)(h,2),m=v[0],g=v[1],y=e(g,n.slice(1),r.set(u,m));a=l.default.List.isList(y)?a.concat(y):a.push(y)}}catch(e){c=!0,f=e}finally{try{!s&&d.return&&d.return()}finally{if(c)throw f}}return a},t.extractFileNameFromContentDispositionHeader=function(e){var t=void 0;if([/filename\*=[^']+'\w*'"([^"]+)";?/i,/filename\*=[^']+'\w*'([^;]+);?/i,/filename="([^;]*);?"/i,/filename=([^;]*);?/i].some(function(n){return null!==(t=n.exec(e))}),null!==t&&t.length>1)try{return decodeURIComponent(t[1])}catch(e){console.error(e)}return null},t.pascalCase=C,t.pascalCaseFilename=function(e){return C(e.replace(/\.[^./]*$/,""))},t.sanitizeUrl=function(e){if("string"!=typeof e||""===e)return"";return(0,c.sanitizeUrl)(e)},t.getAcceptControllingResponse=function(e){if(!l.default.OrderedMap.isOrderedMap(e))return null;if(!e.size)return null;var t=e.find(function(e,t){return t.startsWith("2")&&(0,u.default)(e.get("content")||{}).length>0}),n=e.get("default")||l.default.OrderedMap(),r=(n.get("content")||l.default.OrderedMap()).keySeq().toJS().length?n:null;return t||r},t.deeplyStripKey=function e(t,n){var r=arguments.length>2&&void 0!==arguments[2]?arguments[2]:function(){return!0};if("object"!==(void 0===t?"undefined":(0,s.default)(t))||Array.isArray(t)||null===t||!n)return t;var o=(0,a.default)({},t);(0,u.default)(o).forEach(function(t){t===n&&r(o[t],t)?delete o[t]:o[t]=e(o[t],n,r)});return o},t.stringify=function(e){if("string"==typeof e)return e;e.toJS&&(e=e.toJS());if("object"===(void 0===e?"undefined":(0,s.default)(e))&&null!==e)try{return(0,r.default)(e,null,2)}catch(t){return String(e)}return e.toString()},t.numberToString=function(e){if("number"==typeof e)return e.toString();return e},t.paramToIdentifier=q,t.paramToValue=function(e,t){return q(e,{returnAll:!0}).map(function(e){return t[e]}).filter(function(e){return void 0!==e})[0]};var l=_(n(7)),c=n(572),f=_(n(573)),p=_(n(281)),d=_(n(285)),h=_(n(288)),v=_(n(651)),m=_(n(105)),g=n(194),y=_(n(32)),b=_(n(724));function _(e){return e&&e.__esModule?e:{default:e}}var w="default",E=t.isImmutable=function(e){return l.default.Iterable.isIterable(e)};function x(e){return Array.isArray(e)?e:[e]}function S(e){return!!e&&"object"===(void 0===e?"undefined":(0,s.default)(e))}t.memoize=d.default;function C(e){return(0,p.default)((0,f.default)(e))}t.propChecker=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:[],r=arguments.length>3&&void 0!==arguments[3]?arguments[3]:[];return(0,u.default)(e).length!==(0,u.default)(t).length||((0,v.default)(e,function(e,n){if(r.includes(n))return!1;var o=t[n];return l.default.Iterable.isIterable(e)?!l.default.is(e,o):("object"!==(void 0===e?"undefined":(0,s.default)(e))||"object"!==(void 0===o?"undefined":(0,s.default)(o)))&&e!==o})||n.some(function(n){return!(0,m.default)(e[n],t[n])}))};var k=t.validateMaximum=function(e,t){if(e>t)return"Value must be less than Maximum"},A=t.validateMinimum=function(e,t){if(et)return"Value must be less than MaxLength"},D=t.validateMinLength=function(e,t){if(e.length2&&void 0!==arguments[2]?arguments[2]:{},r=n.isOAS3,o=void 0!==r&&r,i=n.bypassRequiredCheck,a=void 0!==i&&i,u=[],c=e.get("required"),f=o?e.get("schema"):e;if(!f)return u;var p=f.get("maximum"),d=f.get("minimum"),h=f.get("type"),v=f.get("format"),m=f.get("maxLength"),g=f.get("minLength"),b=f.get("pattern");if(h&&(c||t)){var _="string"===h&&t,w="array"===h&&Array.isArray(t)&&t.length,E="array"===h&&l.default.List.isList(t)&&t.count(),x="file"===h&&t instanceof y.default.File,S="boolean"===h&&(t||!1===t),C="number"===h&&(t||0===t),U="integer"===h&&(t||0===t),q=!1;if(o&&"object"===h)if("object"===(void 0===t?"undefined":(0,s.default)(t)))q=!0;else if("string"==typeof t)try{JSON.parse(t),q=!0}catch(e){return u.push("Parameter string value must be valid JSON"),u}var F=[_,w,E,x,S,C,U,q].some(function(e){return!!e});if(c&&!F&&!a)return u.push("Required field is not provided"),u;if(b){var z=L(t,b);z&&u.push(z)}if(m||0===m){var B=R(t,m);B&&u.push(B)}if(g){var V=D(t,g);V&&u.push(V)}if(p||0===p){var H=k(t,p);H&&u.push(H)}if(d||0===d){var W=A(t,d);W&&u.push(W)}if("string"===h){var J=void 0;if(!(J="date-time"===v?j(t):"uuid"===v?N(t):I(t)))return u;u.push(J)}else if("boolean"===h){var Y=M(t);if(!Y)return u;u.push(Y)}else if("number"===h){var K=O(t);if(!K)return u;u.push(K)}else if("integer"===h){var G=P(t);if(!G)return u;u.push(G)}else if("array"===h){var $;if(!E||!t.count())return u;$=f.getIn(["items","type"]),t.forEach(function(e,t){var n=void 0;"number"===$?n=O(e):"integer"===$?n=P(e):"string"===$&&(n=I(e)),n&&u.push({index:t,error:n})})}else if("file"===h){var Z=T(t);if(!Z)return u;u.push(Z)}}return u},t.getSampleSchema=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"",n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:{};if(/xml/.test(t)){if(!e.xml||!e.xml.name){if(e.xml=e.xml||{},!e.$$ref)return e.type||e.items||e.properties||e.additionalProperties?'\n\x3c!-- XML example cannot be generated; root element name is undefined --\x3e':null;var o=e.$$ref.match(/\S*\/(\S+)$/);e.xml.name=o[1]}return(0,g.memoizedCreateXMLExample)(e,n)}var i=(0,g.memoizedSampleFromSchema)(e,n);return"object"===(void 0===i?"undefined":(0,s.default)(i))?(0,r.default)(i,null,2):i},t.parseSearch=function(){var e={},t=y.default.location.search;if(!t)return{};if(""!=t){var n=t.substr(1).split("&");for(var r in n)n.hasOwnProperty(r)&&(r=n[r].split("="),e[decodeURIComponent(r[0])]=r[1]&&decodeURIComponent(r[1])||"")}return e},t.serializeSearch=function(e){return(0,u.default)(e).map(function(t){return encodeURIComponent(t)+"="+encodeURIComponent(e[t])}).join("&")},t.btoa=function(t){return(t instanceof e?t:new e(t.toString(),"utf-8")).toString("base64")},t.sorters={operationsSorter:{alpha:function(e,t){return e.get("path").localeCompare(t.get("path"))},method:function(e,t){return e.get("method").localeCompare(t.get("method"))}},tagsSorter:{alpha:function(e,t){return e.localeCompare(t)}}},t.buildFormData=function(e){var t=[];for(var n in e){var r=e[n];void 0!==r&&""!==r&&t.push([n,"=",encodeURIComponent(r).replace(/%20/g,"+")].join(""))}return t.join("&")},t.shallowEqualKeys=function(e,t,n){return!!(0,h.default)(n,function(n){return(0,m.default)(e[n],t[n])})};var U=t.createDeepLinkPath=function(e){return"string"==typeof e||e instanceof String?e.trim().replace(/\s/g,"%20"):""};t.escapeDeepLinkPath=function(e){return(0,b.default)(U(e).replace(/%20/g,"_"))},t.getExtensions=function(e){return e.filter(function(e,t){return/^x-/.test(t)})},t.getCommonExtensions=function(e){return e.filter(function(e,t){return/^pattern|maxLength|minLength|maximum|minimum/.test(t)})};function q(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},n=t.returnAll,r=void 0!==n&&n,o=t.allowHashes,i=void 0===o||o;if(!l.default.Map.isMap(e))throw new Error("paramToIdentifier: received a non-Im.Map parameter as input");var a=e.get("name"),u=e.get("in"),s=[];return e&&e.hashCode&&u&&a&&i&&s.push(u+"."+a+".hash-"+e.hashCode()),u&&a&&s.push(u+"."+a),s.push(a),r?s:s[0]||""}}).call(t,n(54).Buffer)},function(e,t,n){"use strict";var r=n(34);e.exports=r},function(e,t,n){"use strict";e.exports=function(e){for(var t=arguments.length-1,n="Minified React error #"+e+"; visit http://facebook.github.io/react/docs/error-decoder.html?invariant="+e,r=0;r>",i={listOf:function(e){return l(e,"List",r.List.isList)},mapOf:function(e,t){return c(e,t,"Map",r.Map.isMap)},orderedMapOf:function(e,t){return c(e,t,"OrderedMap",r.OrderedMap.isOrderedMap)},setOf:function(e){return l(e,"Set",r.Set.isSet)},orderedSetOf:function(e){return l(e,"OrderedSet",r.OrderedSet.isOrderedSet)},stackOf:function(e){return l(e,"Stack",r.Stack.isStack)},iterableOf:function(e){return l(e,"Iterable",r.Iterable.isIterable)},recordOf:function(e){return u(function(t,n,o,i,u){for(var s=arguments.length,l=Array(s>5?s-5:0),c=5;c6?s-6:0),c=6;c5?l-5:0),f=5;f5?i-5:0),u=5;u key("+c[f]+")"].concat(a));if(d instanceof Error)return d}})).apply(void 0,i);var s})}function f(e){var t=void 0===arguments[1]?"Iterable":arguments[1],n=void 0===arguments[2]?r.Iterable.isIterable:arguments[2];return u(function(r,o,i,u,s){for(var l=arguments.length,c=Array(l>5?l-5:0),f=5;f?@[\]^_`{|}~-])/g;function a(e){return!(e>=55296&&e<=57343)&&(!(e>=64976&&e<=65007)&&(65535!=(65535&e)&&65534!=(65535&e)&&(!(e>=0&&e<=8)&&(11!==e&&(!(e>=14&&e<=31)&&(!(e>=127&&e<=159)&&!(e>1114111)))))))}function u(e){if(e>65535){var t=55296+((e-=65536)>>10),n=56320+(1023&e);return String.fromCharCode(t,n)}return String.fromCharCode(e)}var s=/&([a-z#][a-z0-9]{1,31});/gi,l=/^#((?:x[a-f0-9]{1,8}|[0-9]{1,8}))/i,c=n(417);function f(e,t){var n=0;return o(c,t)?c[t]:35===t.charCodeAt(0)&&l.test(t)&&a(n="x"===t[1].toLowerCase()?parseInt(t.slice(2),16):parseInt(t.slice(1),10))?u(n):e}var p=/[&<>"]/,d=/[&<>"]/g,h={"&":"&","<":"<",">":">",'"':"""};function v(e){return h[e]}t.assign=function(e){return[].slice.call(arguments,1).forEach(function(t){if(t){if("object"!=typeof t)throw new TypeError(t+"must be object");Object.keys(t).forEach(function(n){e[n]=t[n]})}}),e},t.isString=function(e){return"[object String]"===function(e){return Object.prototype.toString.call(e)}(e)},t.has=o,t.unescapeMd=function(e){return e.indexOf("\\")<0?e:e.replace(i,"$1")},t.isValidEntityCode=a,t.fromCodePoint=u,t.replaceEntities=function(e){return e.indexOf("&")<0?e:e.replace(s,f)},t.escapeHtml=function(e){return p.test(e)?e.replace(d,v):e}},function(e,t){e.exports=function(e){return"object"==typeof e?null!==e:"function"==typeof e}},function(e,t,n){var r=n(33),o=n(61),i=n(59),a=n(73),u=n(120),s=function(e,t,n){var l,c,f,p,d=e&s.F,h=e&s.G,v=e&s.S,m=e&s.P,g=e&s.B,y=h?r:v?r[t]||(r[t]={}):(r[t]||{}).prototype,b=h?o:o[t]||(o[t]={}),_=b.prototype||(b.prototype={});for(l in h&&(n=t),n)f=((c=!d&&y&&void 0!==y[l])?y:n)[l],p=g&&c?u(f,r):m&&"function"==typeof f?u(Function.call,f):f,y&&a(y,l,f,e&s.U),b[l]!=f&&i(b,l,p),m&&_[l]!=f&&(_[l]=f)};r.core=o,s.F=1,s.G=2,s.S=4,s.P=8,s.B=16,s.W=32,s.U=64,s.R=128,e.exports=s},function(e,t,n){var r=n(29),o=n(101),i=n(53),a=/"/g,u=function(e,t,n,r){var o=String(i(e)),u="<"+t;return""!==n&&(u+=" "+n+'="'+String(r).replace(a,""")+'"'),u+">"+o+""};e.exports=function(e,t){var n={};n[e]=t(u),r(r.P+r.F*o(function(){var t=""[e]('"');return t!==t.toLowerCase()||t.split('"').length>3}),"String",n)}},function(e,t){var n;n=function(){return this}();try{n=n||Function("return this")()||(0,eval)("this")}catch(e){"object"==typeof window&&(n=window)}e.exports=n},function(e,t,n){"use strict";var r,o=n(91),i=(r=o)&&r.__esModule?r:{default:r};e.exports=function(){var e={location:{},history:{},open:function(){},close:function(){},File:function(){}};if("undefined"==typeof window)return e;try{e=window;var t=!0,n=!1,r=void 0;try{for(var o,a=(0,i.default)(["File","Blob","FormData"]);!(t=(o=a.next()).done);t=!0){var u=o.value;u in window&&(e[u]=window[u])}}catch(e){n=!0,r=e}finally{try{!t&&a.return&&a.return()}finally{if(n)throw r}}}catch(e){console.error(e)}return e}()},function(e,t){var n=e.exports="undefined"!=typeof window&&window.Math==Math?window:"undefined"!=typeof self&&self.Math==Math?self:Function("return this")();"number"==typeof __g&&(__g=n)},function(e,t,n){"use strict";function r(e){return function(){return e}}var o=function(){};o.thatReturns=r,o.thatReturnsFalse=r(!1),o.thatReturnsTrue=r(!0),o.thatReturnsNull=r(null),o.thatReturnsThis=function(){return this},o.thatReturnsArgument=function(e){return e},e.exports=o},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=i(n(25));t.isOAS3=a,t.isSwagger2=function(e){var t=e.get("swagger");if("string"!=typeof t)return!1;return t.startsWith("2.0")},t.OAS3ComponentWrapFactory=function(e){return function(t,n){return function(i){if(n&&n.specSelectors&&n.specSelectors.specJson){var u=n.specSelectors.specJson();return a(u)?o.default.createElement(e,(0,r.default)({},i,n,{Ori:t})):o.default.createElement(t,i)}return console.warn("OAS3 wrapper: couldn't get spec"),null}}};var o=i(n(0));function i(e){return e&&e.__esModule?e:{default:e}}function a(e){var t=e.get("openapi");return"string"==typeof t&&(t.startsWith("3.0.")&&t.length>4)}},function(e,t,n){var r=n(28);e.exports=function(e){if(!r(e))throw TypeError(e+" is not an object!");return e}},function(e,t,n){var r=n(279),o="object"==typeof self&&self&&self.Object===Object&&self,i=r||o||Function("return this")();e.exports=i},function(e,t){e.exports=function(e){var t=typeof e;return null!=e&&("object"==t||"function"==t)}},function(e,t,n){"use strict";var r=null;e.exports={debugTool:r}},function(e,t,n){var r=n(36),o=n(239),i=n(158),a=Object.defineProperty;t.f=n(44)?Object.defineProperty:function(e,t,n){if(r(e),t=i(t,!0),r(n),o)try{return a(e,t,n)}catch(e){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(e[t]=n.value),e}},function(e,t,n){e.exports={default:n(517),__esModule:!0}},function(e,t,n){e.exports={default:n(518),__esModule:!0}},function(e,t,n){"use strict";var r=n(11),o=n(13),i=n(354),a=n(69),u=n(355),s=n(88),l=n(148),c=n(8),f=[],p=0,d=i.getPooled(),h=!1,v=null;function m(){E.ReactReconcileTransaction&&v||r("123")}var g=[{initialize:function(){this.dirtyComponentsLength=f.length},close:function(){this.dirtyComponentsLength!==f.length?(f.splice(0,this.dirtyComponentsLength),w()):f.length=0}},{initialize:function(){this.callbackQueue.reset()},close:function(){this.callbackQueue.notifyAll()}}];function y(){this.reinitializeTransaction(),this.dirtyComponentsLength=null,this.callbackQueue=i.getPooled(),this.reconcileTransaction=E.ReactReconcileTransaction.getPooled(!0)}function b(e,t){return e._mountOrder-t._mountOrder}function _(e){var t=e.dirtyComponentsLength;t!==f.length&&r("124",t,f.length),f.sort(b),p++;for(var n=0;n + * @license MIT + */ +var r=n(529),o=n(530),i=n(262);function a(){return s.TYPED_ARRAY_SUPPORT?2147483647:1073741823}function u(e,t){if(a()=a())throw new RangeError("Attempt to allocate Buffer larger than maximum size: 0x"+a().toString(16)+" bytes");return 0|e}function h(e,t){if(s.isBuffer(e))return e.length;if("undefined"!=typeof ArrayBuffer&&"function"==typeof ArrayBuffer.isView&&(ArrayBuffer.isView(e)||e instanceof ArrayBuffer))return e.byteLength;"string"!=typeof e&&(e=""+e);var n=e.length;if(0===n)return 0;for(var r=!1;;)switch(t){case"ascii":case"latin1":case"binary":return n;case"utf8":case"utf-8":case void 0:return F(e).length;case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return 2*n;case"hex":return n>>>1;case"base64":return z(e).length;default:if(r)return F(e).length;t=(""+t).toLowerCase(),r=!0}}function v(e,t,n){var r=e[t];e[t]=e[n],e[n]=r}function m(e,t,n,r,o){if(0===e.length)return-1;if("string"==typeof n?(r=n,n=0):n>2147483647?n=2147483647:n<-2147483648&&(n=-2147483648),n=+n,isNaN(n)&&(n=o?0:e.length-1),n<0&&(n=e.length+n),n>=e.length){if(o)return-1;n=e.length-1}else if(n<0){if(!o)return-1;n=0}if("string"==typeof t&&(t=s.from(t,r)),s.isBuffer(t))return 0===t.length?-1:g(e,t,n,r,o);if("number"==typeof t)return t&=255,s.TYPED_ARRAY_SUPPORT&&"function"==typeof Uint8Array.prototype.indexOf?o?Uint8Array.prototype.indexOf.call(e,t,n):Uint8Array.prototype.lastIndexOf.call(e,t,n):g(e,[t],n,r,o);throw new TypeError("val must be string, number or Buffer")}function g(e,t,n,r,o){var i,a=1,u=e.length,s=t.length;if(void 0!==r&&("ucs2"===(r=String(r).toLowerCase())||"ucs-2"===r||"utf16le"===r||"utf-16le"===r)){if(e.length<2||t.length<2)return-1;a=2,u/=2,s/=2,n/=2}function l(e,t){return 1===a?e[t]:e.readUInt16BE(t*a)}if(o){var c=-1;for(i=n;iu&&(n=u-s),i=n;i>=0;i--){for(var f=!0,p=0;po&&(r=o):r=o;var i=t.length;if(i%2!=0)throw new TypeError("Invalid hex string");r>i/2&&(r=i/2);for(var a=0;a>8,o=n%256,i.push(o),i.push(r);return i}(t,e.length-n),e,n,r)}function S(e,t,n){return 0===t&&n===e.length?r.fromByteArray(e):r.fromByteArray(e.slice(t,n))}function C(e,t,n){n=Math.min(e.length,n);for(var r=[],o=t;o239?4:l>223?3:l>191?2:1;if(o+f<=n)switch(f){case 1:l<128&&(c=l);break;case 2:128==(192&(i=e[o+1]))&&(s=(31&l)<<6|63&i)>127&&(c=s);break;case 3:i=e[o+1],a=e[o+2],128==(192&i)&&128==(192&a)&&(s=(15&l)<<12|(63&i)<<6|63&a)>2047&&(s<55296||s>57343)&&(c=s);break;case 4:i=e[o+1],a=e[o+2],u=e[o+3],128==(192&i)&&128==(192&a)&&128==(192&u)&&(s=(15&l)<<18|(63&i)<<12|(63&a)<<6|63&u)>65535&&s<1114112&&(c=s)}null===c?(c=65533,f=1):c>65535&&(c-=65536,r.push(c>>>10&1023|55296),c=56320|1023&c),r.push(c),o+=f}return function(e){var t=e.length;if(t<=k)return String.fromCharCode.apply(String,e);var n="",r=0;for(;rthis.length)return"";if((void 0===n||n>this.length)&&(n=this.length),n<=0)return"";if((n>>>=0)<=(t>>>=0))return"";for(e||(e="utf8");;)switch(e){case"hex":return P(this,t,n);case"utf8":case"utf-8":return C(this,t,n);case"ascii":return A(this,t,n);case"latin1":case"binary":return O(this,t,n);case"base64":return S(this,t,n);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return T(this,t,n);default:if(r)throw new TypeError("Unknown encoding: "+e);e=(e+"").toLowerCase(),r=!0}}.apply(this,arguments)},s.prototype.equals=function(e){if(!s.isBuffer(e))throw new TypeError("Argument must be a Buffer");return this===e||0===s.compare(this,e)},s.prototype.inspect=function(){var e="",n=t.INSPECT_MAX_BYTES;return this.length>0&&(e=this.toString("hex",0,n).match(/.{2}/g).join(" "),this.length>n&&(e+=" ... ")),""},s.prototype.compare=function(e,t,n,r,o){if(!s.isBuffer(e))throw new TypeError("Argument must be a Buffer");if(void 0===t&&(t=0),void 0===n&&(n=e?e.length:0),void 0===r&&(r=0),void 0===o&&(o=this.length),t<0||n>e.length||r<0||o>this.length)throw new RangeError("out of range index");if(r>=o&&t>=n)return 0;if(r>=o)return-1;if(t>=n)return 1;if(t>>>=0,n>>>=0,r>>>=0,o>>>=0,this===e)return 0;for(var i=o-r,a=n-t,u=Math.min(i,a),l=this.slice(r,o),c=e.slice(t,n),f=0;fo)&&(n=o),e.length>0&&(n<0||t<0)||t>this.length)throw new RangeError("Attempt to write outside buffer bounds");r||(r="utf8");for(var i=!1;;)switch(r){case"hex":return y(this,e,t,n);case"utf8":case"utf-8":return b(this,e,t,n);case"ascii":return _(this,e,t,n);case"latin1":case"binary":return w(this,e,t,n);case"base64":return E(this,e,t,n);case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return x(this,e,t,n);default:if(i)throw new TypeError("Unknown encoding: "+r);r=(""+r).toLowerCase(),i=!0}},s.prototype.toJSON=function(){return{type:"Buffer",data:Array.prototype.slice.call(this._arr||this,0)}};var k=4096;function A(e,t,n){var r="";n=Math.min(e.length,n);for(var o=t;or)&&(n=r);for(var o="",i=t;in)throw new RangeError("Trying to access beyond buffer length")}function I(e,t,n,r,o,i){if(!s.isBuffer(e))throw new TypeError('"buffer" argument must be a Buffer instance');if(t>o||te.length)throw new RangeError("Index out of range")}function j(e,t,n,r){t<0&&(t=65535+t+1);for(var o=0,i=Math.min(e.length-n,2);o>>8*(r?o:1-o)}function N(e,t,n,r){t<0&&(t=4294967295+t+1);for(var o=0,i=Math.min(e.length-n,4);o>>8*(r?o:3-o)&255}function R(e,t,n,r,o,i){if(n+r>e.length)throw new RangeError("Index out of range");if(n<0)throw new RangeError("Index out of range")}function D(e,t,n,r,i){return i||R(e,0,n,4),o.write(e,t,n,r,23,4),n+4}function L(e,t,n,r,i){return i||R(e,0,n,8),o.write(e,t,n,r,52,8),n+8}s.prototype.slice=function(e,t){var n,r=this.length;if(e=~~e,t=void 0===t?r:~~t,e<0?(e+=r)<0&&(e=0):e>r&&(e=r),t<0?(t+=r)<0&&(t=0):t>r&&(t=r),t0&&(o*=256);)r+=this[e+--t]*o;return r},s.prototype.readUInt8=function(e,t){return t||M(e,1,this.length),this[e]},s.prototype.readUInt16LE=function(e,t){return t||M(e,2,this.length),this[e]|this[e+1]<<8},s.prototype.readUInt16BE=function(e,t){return t||M(e,2,this.length),this[e]<<8|this[e+1]},s.prototype.readUInt32LE=function(e,t){return t||M(e,4,this.length),(this[e]|this[e+1]<<8|this[e+2]<<16)+16777216*this[e+3]},s.prototype.readUInt32BE=function(e,t){return t||M(e,4,this.length),16777216*this[e]+(this[e+1]<<16|this[e+2]<<8|this[e+3])},s.prototype.readIntLE=function(e,t,n){e|=0,t|=0,n||M(e,t,this.length);for(var r=this[e],o=1,i=0;++i=(o*=128)&&(r-=Math.pow(2,8*t)),r},s.prototype.readIntBE=function(e,t,n){e|=0,t|=0,n||M(e,t,this.length);for(var r=t,o=1,i=this[e+--r];r>0&&(o*=256);)i+=this[e+--r]*o;return i>=(o*=128)&&(i-=Math.pow(2,8*t)),i},s.prototype.readInt8=function(e,t){return t||M(e,1,this.length),128&this[e]?-1*(255-this[e]+1):this[e]},s.prototype.readInt16LE=function(e,t){t||M(e,2,this.length);var n=this[e]|this[e+1]<<8;return 32768&n?4294901760|n:n},s.prototype.readInt16BE=function(e,t){t||M(e,2,this.length);var n=this[e+1]|this[e]<<8;return 32768&n?4294901760|n:n},s.prototype.readInt32LE=function(e,t){return t||M(e,4,this.length),this[e]|this[e+1]<<8|this[e+2]<<16|this[e+3]<<24},s.prototype.readInt32BE=function(e,t){return t||M(e,4,this.length),this[e]<<24|this[e+1]<<16|this[e+2]<<8|this[e+3]},s.prototype.readFloatLE=function(e,t){return t||M(e,4,this.length),o.read(this,e,!0,23,4)},s.prototype.readFloatBE=function(e,t){return t||M(e,4,this.length),o.read(this,e,!1,23,4)},s.prototype.readDoubleLE=function(e,t){return t||M(e,8,this.length),o.read(this,e,!0,52,8)},s.prototype.readDoubleBE=function(e,t){return t||M(e,8,this.length),o.read(this,e,!1,52,8)},s.prototype.writeUIntLE=function(e,t,n,r){(e=+e,t|=0,n|=0,r)||I(this,e,t,n,Math.pow(2,8*n)-1,0);var o=1,i=0;for(this[t]=255&e;++i=0&&(i*=256);)this[t+o]=e/i&255;return t+n},s.prototype.writeUInt8=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,1,255,0),s.TYPED_ARRAY_SUPPORT||(e=Math.floor(e)),this[t]=255&e,t+1},s.prototype.writeUInt16LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,65535,0),s.TYPED_ARRAY_SUPPORT?(this[t]=255&e,this[t+1]=e>>>8):j(this,e,t,!0),t+2},s.prototype.writeUInt16BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,65535,0),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>8,this[t+1]=255&e):j(this,e,t,!1),t+2},s.prototype.writeUInt32LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,4294967295,0),s.TYPED_ARRAY_SUPPORT?(this[t+3]=e>>>24,this[t+2]=e>>>16,this[t+1]=e>>>8,this[t]=255&e):N(this,e,t,!0),t+4},s.prototype.writeUInt32BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,4294967295,0),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>24,this[t+1]=e>>>16,this[t+2]=e>>>8,this[t+3]=255&e):N(this,e,t,!1),t+4},s.prototype.writeIntLE=function(e,t,n,r){if(e=+e,t|=0,!r){var o=Math.pow(2,8*n-1);I(this,e,t,n,o-1,-o)}var i=0,a=1,u=0;for(this[t]=255&e;++i>0)-u&255;return t+n},s.prototype.writeIntBE=function(e,t,n,r){if(e=+e,t|=0,!r){var o=Math.pow(2,8*n-1);I(this,e,t,n,o-1,-o)}var i=n-1,a=1,u=0;for(this[t+i]=255&e;--i>=0&&(a*=256);)e<0&&0===u&&0!==this[t+i+1]&&(u=1),this[t+i]=(e/a>>0)-u&255;return t+n},s.prototype.writeInt8=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,1,127,-128),s.TYPED_ARRAY_SUPPORT||(e=Math.floor(e)),e<0&&(e=255+e+1),this[t]=255&e,t+1},s.prototype.writeInt16LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,32767,-32768),s.TYPED_ARRAY_SUPPORT?(this[t]=255&e,this[t+1]=e>>>8):j(this,e,t,!0),t+2},s.prototype.writeInt16BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,2,32767,-32768),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>8,this[t+1]=255&e):j(this,e,t,!1),t+2},s.prototype.writeInt32LE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,2147483647,-2147483648),s.TYPED_ARRAY_SUPPORT?(this[t]=255&e,this[t+1]=e>>>8,this[t+2]=e>>>16,this[t+3]=e>>>24):N(this,e,t,!0),t+4},s.prototype.writeInt32BE=function(e,t,n){return e=+e,t|=0,n||I(this,e,t,4,2147483647,-2147483648),e<0&&(e=4294967295+e+1),s.TYPED_ARRAY_SUPPORT?(this[t]=e>>>24,this[t+1]=e>>>16,this[t+2]=e>>>8,this[t+3]=255&e):N(this,e,t,!1),t+4},s.prototype.writeFloatLE=function(e,t,n){return D(this,e,t,!0,n)},s.prototype.writeFloatBE=function(e,t,n){return D(this,e,t,!1,n)},s.prototype.writeDoubleLE=function(e,t,n){return L(this,e,t,!0,n)},s.prototype.writeDoubleBE=function(e,t,n){return L(this,e,t,!1,n)},s.prototype.copy=function(e,t,n,r){if(n||(n=0),r||0===r||(r=this.length),t>=e.length&&(t=e.length),t||(t=0),r>0&&r=this.length)throw new RangeError("sourceStart out of bounds");if(r<0)throw new RangeError("sourceEnd out of bounds");r>this.length&&(r=this.length),e.length-t=0;--o)e[o+t]=this[o+n];else if(i<1e3||!s.TYPED_ARRAY_SUPPORT)for(o=0;o>>=0,n=void 0===n?this.length:n>>>0,e||(e=0),"number"==typeof e)for(i=t;i55295&&n<57344){if(!o){if(n>56319){(t-=3)>-1&&i.push(239,191,189);continue}if(a+1===r){(t-=3)>-1&&i.push(239,191,189);continue}o=n;continue}if(n<56320){(t-=3)>-1&&i.push(239,191,189),o=n;continue}n=65536+(o-55296<<10|n-56320)}else o&&(t-=3)>-1&&i.push(239,191,189);if(o=null,n<128){if((t-=1)<0)break;i.push(n)}else if(n<2048){if((t-=2)<0)break;i.push(n>>6|192,63&n|128)}else if(n<65536){if((t-=3)<0)break;i.push(n>>12|224,n>>6&63|128,63&n|128)}else{if(!(n<1114112))throw new Error("Invalid code point");if((t-=4)<0)break;i.push(n>>18|240,n>>12&63|128,n>>6&63|128,63&n|128)}}return i}function z(e){return r.toByteArray(function(e){if((e=function(e){return e.trim?e.trim():e.replace(/^\s+|\s+$/g,"")}(e).replace(U,"")).length<2)return"";for(;e.length%4!=0;)e+="=";return e}(e))}function B(e,t,n,r){for(var o=0;o=t.length||o>=e.length);++o)t[o+n]=e[o];return o}}).call(t,n(31))},function(e,t,n){var r=n(278);e.exports=function(e){return null==e?"":r(e)}},function(e,t){var n,r,o=e.exports={};function i(){throw new Error("setTimeout has not been defined")}function a(){throw new Error("clearTimeout has not been defined")}function u(e){if(n===setTimeout)return setTimeout(e,0);if((n===i||!n)&&setTimeout)return n=setTimeout,setTimeout(e,0);try{return n(e,0)}catch(t){try{return n.call(null,e,0)}catch(t){return n.call(this,e,0)}}}!function(){try{n="function"==typeof setTimeout?setTimeout:i}catch(e){n=i}try{r="function"==typeof clearTimeout?clearTimeout:a}catch(e){r=a}}();var s,l=[],c=!1,f=-1;function p(){c&&s&&(c=!1,s.length?l=s.concat(l):f=-1,l.length&&d())}function d(){if(!c){var e=u(p);c=!0;for(var t=l.length;t;){for(s=l,l=[];++f1)for(var n=1;n1?t-1:0),r=1;r2?n-2:0),o=2;o1){for(var h=Array(d),v=0;v1){for(var g=Array(m),y=0;y=0||Object.prototype.hasOwnProperty.call(e,r)&&(n[r]=e[r]);return n}},function(e,t,n){"use strict";function r(e){return void 0===e||null===e}e.exports.isNothing=r,e.exports.isObject=function(e){return"object"==typeof e&&null!==e},e.exports.toArray=function(e){return Array.isArray(e)?e:r(e)?[]:[e]},e.exports.repeat=function(e,t){var n,r="";for(n=0;n=t.length?{value:void 0,done:!0}:(e=r(t,n),this._i+=e.length,{value:e,done:!1})})},function(e,t){var n={}.toString;e.exports=function(e){return n.call(e).slice(8,-1)}},function(e,t,n){e.exports=!n(101)(function(){return 7!=Object.defineProperty({},"a",{get:function(){return 7}}).a})},function(e,t){e.exports=function(e){try{return!!e()}catch(e){return!0}}},function(e,t){e.exports={}},function(e,t,n){var r=n(119),o=Math.min;e.exports=function(e){return e>0?o(r(e),9007199254740991):0}},function(e,t,n){"use strict";e.exports=function(e){for(var t=arguments.length-1,n="Minified React error #"+e+"; visit http://facebook.github.io/react/docs/error-decoder.html?invariant="+e,r=0;r0?o(r(e),9007199254740991):0}},function(e,t){var n=0,r=Math.random();e.exports=function(e){return"Symbol(".concat(void 0===e?"":e,")_",(++n+r).toString(36))}},function(e,t,n){var r=n(60),o=n(460),i=n(461),a=Object.defineProperty;t.f=n(100)?Object.defineProperty:function(e,t,n){if(r(e),t=i(t,!0),r(n),o)try{return a(e,t,n)}catch(e){}if("get"in n||"set"in n)throw TypeError("Accessors not supported!");return"value"in n&&(e[t]=n.value),e}},function(e,t){var n={}.hasOwnProperty;e.exports=function(e,t){return n.call(e,t)}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t,n){var r=n(121);e.exports=function(e,t,n){if(r(e),void 0===t)return e;switch(n){case 1:return function(n){return e.call(t,n)};case 2:return function(n,r){return e.call(t,n,r)};case 3:return function(n,r,o){return e.call(t,n,r,o)}}return function(){return e.apply(t,arguments)}}},function(e,t){e.exports=function(e){if("function"!=typeof e)throw TypeError(e+" is not a function!");return e}},function(e,t,n){var r=n(466),o=n(53);e.exports=function(e){return r(o(e))}},function(e,t,n){"use strict";var r=n(59),o=n(73),i=n(101),a=n(53),u=n(18);e.exports=function(e,t,n){var s=u(e),l=n(a,s,""[e]),c=l[0],f=l[1];i(function(){var t={};return t[s]=function(){return 7},7!=""[e](t)})&&(o(String.prototype,e,c),r(RegExp.prototype,s,2==t?function(e,t){return f.call(e,this,t)}:function(e){return f.call(e,this)}))}},function(e,t,n){var r=n(116)("meta"),o=n(28),i=n(52),a=n(40).f,u=0,s=Object.isExtensible||function(){return!0},l=!n(51)(function(){return s(Object.preventExtensions({}))}),c=function(e){a(e,r,{value:{i:"O"+ ++u,w:{}}})},f=e.exports={KEY:r,NEED:!1,fastKey:function(e,t){if(!o(e))return"symbol"==typeof e?e:("string"==typeof e?"S":"P")+e;if(!i(e,r)){if(!s(e))return"F";if(!t)return"E";c(e)}return e[r].i},getWeak:function(e,t){if(!i(e,r)){if(!s(e))return!0;if(!t)return!1;c(e)}return e[r].w},onFreeze:function(e){return l&&f.NEED&&s(e)&&!i(e,r)&&c(e),e}}},function(e,t){t.f={}.propertyIsEnumerable},function(e,t,n){"use strict";var r={};e.exports=r},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.CLEAR_BY=t.CLEAR=t.NEW_AUTH_ERR=t.NEW_SPEC_ERR_BATCH=t.NEW_SPEC_ERR=t.NEW_THROWN_ERR_BATCH=t.NEW_THROWN_ERR=void 0,t.newThrownErr=function(e){return{type:a,payload:(0,i.default)(e)}},t.newThrownErrBatch=function(e){return{type:u,payload:e}},t.newSpecErr=function(e){return{type:s,payload:e}},t.newSpecErrBatch=function(e){return{type:l,payload:e}},t.newAuthErr=function(e){return{type:c,payload:e}},t.clear=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return{type:f,payload:e}},t.clearBy=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:function(){return!0};return{type:p,payload:e}};var r,o=n(180),i=(r=o)&&r.__esModule?r:{default:r};var a=t.NEW_THROWN_ERR="err_new_thrown_err",u=t.NEW_THROWN_ERR_BATCH="err_new_thrown_err_batch",s=t.NEW_SPEC_ERR="err_new_spec_err",l=t.NEW_SPEC_ERR_BATCH="err_new_spec_err_batch",c=t.NEW_AUTH_ERR="err_new_auth_err",f=t.CLEAR="err_clear",p=t.CLEAR_BY="err_clear_by"},function(e,t,n){var r=n(62),o=n(47),i="[object Symbol]";e.exports=function(e){return"symbol"==typeof e||o(e)&&r(e)==i}},function(e,t,n){var r=n(63)(Object,"create");e.exports=r},function(e,t,n){var r=n(601),o=n(602),i=n(603),a=n(604),u=n(605);function s(e){var t=-1,n=null==e?0:e.length;for(this.clear();++t-1&&e%1==0&&e1&&void 0!==arguments[1]?arguments[1]:"";if(u.List.isList(e))return e.some(function(e){return u.Map.isMap(e)&&e.get("in")===t})},t.parametersIncludeType=T,t.contentTypeValues=function(e,t){t=t||[];var n=d(e).getIn(["paths"].concat((0,o.default)(t)),(0,u.fromJS)({})),r=e.getIn(["meta","paths"].concat((0,o.default)(t)),(0,u.fromJS)({})),i=M(e,t),a=n.get("parameters")||new u.List,s=r.get("consumes_value")?r.get("consumes_value"):T(a,"file")?"multipart/form-data":T(a,"formData")?"application/x-www-form-urlencoded":void 0;return(0,u.fromJS)({requestContentType:s,responseContentType:i})},t.currentProducesFor=M,t.producesOptionsFor=function(e,t){t=t||[];var n=d(e),i=n.getIn(["paths"].concat((0,o.default)(t)),null);if(null===i)return;var a=t,u=(0,r.default)(a,1)[0],s=i.get("produces",null),l=n.getIn(["paths",u,"produces"],null),c=n.getIn(["produces"],null);return s||l||c},t.consumesOptionsFor=function(e,t){t=t||[];var n=d(e),i=n.getIn(["paths"].concat((0,o.default)(t)),null);if(null===i)return;var a=t,u=(0,r.default)(a,1)[0],s=i.get("consumes",null),l=n.getIn(["paths",u,"consumes"],null),c=n.getIn(["consumes"],null);return s||l||c};var i=n(58),a=n(9),u=n(7);function s(e){return e&&e.__esModule?e:{default:e}}var l=["get","put","post","delete","options","head","patch","trace"],c=function(e){return e||(0,u.Map)()},f=(t.lastError=(0,i.createSelector)(c,function(e){return e.get("lastError")}),t.url=(0,i.createSelector)(c,function(e){return e.get("url")}),t.specStr=(0,i.createSelector)(c,function(e){return e.get("spec")||""}),t.specSource=(0,i.createSelector)(c,function(e){return e.get("specSource")||"not-editor"}),t.specJson=(0,i.createSelector)(c,function(e){return e.get("json",(0,u.Map)())})),p=(t.specResolved=(0,i.createSelector)(c,function(e){return e.get("resolved",(0,u.Map)())}),t.specResolvedSubtree=function(e,t){return e.getIn(["resolvedSubtrees"].concat((0,o.default)(t)),void 0)},function e(t,n){return u.Map.isMap(t)&&u.Map.isMap(n)?n.get("$$ref")?n:(0,u.OrderedMap)().mergeWith(e,t,n):n}),d=t.specJsonWithResolvedSubtrees=(0,i.createSelector)(c,function(e){return(0,u.OrderedMap)().mergeWith(p,e.get("json"),e.get("resolvedSubtrees"))}),h=t.spec=function(e){return f(e)},v=(t.isOAS3=(0,i.createSelector)(h,function(){return!1}),t.info=(0,i.createSelector)(h,function(e){return j(e&&e.get("info"))})),m=(t.externalDocs=(0,i.createSelector)(h,function(e){return j(e&&e.get("externalDocs"))}),t.version=(0,i.createSelector)(v,function(e){return e&&e.get("version")})),g=(t.semver=(0,i.createSelector)(m,function(e){return/v?([0-9]*)\.([0-9]*)\.([0-9]*)/i.exec(e).slice(1)}),t.paths=(0,i.createSelector)(d,function(e){return e.get("paths")})),y=t.operations=(0,i.createSelector)(g,function(e){if(!e||e.size<1)return(0,u.List)();var t=(0,u.List)();return e&&e.forEach?(e.forEach(function(e,n){if(!e||!e.forEach)return{};e.forEach(function(e,r){l.indexOf(r)<0||(t=t.push((0,u.fromJS)({path:n,method:r,operation:e,id:r+"-"+n})))})}),t):(0,u.List)()}),b=t.consumes=(0,i.createSelector)(h,function(e){return(0,u.Set)(e.get("consumes"))}),_=t.produces=(0,i.createSelector)(h,function(e){return(0,u.Set)(e.get("produces"))}),w=(t.security=(0,i.createSelector)(h,function(e){return e.get("security",(0,u.List)())}),t.securityDefinitions=(0,i.createSelector)(h,function(e){return e.get("securityDefinitions")}),t.findDefinition=function(e,t){var n=e.getIn(["resolvedSubtrees","definitions",t],null),r=e.getIn(["json","definitions",t],null);return n||r||null},t.definitions=(0,i.createSelector)(h,function(e){var t=e.get("definitions");return u.Map.isMap(t)?t:(0,u.Map)()}),t.basePath=(0,i.createSelector)(h,function(e){return e.get("basePath")}),t.host=(0,i.createSelector)(h,function(e){return e.get("host")}),t.schemes=(0,i.createSelector)(h,function(e){return e.get("schemes",(0,u.Map)())}),t.operationsWithRootInherited=(0,i.createSelector)(y,b,_,function(e,t,n){return e.map(function(e){return e.update("operation",function(e){if(e){if(!u.Map.isMap(e))return;return e.withMutations(function(e){return e.get("consumes")||e.update("consumes",function(e){return(0,u.Set)(e).merge(t)}),e.get("produces")||e.update("produces",function(e){return(0,u.Set)(e).merge(n)}),e})}return(0,u.Map)()})})})),E=t.tags=(0,i.createSelector)(h,function(e){var t=e.get("tags",(0,u.List)());return u.List.isList(t)?t.filter(function(e){return u.Map.isMap(e)}):(0,u.List)()}),x=t.tagDetails=function(e,t){return(E(e)||(0,u.List)()).filter(u.Map.isMap).find(function(e){return e.get("name")===t},(0,u.Map)())},S=t.operationsWithTags=(0,i.createSelector)(w,E,function(e,t){return e.reduce(function(e,t){var n=(0,u.Set)(t.getIn(["operation","tags"]));return n.count()<1?e.update("default",(0,u.List)(),function(e){return e.push(t)}):n.reduce(function(e,n){return e.update(n,(0,u.List)(),function(e){return e.push(t)})},e)},t.reduce(function(e,t){return e.set(t.get("name"),(0,u.List)())},(0,u.OrderedMap)()))}),C=(t.taggedOperations=function(e){return function(t){var n=(0,t.getConfigs)(),r=n.tagsSorter,o=n.operationsSorter;return S(e).sortBy(function(e,t){return t},function(e,t){var n="function"==typeof r?r:a.sorters.tagsSorter[r];return n?n(e,t):null}).map(function(t,n){var r="function"==typeof o?o:a.sorters.operationsSorter[o],i=r?t.sort(r):t;return(0,u.Map)({tagDetails:x(e,n),operations:i})})}},t.responses=(0,i.createSelector)(c,function(e){return e.get("responses",(0,u.Map)())})),k=t.requests=(0,i.createSelector)(c,function(e){return e.get("requests",(0,u.Map)())}),A=t.mutatedRequests=(0,i.createSelector)(c,function(e){return e.get("mutatedRequests",(0,u.Map)())}),O=(t.responseFor=function(e,t,n){return C(e).getIn([t,n],null)},t.requestFor=function(e,t,n){return k(e).getIn([t,n],null)},t.mutatedRequestFor=function(e,t,n){return A(e).getIn([t,n],null)},t.allowTryItOutFor=function(){return!0},t.parameterWithMetaByIdentity=function(e,t,n){var r=d(e).getIn(["paths"].concat((0,o.default)(t),["parameters"]),(0,u.OrderedMap)()),i=e.getIn(["meta","paths"].concat((0,o.default)(t),["parameters"]),(0,u.OrderedMap)());return r.map(function(e){var t=i.get(n.get("in")+"."+n.get("name")),r=i.get(n.get("in")+"."+n.get("name")+".hash-"+n.hashCode());return(0,u.OrderedMap)().merge(e,t,r)}).find(function(e){return e.get("in")===n.get("in")&&e.get("name")===n.get("name")},(0,u.OrderedMap)())}),P=(t.parameterInclusionSettingFor=function(e,t,n,r){var i=r+"."+n;return e.getIn(["meta","paths"].concat((0,o.default)(t),["parameter_inclusions",i]),!1)},t.parameterWithMeta=function(e,t,n,r){var i=d(e).getIn(["paths"].concat((0,o.default)(t),["parameters"]),(0,u.OrderedMap)()).find(function(e){return e.get("in")===r&&e.get("name")===n},(0,u.OrderedMap)());return O(e,t,i)},t.operationWithMeta=function(e,t,n){var r=d(e).getIn(["paths",t,n],(0,u.OrderedMap)()),o=e.getIn(["meta","paths",t,n],(0,u.OrderedMap)()),i=r.get("parameters",(0,u.List)()).map(function(r){return O(e,[t,n],r)});return(0,u.OrderedMap)().merge(r,o).set("parameters",i)});t.hasHost=(0,i.createSelector)(h,function(e){var t=e.get("host");return"string"==typeof t&&t.length>0&&"/"!==t[0]});function T(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"";if(u.List.isList(e))return e.some(function(e){return u.Map.isMap(e)&&e.get("type")===t})}function M(e,t){t=t||[];var n=d(e).getIn(["paths"].concat((0,o.default)(t)),null);if(null!==n){var r=e.getIn(["meta","paths"].concat((0,o.default)(t),["produces_value"]),null),i=n.getIn(["produces",0],null);return r||i||"application/json"}}var I=t.operationScheme=function(e,t,n){var r=e.get("url").match(/^([a-z][a-z0-9+\-.]*):/),o=Array.isArray(r)?r[1]:null;return e.getIn(["scheme",t,n])||e.getIn(["scheme","_defaultScheme"])||o||""};t.canExecuteScheme=function(e,t,n){return["http","https"].indexOf(I(e,t,n))>-1},t.validateBeforeExecute=function(e,t){t=t||[];var n=!0;return e.getIn(["meta","paths"].concat((0,o.default)(t),["parameters"]),(0,u.fromJS)([])).forEach(function(e){var t=e.get("errors");t&&t.count()&&(n=!1)}),n};function j(e){return u.Map.isMap(e)?e:new u.Map}},function(e,t,n){var r=n(49),o=n(329),i=n(330),a=n(36),u=n(115),s=n(165),l={},c={};(t=e.exports=function(e,t,n,f,p){var d,h,v,m,g=p?function(){return e}:s(e),y=r(n,f,t?2:1),b=0;if("function"!=typeof g)throw TypeError(e+" is not iterable!");if(i(g)){for(d=u(e.length);d>b;b++)if((m=t?y(a(h=e[b])[0],h[1]):y(e[b]))===l||m===c)return m}else for(v=g.call(e);!(h=v.next()).done;)if((m=o(v,y,h.value,t))===l||m===c)return m}).BREAK=l,t.RETURN=c},function(e,t,n){"use strict";var r=n(86);e.exports=r.DEFAULT=new r({include:[n(108)],explicit:[n(758),n(759),n(760)]})},function(e,t,n){var r=n(344),o=n(105),i=Object.prototype.hasOwnProperty;e.exports=function(e,t,n){var a=e[t];i.call(e,t)&&o(a,n)&&(void 0!==n||t in e)||r(e,t,n)}},function(e,t,n){"use strict";var r=n(11),o=(n(8),{}),i={reinitializeTransaction:function(){this.transactionWrappers=this.getTransactionWrappers(),this.wrapperInitData?this.wrapperInitData.length=0:this.wrapperInitData=[],this._isInTransaction=!1},_isInTransaction:!1,getTransactionWrappers:null,isInTransaction:function(){return!!this._isInTransaction},perform:function(e,t,n,o,i,a,u,s){var l,c;this.isInTransaction()&&r("27");try{this._isInTransaction=!0,l=!0,this.initializeAll(0),c=e.call(t,n,o,i,a,u,s),l=!1}finally{try{if(l)try{this.closeAll(0)}catch(e){}else this.closeAll(0)}finally{this._isInTransaction=!1}}return c},initializeAll:function(e){for(var t=this.transactionWrappers,n=e;n]/,s=n(219)(function(e,t){if(e.namespaceURI!==i.svg||"innerHTML"in e)e.innerHTML=t;else{(r=r||document.createElement("div")).innerHTML=""+t+"";for(var n=r.firstChild;n.firstChild;)e.appendChild(n.firstChild)}});if(o.canUseDOM){var l=document.createElement("div");l.innerHTML=" ",""===l.innerHTML&&(s=function(e,t){if(e.parentNode&&e.parentNode.replaceChild(e,e),a.test(t)||"<"===t[0]&&u.test(t)){e.innerHTML=String.fromCharCode(65279)+t;var n=e.firstChild;1===n.data.length?e.removeChild(n):n.deleteData(0,1)}else e.innerHTML=t}),l=null}e.exports=s},function(e,t,n){"use strict";var r=/["'&<>]/;e.exports=function(e){return"boolean"==typeof e||"number"==typeof e?""+e:function(e){var t,n=""+e,o=r.exec(n);if(!o)return n;var i="",a=0,u=0;for(a=o.index;adocument.F=Object<\/script>"),e.close(),s=e.F;r--;)delete s.prototype[i[r]];return s()};e.exports=Object.create||function(e,t){var n;return null!==e?(u.prototype=r(e),n=new u,u.prototype=null,n[a]=e):n=s(),void 0===t?n:o(n,t)}},function(e,t){var n=Math.ceil,r=Math.floor;e.exports=function(e){return isNaN(e=+e)?0:(e>0?r:n)(e)}},function(e,t,n){var r=n(163)("keys"),o=n(116);e.exports=function(e){return r[e]||(r[e]=o(e))}},function(e,t,n){var r=n(21),o=r["__core-js_shared__"]||(r["__core-js_shared__"]={});e.exports=function(e){return o[e]||(o[e]={})}},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t,n){var r=n(166),o=n(19)("iterator"),i=n(70);e.exports=n(15).getIteratorMethod=function(e){if(void 0!=e)return e[o]||e["@@iterator"]||i[r(e)]}},function(e,t,n){var r=n(93),o=n(19)("toStringTag"),i="Arguments"==r(function(){return arguments}());e.exports=function(e){var t,n,a;return void 0===e?"Undefined":null===e?"Null":"string"==typeof(n=function(e,t){try{return e[t]}catch(e){}}(t=Object(e),o))?n:i?r(t):"Object"==(a=r(t))&&"function"==typeof t.callee?"Arguments":a}},function(e,t,n){var r=n(99),o=n(18)("toStringTag"),i="Arguments"==r(function(){return arguments}());e.exports=function(e){var t,n,a;return void 0===e?"Undefined":null===e?"Null":"string"==typeof(n=function(e,t){try{return e[t]}catch(e){}}(t=Object(e),o))?n:i?r(t):"Object"==(a=r(t))&&"function"==typeof t.callee?"Arguments":a}},function(e,t){var n=0,r=Math.random();e.exports=function(e){return"Symbol(".concat(void 0===e?"":e,")_",(++n+r).toString(36))}},function(e,t,n){var r=n(74),o=n(33).document,i=r(o)&&r(o.createElement);e.exports=function(e){return i?o.createElement(e):{}}},function(e,t,n){var r=n(243)("keys"),o=n(168);e.exports=function(e){return r[e]||(r[e]=o(e))}},function(e,t,n){var r=n(117).f,o=n(118),i=n(18)("toStringTag");e.exports=function(e,t,n){e&&!o(e=n?e:e.prototype,i)&&r(e,i,{configurable:!0,value:t})}},function(e,t,n){"use strict";var r=n(121);e.exports.f=function(e){return new function(e){var t,n;this.promise=new e(function(e,r){if(void 0!==t||void 0!==n)throw TypeError("Bad Promise constructor");t=e,n=r}),this.resolve=r(t),this.reject=r(n)}(e)}},function(e,t,n){var r=n(257),o=n(53);e.exports=function(e,t,n){if(r(t))throw TypeError("String#"+n+" doesn't accept regex!");return String(o(e))}},function(e,t,n){var r=n(18)("match");e.exports=function(e){var t=/./;try{"/./"[e](t)}catch(n){try{return t[r]=!1,!"/./"[e](t)}catch(e){}}return!0}},function(e,t,n){t.f=n(19)},function(e,t,n){var r=n(21),o=n(15),i=n(114),a=n(175),u=n(40).f;e.exports=function(e){var t=o.Symbol||(o.Symbol=i?{}:r.Symbol||{});"_"==e.charAt(0)||e in t||u(t,e,{value:a.f(e)})}},function(e,t){t.f=Object.getOwnPropertySymbols},function(e,t){},function(e,t,n){"use strict";(function(t){ +/*! + * @description Recursive object extending + * @author Viacheslav Lotsmanov + * @license MIT + * + * The MIT License (MIT) + * + * Copyright (c) 2013-2018 Viacheslav Lotsmanov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy of + * this software and associated documentation files (the "Software"), to deal in + * the Software without restriction, including without limitation the rights to + * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of + * the Software, and to permit persons to whom the Software is furnished to do so, + * subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS + * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR + * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER + * IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + */ +function n(e){return e instanceof t||e instanceof Date||e instanceof RegExp}function r(e){if(e instanceof t){var n=t.alloc?t.alloc(e.length):new t(e.length);return e.copy(n),n}if(e instanceof Date)return new Date(e.getTime());if(e instanceof RegExp)return new RegExp(e);throw new Error("Unexpected situation")}function o(e,t){return"__proto__"===t?void 0:e[t]}var i=e.exports=function(){if(arguments.length<1||"object"!=typeof arguments[0])return!1;if(arguments.length<2)return arguments[0];var e,t,a=arguments[0];return Array.prototype.slice.call(arguments,1).forEach(function(u){"object"!=typeof u||null===u||Array.isArray(u)||Object.keys(u).forEach(function(s){return t=o(a,s),(e=o(u,s))===a?void 0:"object"!=typeof e||null===e?void(a[s]=e):Array.isArray(e)?void(a[s]=function e(t){var o=[];return t.forEach(function(t,a){"object"==typeof t&&null!==t?Array.isArray(t)?o[a]=e(t):n(t)?o[a]=r(t):o[a]=i({},t):o[a]=t}),o}(e)):n(e)?void(a[s]=r(e)):"object"!=typeof t||null===t||Array.isArray(t)?void(a[s]=i({},e)):void(a[s]=i(t,e))})}),a}}).call(t,n(54).Buffer)},function(e,t,n){"use strict";e.exports=function(e){return"object"==typeof e?function e(t,n){var r;r=Array.isArray(t)?[]:{};n.push(t);Object.keys(t).forEach(function(o){var i=t[o];"function"!=typeof i&&(i&&"object"==typeof i?-1!==n.indexOf(t[o])?r[o]="[Circular]":r[o]=e(t[o],n.slice(0)):r[o]=i)});"string"==typeof t.name&&(r.name=t.name);"string"==typeof t.message&&(r.message=t.message);"string"==typeof t.stack&&(r.stack=t.stack);return r}(e,[]):"function"==typeof e?"[Function: "+(e.name||"anonymous")+"]":e}},function(e,t,n){"use strict";var r="function"==typeof Symbol&&"symbol"==typeof Symbol.iterator?function(e){return typeof e}:function(e){return e&&"function"==typeof Symbol&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e};function o(e){return null===e?"null":void 0===e?"undefined":"object"===(void 0===e?"undefined":r(e))?Array.isArray(e)?"array":"object":void 0===e?"undefined":r(e)}function i(e){return"object"===o(e)?u(e):"array"===o(e)?a(e):e}function a(e){return e.map(i)}function u(e){var t={};for(var n in e)e.hasOwnProperty(n)&&(t[n]=i(e[n]));return t}function s(e){for(var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:[],n={arrayBehaviour:(arguments.length>2&&void 0!==arguments[2]?arguments[2]:{}).arrayBehaviour||"replace"},r=t.map(function(e){return e||{}}),i=e||{},l=0;l1?t-1:0),r=1;r-1&&e%1==0&&e<=n}},function(e,t){e.exports=function(e){return function(t){return e(t)}}},function(e,t,n){(function(e){var r=n(279),o="object"==typeof t&&t&&!t.nodeType&&t,i=o&&"object"==typeof e&&e&&!e.nodeType&&e,a=i&&i.exports===o&&r.process,u=function(){try{var e=i&&i.require&&i.require("util").types;return e||a&&a.binding&&a.binding("util")}catch(e){}}();e.exports=u}).call(t,n(134)(e))},function(e,t,n){var r=n(24),o=n(128),i=/\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/,a=/^\w*$/;e.exports=function(e,t){if(r(e))return!1;var n=typeof e;return!("number"!=n&&"symbol"!=n&&"boolean"!=n&&null!=e&&!o(e))||a.test(e)||!i.test(e)||null!=t&&e in Object(t)}},function(e,t){e.exports=function(e){return e}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.memoizedSampleFromSchema=t.memoizedCreateXMLExample=t.sampleXmlFromSchema=t.inferSchema=t.sampleFromSchema=void 0,t.createXMLExample=p;var r=n(9),o=u(n(657)),i=u(n(670)),a=u(n(181));function u(e){return e&&e.__esModule?e:{default:e}}var s={string:function(){return"string"},string_email:function(){return"user@example.com"},"string_date-time":function(){return(new Date).toISOString()},string_date:function(){return(new Date).toISOString().substring(0,10)},string_uuid:function(){return"3fa85f64-5717-4562-b3fc-2c963f66afa6"},string_hostname:function(){return"example.com"},string_ipv4:function(){return"198.51.100.42"},string_ipv6:function(){return"2001:0db8:5b96:0000:0000:426f:8e17:642a"},number:function(){return 0},number_float:function(){return 0},integer:function(){return 0},boolean:function(e){return"boolean"!=typeof e.default||e.default}},l=function(e){var t=e=(0,r.objectify)(e),n=t.type,o=t.format,i=s[n+"_"+o]||s[n];return(0,r.isFunc)(i)?i(e):"Unknown Type: "+e.type},c=t.sampleFromSchema=function e(t){var n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},o=(0,r.objectify)(t),i=o.type,a=o.example,u=o.properties,s=o.additionalProperties,c=o.items,f=n.includeReadOnly,p=n.includeWriteOnly;if(void 0!==a)return(0,r.deeplyStripKey)(a,"$$ref",function(e){return"string"==typeof e&&e.indexOf("#")>-1});if(!i)if(u)i="object";else{if(!c)return;i="array"}if("object"===i){var d=(0,r.objectify)(u),h={};for(var v in d)d[v]&&d[v].deprecated||d[v]&&d[v].readOnly&&!f||d[v]&&d[v].writeOnly&&!p||(h[v]=e(d[v],n));if(!0===s)h.additionalProp1={};else if(s)for(var m=(0,r.objectify)(s),g=e(m,n),y=1;y<4;y++)h["additionalProp"+y]=g;return h}return"array"===i?Array.isArray(c.anyOf)?c.anyOf.map(function(t){return e(t,n)}):Array.isArray(c.oneOf)?c.oneOf.map(function(t){return e(t,n)}):[e(c,n)]:t.enum?t.default?t.default:(0,r.normalizeArray)(t.enum)[0]:"file"!==i?l(t):void 0},f=(t.inferSchema=function(e){return e.schema&&(e=e.schema),e.properties&&(e.type="object"),e},t.sampleXmlFromSchema=function e(t){var n,o=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{},i=(0,a.default)({},(0,r.objectify)(t)),u=i.type,s=i.properties,c=i.additionalProperties,f=i.items,p=i.example,d=o.includeReadOnly,h=o.includeWriteOnly,v=i.default,m={},g={},y=t.xml,b=y.name,_=y.prefix,w=y.namespace,E=i.enum,x=void 0;if(!u)if(s||c)u="object";else{if(!f)return;u="array"}(b=b||"notagname",n=(_?_+":":"")+b,w)&&(g[_?"xmlns:"+_:"xmlns"]=w);if("array"===u&&f){if(f.xml=f.xml||y||{},f.xml.name=f.xml.name||y.name,y.wrapped)return m[n]=[],Array.isArray(p)?p.forEach(function(t){f.example=t,m[n].push(e(f,o))}):Array.isArray(v)?v.forEach(function(t){f.default=t,m[n].push(e(f,o))}):m[n]=[e(f,o)],g&&m[n].push({_attr:g}),m;var S=[];return Array.isArray(p)?(p.forEach(function(t){f.example=t,S.push(e(f,o))}),S):Array.isArray(v)?(v.forEach(function(t){f.default=t,S.push(e(f,o))}),S):e(f,o)}if("object"===u){var C=(0,r.objectify)(s);for(var k in m[n]=[],p=p||{},C)if(C.hasOwnProperty(k)&&(!C[k].readOnly||d)&&(!C[k].writeOnly||h))if(C[k].xml=C[k].xml||{},C[k].xml.attribute){var A=Array.isArray(C[k].enum)&&C[k].enum[0],O=C[k].example,P=C[k].default;g[C[k].xml.name||k]=void 0!==O&&O||void 0!==p[k]&&p[k]||void 0!==P&&P||A||l(C[k])}else{C[k].xml.name=C[k].xml.name||k,void 0===C[k].example&&void 0!==p[k]&&(C[k].example=p[k]);var T=e(C[k]);Array.isArray(T)?m[n]=m[n].concat(T):m[n].push(T)}return!0===c?m[n].push({additionalProp:"Anything can be here"}):c&&m[n].push({additionalProp:l(c)}),g&&m[n].push({_attr:g}),m}return x=void 0!==p?p:void 0!==v?v:Array.isArray(E)?E[0]:l(t),m[n]=g?[{_attr:g},x]:x,m});function p(e,t){var n=f(e,t);if(n)return(0,o.default)(n,{declaration:!0,indent:"\t"})}t.memoizedCreateXMLExample=(0,i.default)(p),t.memoizedSampleFromSchema=(0,i.default)(c)},function(e,t){function n(){this._events=this._events||{},this._maxListeners=this._maxListeners||void 0}function r(e){return"function"==typeof e}function o(e){return"object"==typeof e&&null!==e}function i(e){return void 0===e}e.exports=n,n.EventEmitter=n,n.prototype._events=void 0,n.prototype._maxListeners=void 0,n.defaultMaxListeners=10,n.prototype.setMaxListeners=function(e){if("number"!=typeof e||e<0||isNaN(e))throw TypeError("n must be a positive number");return this._maxListeners=e,this},n.prototype.emit=function(e){var t,n,a,u,s,l;if(this._events||(this._events={}),"error"===e&&(!this._events.error||o(this._events.error)&&!this._events.error.length)){if((t=arguments[1])instanceof Error)throw t;var c=new Error('Uncaught, unspecified "error" event. ('+t+")");throw c.context=t,c}if(i(n=this._events[e]))return!1;if(r(n))switch(arguments.length){case 1:n.call(this);break;case 2:n.call(this,arguments[1]);break;case 3:n.call(this,arguments[1],arguments[2]);break;default:u=Array.prototype.slice.call(arguments,1),n.apply(this,u)}else if(o(n))for(u=Array.prototype.slice.call(arguments,1),a=(l=n.slice()).length,s=0;s0&&this._events[e].length>a&&(this._events[e].warned=!0,console.error("(node) warning: possible EventEmitter memory leak detected. %d listeners added. Use emitter.setMaxListeners() to increase limit.",this._events[e].length),"function"==typeof console.trace&&console.trace()),this},n.prototype.on=n.prototype.addListener,n.prototype.once=function(e,t){if(!r(t))throw TypeError("listener must be a function");var n=!1;function o(){this.removeListener(e,o),n||(n=!0,t.apply(this,arguments))}return o.listener=t,this.on(e,o),this},n.prototype.removeListener=function(e,t){var n,i,a,u;if(!r(t))throw TypeError("listener must be a function");if(!this._events||!this._events[e])return this;if(a=(n=this._events[e]).length,i=-1,n===t||r(n.listener)&&n.listener===t)delete this._events[e],this._events.removeListener&&this.emit("removeListener",e,t);else if(o(n)){for(u=a;u-- >0;)if(n[u]===t||n[u].listener&&n[u].listener===t){i=u;break}if(i<0)return this;1===n.length?(n.length=0,delete this._events[e]):n.splice(i,1),this._events.removeListener&&this.emit("removeListener",e,t)}return this},n.prototype.removeAllListeners=function(e){var t,n;if(!this._events)return this;if(!this._events.removeListener)return 0===arguments.length?this._events={}:this._events[e]&&delete this._events[e],this;if(0===arguments.length){for(t in this._events)"removeListener"!==t&&this.removeAllListeners(t);return this.removeAllListeners("removeListener"),this._events={},this}if(r(n=this._events[e]))this.removeListener(e,n);else if(n)for(;n.length;)this.removeListener(e,n[n.length-1]);return delete this._events[e],this},n.prototype.listeners=function(e){return this._events&&this._events[e]?r(this._events[e])?[this._events[e]]:this._events[e].slice():[]},n.prototype.listenerCount=function(e){if(this._events){var t=this._events[e];if(r(t))return 1;if(t)return t.length}return 0},n.listenerCount=function(e,t){return e.listenerCount(t)}},function(e,t,n){(t=e.exports=n(306)).Stream=t,t.Readable=t,t.Writable=n(197),t.Duplex=n(65),t.Transform=n(311),t.PassThrough=n(665)},function(e,t,n){"use strict";(function(t,r,o){var i=n(140);function a(e){var t=this;this.next=null,this.entry=null,this.finish=function(){!function(e,t,n){var r=e.entry;e.entry=null;for(;r;){var o=r.callback;t.pendingcb--,o(n),r=r.next}t.corkedRequestsFree?t.corkedRequestsFree.next=e:t.corkedRequestsFree=e}(t,e)}}e.exports=y;var u,s=!t.browser&&["v0.10","v0.9."].indexOf(t.version.slice(0,5))>-1?r:i.nextTick;y.WritableState=g;var l=n(106);l.inherits=n(81);var c={deprecate:n(664)},f=n(307),p=n(141).Buffer,d=o.Uint8Array||function(){};var h,v=n(308);function m(){}function g(e,t){u=u||n(65),e=e||{};var r=t instanceof u;this.objectMode=!!e.objectMode,r&&(this.objectMode=this.objectMode||!!e.writableObjectMode);var o=e.highWaterMark,l=e.writableHighWaterMark,c=this.objectMode?16:16384;this.highWaterMark=o||0===o?o:r&&(l||0===l)?l:c,this.highWaterMark=Math.floor(this.highWaterMark),this.finalCalled=!1,this.needDrain=!1,this.ending=!1,this.ended=!1,this.finished=!1,this.destroyed=!1;var f=!1===e.decodeStrings;this.decodeStrings=!f,this.defaultEncoding=e.defaultEncoding||"utf8",this.length=0,this.writing=!1,this.corked=0,this.sync=!0,this.bufferProcessing=!1,this.onwrite=function(e){!function(e,t){var n=e._writableState,r=n.sync,o=n.writecb;if(function(e){e.writing=!1,e.writecb=null,e.length-=e.writelen,e.writelen=0}(n),t)!function(e,t,n,r,o){--t.pendingcb,n?(i.nextTick(o,r),i.nextTick(S,e,t),e._writableState.errorEmitted=!0,e.emit("error",r)):(o(r),e._writableState.errorEmitted=!0,e.emit("error",r),S(e,t))}(e,n,r,t,o);else{var a=E(n);a||n.corked||n.bufferProcessing||!n.bufferedRequest||w(e,n),r?s(_,e,n,a,o):_(e,n,a,o)}}(t,e)},this.writecb=null,this.writelen=0,this.bufferedRequest=null,this.lastBufferedRequest=null,this.pendingcb=0,this.prefinished=!1,this.errorEmitted=!1,this.bufferedRequestCount=0,this.corkedRequestsFree=new a(this)}function y(e){if(u=u||n(65),!(h.call(y,this)||this instanceof u))return new y(e);this._writableState=new g(e,this),this.writable=!0,e&&("function"==typeof e.write&&(this._write=e.write),"function"==typeof e.writev&&(this._writev=e.writev),"function"==typeof e.destroy&&(this._destroy=e.destroy),"function"==typeof e.final&&(this._final=e.final)),f.call(this)}function b(e,t,n,r,o,i,a){t.writelen=r,t.writecb=a,t.writing=!0,t.sync=!0,n?e._writev(o,t.onwrite):e._write(o,i,t.onwrite),t.sync=!1}function _(e,t,n,r){n||function(e,t){0===t.length&&t.needDrain&&(t.needDrain=!1,e.emit("drain"))}(e,t),t.pendingcb--,r(),S(e,t)}function w(e,t){t.bufferProcessing=!0;var n=t.bufferedRequest;if(e._writev&&n&&n.next){var r=t.bufferedRequestCount,o=new Array(r),i=t.corkedRequestsFree;i.entry=n;for(var u=0,s=!0;n;)o[u]=n,n.isBuf||(s=!1),n=n.next,u+=1;o.allBuffers=s,b(e,t,!0,t.length,o,"",i.finish),t.pendingcb++,t.lastBufferedRequest=null,i.next?(t.corkedRequestsFree=i.next,i.next=null):t.corkedRequestsFree=new a(t),t.bufferedRequestCount=0}else{for(;n;){var l=n.chunk,c=n.encoding,f=n.callback;if(b(e,t,!1,t.objectMode?1:l.length,l,c,f),n=n.next,t.bufferedRequestCount--,t.writing)break}null===n&&(t.lastBufferedRequest=null)}t.bufferedRequest=n,t.bufferProcessing=!1}function E(e){return e.ending&&0===e.length&&null===e.bufferedRequest&&!e.finished&&!e.writing}function x(e,t){e._final(function(n){t.pendingcb--,n&&e.emit("error",n),t.prefinished=!0,e.emit("prefinish"),S(e,t)})}function S(e,t){var n=E(t);return n&&(!function(e,t){t.prefinished||t.finalCalled||("function"==typeof e._final?(t.pendingcb++,t.finalCalled=!0,i.nextTick(x,e,t)):(t.prefinished=!0,e.emit("prefinish")))}(e,t),0===t.pendingcb&&(t.finished=!0,e.emit("finish"))),n}l.inherits(y,f),g.prototype.getBuffer=function(){for(var e=this.bufferedRequest,t=[];e;)t.push(e),e=e.next;return t},function(){try{Object.defineProperty(g.prototype,"buffer",{get:c.deprecate(function(){return this.getBuffer()},"_writableState.buffer is deprecated. Use _writableState.getBuffer instead.","DEP0003")})}catch(e){}}(),"function"==typeof Symbol&&Symbol.hasInstance&&"function"==typeof Function.prototype[Symbol.hasInstance]?(h=Function.prototype[Symbol.hasInstance],Object.defineProperty(y,Symbol.hasInstance,{value:function(e){return!!h.call(this,e)||this===y&&(e&&e._writableState instanceof g)}})):h=function(e){return e instanceof this},y.prototype.pipe=function(){this.emit("error",new Error("Cannot pipe, not readable"))},y.prototype.write=function(e,t,n){var r,o=this._writableState,a=!1,u=!o.objectMode&&(r=e,p.isBuffer(r)||r instanceof d);return u&&!p.isBuffer(e)&&(e=function(e){return p.from(e)}(e)),"function"==typeof t&&(n=t,t=null),u?t="buffer":t||(t=o.defaultEncoding),"function"!=typeof n&&(n=m),o.ended?function(e,t){var n=new Error("write after end");e.emit("error",n),i.nextTick(t,n)}(this,n):(u||function(e,t,n,r){var o=!0,a=!1;return null===n?a=new TypeError("May not write null values to stream"):"string"==typeof n||void 0===n||t.objectMode||(a=new TypeError("Invalid non-string/buffer chunk")),a&&(e.emit("error",a),i.nextTick(r,a),o=!1),o}(this,o,e,n))&&(o.pendingcb++,a=function(e,t,n,r,o,i){if(!n){var a=function(e,t,n){e.objectMode||!1===e.decodeStrings||"string"!=typeof t||(t=p.from(t,n));return t}(t,r,o);r!==a&&(n=!0,o="buffer",r=a)}var u=t.objectMode?1:r.length;t.length+=u;var s=t.length-1))throw new TypeError("Unknown encoding: "+e);return this._writableState.defaultEncoding=e,this},Object.defineProperty(y.prototype,"writableHighWaterMark",{enumerable:!1,get:function(){return this._writableState.highWaterMark}}),y.prototype._write=function(e,t,n){n(new Error("_write() is not implemented"))},y.prototype._writev=null,y.prototype.end=function(e,t,n){var r=this._writableState;"function"==typeof e?(n=e,e=null,t=null):"function"==typeof t&&(n=t,t=null),null!==e&&void 0!==e&&this.write(e,t),r.corked&&(r.corked=1,this.uncork()),r.ending||r.finished||function(e,t,n){t.ending=!0,S(e,t),n&&(t.finished?i.nextTick(n):e.once("finish",n));t.ended=!0,e.writable=!1}(this,r,n)},Object.defineProperty(y.prototype,"destroyed",{get:function(){return void 0!==this._writableState&&this._writableState.destroyed},set:function(e){this._writableState&&(this._writableState.destroyed=e)}}),y.prototype.destroy=v.destroy,y.prototype._undestroy=v.undestroy,y.prototype._destroy=function(e,t){this.end(),t(e)}}).call(t,n(56),n(309).setImmediate,n(31))},function(e,t,n){"use strict";e.exports=function(e){return"function"==typeof e}},function(e,t,n){"use strict";e.exports=n(691)()?Array.from:n(692)},function(e,t,n){"use strict";var r=n(705),o=n(67),i=n(82),a=Array.prototype.indexOf,u=Object.prototype.hasOwnProperty,s=Math.abs,l=Math.floor;e.exports=function(e){var t,n,c,f;if(!r(e))return a.apply(this,arguments);for(n=o(i(this).length),c=arguments[1],t=c=isNaN(c)?0:c>=0?l(c):o(this.length)-l(s(c));t1&&void 0!==arguments[1])||arguments[1];return e=(0,r.normalizeArray)(e),{type:u,payload:{thing:e,shown:t}}},t.changeMode=function(e){var t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"";return e=(0,r.normalizeArray)(e),{type:a,payload:{thing:e,mode:t}}};var r=n(9),o=t.UPDATE_LAYOUT="layout_update_layout",i=t.UPDATE_FILTER="layout_update_filter",a=t.UPDATE_MODE="layout_update_mode",u=t.SHOW="layout_show"},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.execute=t.executeRequest=t.logRequest=t.setMutatedRequest=t.setRequest=t.setResponse=t.updateEmptyParamInclusion=t.validateParams=t.invalidateResolvedSubtreeCache=t.updateResolvedSubtree=t.requestResolvedSubtree=t.resolveSpec=t.parseToJson=t.SET_SCHEME=t.UPDATE_RESOLVED_SUBTREE=t.UPDATE_RESOLVED=t.UPDATE_OPERATION_META_VALUE=t.CLEAR_VALIDATE_PARAMS=t.CLEAR_REQUEST=t.CLEAR_RESPONSE=t.LOG_REQUEST=t.SET_MUTATED_REQUEST=t.SET_REQUEST=t.SET_RESPONSE=t.VALIDATE_PARAMS=t.UPDATE_EMPTY_PARAM_INCLUSION=t.UPDATE_PARAM=t.UPDATE_JSON=t.UPDATE_URL=t.UPDATE_SPEC=void 0;var r=b(n(25)),o=b(n(84)),i=b(n(23)),a=b(n(42)),u=b(n(204)),s=b(n(338)),l=b(n(339)),c=b(n(45));t.updateSpec=function(e){var t=L(e).replace(/\t/g," ");if("string"==typeof e)return{type:_,payload:t}},t.updateResolved=function(e){return{type:N,payload:e}},t.updateUrl=function(e){return{type:w,payload:e}},t.updateJsonSpec=function(e){return{type:E,payload:e}},t.changeParam=function(e,t,n,r,o){return{type:x,payload:{path:e,value:r,paramName:t,paramIn:n,isXml:o}}},t.changeParamByIdentity=function(e,t,n,r){return{type:x,payload:{path:e,param:t,value:n,isXml:r}}},t.clearValidateParams=function(e){return{type:I,payload:{pathMethod:e}}},t.changeConsumesValue=function(e,t){return{type:j,payload:{path:e,value:t,key:"consumes_value"}}},t.changeProducesValue=function(e,t){return{type:j,payload:{path:e,value:t,key:"produces_value"}}},t.clearResponse=function(e,t){return{type:T,payload:{path:e,method:t}}},t.clearRequest=function(e,t){return{type:M,payload:{path:e,method:t}}},t.setScheme=function(e,t,n){return{type:D,payload:{scheme:e,path:t,method:n}}};var f=b(n(208)),p=n(7),d=b(n(210)),h=b(n(180)),v=b(n(342)),m=b(n(764)),g=b(n(766)),y=n(9);function b(e){return e&&e.__esModule?e:{default:e}}var _=t.UPDATE_SPEC="spec_update_spec",w=t.UPDATE_URL="spec_update_url",E=t.UPDATE_JSON="spec_update_json",x=t.UPDATE_PARAM="spec_update_param",S=t.UPDATE_EMPTY_PARAM_INCLUSION="spec_update_empty_param_inclusion",C=t.VALIDATE_PARAMS="spec_validate_param",k=t.SET_RESPONSE="spec_set_response",A=t.SET_REQUEST="spec_set_request",O=t.SET_MUTATED_REQUEST="spec_set_mutated_request",P=t.LOG_REQUEST="spec_log_request",T=t.CLEAR_RESPONSE="spec_clear_response",M=t.CLEAR_REQUEST="spec_clear_request",I=t.CLEAR_VALIDATE_PARAMS="spec_clear_validate_param",j=t.UPDATE_OPERATION_META_VALUE="spec_update_operation_meta_value",N=t.UPDATE_RESOLVED="spec_update_resolved",R=t.UPDATE_RESOLVED_SUBTREE="spec_update_resolved_subtree",D=t.SET_SCHEME="set_scheme",L=function(e){return(0,v.default)(e)?e:""};t.parseToJson=function(e){return function(t){var n=t.specActions,r=t.specSelectors,o=t.errActions,i=r.specStr,a=null;try{e=e||i(),o.clear({source:"parser"}),a=f.default.safeLoad(e)}catch(e){return console.error(e),o.newSpecErr({source:"parser",level:"error",message:e.reason,line:e.mark&&e.mark.line?e.mark.line+1:void 0})}return a&&"object"===(void 0===a?"undefined":(0,c.default)(a))?n.updateJsonSpec(a):{}}};var U=!1,q=(t.resolveSpec=function(e,t){return function(n){var r=n.specActions,o=n.specSelectors,i=n.errActions,a=n.fn,u=a.fetch,s=a.resolve,l=a.AST,c=void 0===l?{}:l,f=n.getConfigs;U||(console.warn("specActions.resolveSpec is deprecated since v3.10.0 and will be removed in v4.0.0; use requestResolvedSubtree instead!"),U=!0);var p=f(),d=p.modelPropertyMacro,h=p.parameterMacro,v=p.requestInterceptor,m=p.responseInterceptor;void 0===e&&(e=o.specJson()),void 0===t&&(t=o.url());var g=c.getLineNumberForPath?c.getLineNumberForPath:function(){},y=o.specStr();return s({fetch:u,spec:e,baseDoc:t,modelPropertyMacro:d,parameterMacro:h,requestInterceptor:v,responseInterceptor:m}).then(function(e){var t=e.spec,n=e.errors;if(i.clear({type:"thrown"}),Array.isArray(n)&&n.length>0){var o=n.map(function(e){return console.error(e),e.line=e.fullPath?g(y,e.fullPath):null,e.path=e.fullPath?e.fullPath.join("."):null,e.level="error",e.type="thrown",e.source="resolver",Object.defineProperty(e,"message",{enumerable:!0,value:e.message}),e});i.newThrownErrBatch(o)}return r.updateResolved(t)})}},[]),F=(0,m.default)((0,l.default)(s.default.mark(function e(){var t,n,r,o,i,a,c,f,d,h,v,m,y,b,_,w,E;return s.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:if(t=q.system){e.next=4;break}return console.error("debResolveSubtrees: don't have a system to operate on, aborting."),e.abrupt("return");case 4:if(n=t.errActions,r=t.errSelectors,o=t.fn,i=o.resolveSubtree,a=o.AST,c=void 0===a?{}:a,f=t.specSelectors,d=t.specActions,i){e.next=8;break}return console.error("Error: Swagger-Client did not provide a `resolveSubtree` method, doing nothing."),e.abrupt("return");case 8:return h=c.getLineNumberForPath?c.getLineNumberForPath:function(){},v=f.specStr(),m=t.getConfigs(),y=m.modelPropertyMacro,b=m.parameterMacro,_=m.requestInterceptor,w=m.responseInterceptor,e.prev=11,e.next=14,q.reduce(function(){var e=(0,l.default)(s.default.mark(function e(t,o){var a,u,l,c,p,d,m;return s.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:return e.next=2,t;case 2:return a=e.sent,u=a.resultMap,l=a.specWithCurrentSubtrees,e.next=7,i(l,o,{baseDoc:f.url(),modelPropertyMacro:y,parameterMacro:b,requestInterceptor:_,responseInterceptor:w});case 7:return c=e.sent,p=c.errors,d=c.spec,r.allErrors().size&&n.clearBy(function(e){return"thrown"!==e.get("type")||"resolver"!==e.get("source")||!e.get("fullPath").every(function(e,t){return e===o[t]||void 0===o[t]})}),Array.isArray(p)&&p.length>0&&(m=p.map(function(e){return e.line=e.fullPath?h(v,e.fullPath):null,e.path=e.fullPath?e.fullPath.join("."):null,e.level="error",e.type="thrown",e.source="resolver",Object.defineProperty(e,"message",{enumerable:!0,value:e.message}),e}),n.newThrownErrBatch(m)),(0,g.default)(u,o,d),(0,g.default)(l,o,d),e.abrupt("return",{resultMap:u,specWithCurrentSubtrees:l});case 15:case"end":return e.stop()}},e,void 0)}));return function(t,n){return e.apply(this,arguments)}}(),u.default.resolve({resultMap:(f.specResolvedSubtree([])||(0,p.Map)()).toJS(),specWithCurrentSubtrees:f.specJson().toJS()}));case 14:E=e.sent,delete q.system,q=[],e.next=22;break;case 19:e.prev=19,e.t0=e.catch(11),console.error(e.t0);case 22:d.updateResolvedSubtree([],E.resultMap);case 23:case"end":return e.stop()}},e,void 0,[[11,19]])})),35);t.requestResolvedSubtree=function(e){return function(t){q.map(function(e){return e.join("@@")}).indexOf(e.join("@@"))>-1||(q.push(e),q.system=t,F())}};t.updateResolvedSubtree=function(e,t){return{type:R,payload:{path:e,value:t}}},t.invalidateResolvedSubtreeCache=function(){return{type:R,payload:{path:[],value:(0,p.Map)()}}},t.validateParams=function(e,t){return{type:C,payload:{pathMethod:e,isOAS3:t}}},t.updateEmptyParamInclusion=function(e,t,n,r){return{type:S,payload:{pathMethod:e,paramName:t,paramIn:n,includeEmptyValue:r}}};t.setResponse=function(e,t,n){return{payload:{path:e,method:t,res:n},type:k}},t.setRequest=function(e,t,n){return{payload:{path:e,method:t,req:n},type:A}},t.setMutatedRequest=function(e,t,n){return{payload:{path:e,method:t,req:n},type:O}},t.logRequest=function(e){return{payload:e,type:P}},t.executeRequest=function(e){return function(t){var n=t.fn,r=t.specActions,o=t.specSelectors,u=t.getConfigs,s=t.oas3Selectors,l=e.pathName,c=e.method,f=e.operation,p=u(),v=p.requestInterceptor,m=p.responseInterceptor,g=f.toJS();if(f&&f.get("parameters")&&f.get("parameters").filter(function(e){return e&&!0===e.get("allowEmptyValue")}).forEach(function(t){if(o.parameterInclusionSettingFor([l,c],t.get("name"),t.get("in"))){e.parameters=e.parameters||{};var n=(0,y.paramToValue)(t,e.parameters);(!n||n&&0===n.size)&&(e.parameters[t.get("name")]="")}}),e.contextUrl=(0,d.default)(o.url()).toString(),g&&g.operationId?e.operationId=g.operationId:g&&l&&c&&(e.operationId=n.opId(g,l,c)),o.isOAS3()){var b=l+":"+c;e.server=s.selectedServer(b)||s.selectedServer();var _=s.serverVariables({server:e.server,namespace:b}).toJS(),w=s.serverVariables({server:e.server}).toJS();e.serverVariables=(0,a.default)(_).length?_:w,e.requestContentType=s.requestContentType(l,c),e.responseContentType=s.responseContentType(l,c)||"*/*";var E=s.requestBodyValue(l,c);(0,y.isJSONObject)(E)?e.requestBody=JSON.parse(E):E&&E.toJS?e.requestBody=E.toJS():e.requestBody=E}var x=(0,i.default)({},e);x=n.buildRequest(x),r.setRequest(e.pathName,e.method,x);e.requestInterceptor=function(t){var n=v.apply(this,[t]),o=(0,i.default)({},n);return r.setMutatedRequest(e.pathName,e.method,o),n},e.responseInterceptor=m;var S=Date.now();return n.execute(e).then(function(t){t.duration=Date.now()-S,r.setResponse(e.pathName,e.method,t)}).catch(function(t){return r.setResponse(e.pathName,e.method,{error:!0,err:(0,h.default)(t)})})}};t.execute=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e.path,n=e.method,i=(0,o.default)(e,["path","method"]);return function(e){var o=e.fn.fetch,a=e.specSelectors,u=e.specActions,s=a.specJsonWithResolvedSubtrees().toJS(),l=a.operationScheme(t,n),c=a.contentTypeValues([t,n]).toJS(),f=c.requestContentType,p=c.responseContentType,d=/xml/i.test(f),h=a.parameterValues([t,n],d).toJS();return u.executeRequest((0,r.default)({},i,{fetch:o,spec:s,pathName:t,method:n,parameters:h,requestContentType:f,scheme:l,responseContentType:p}))}}},function(e,t,n){e.exports={default:n(733),__esModule:!0}},function(e,t){e.exports=function(e,t,n,r){if(!(e instanceof t)||void 0!==r&&r in e)throw TypeError(n+": incorrect invocation!");return e}},function(e,t,n){"use strict";var r=n(94);e.exports.f=function(e){return new function(e){var t,n;this.promise=new e(function(e,r){if(void 0!==t||void 0!==n)throw TypeError("Bad Promise constructor");t=e,n=r}),this.resolve=r(t),this.reject=r(n)}(e)}},function(e,t,n){var r=n(50);e.exports=function(e,t,n){for(var o in t)n&&e[o]?e[o]=t[o]:r(e,o,t[o]);return e}},function(e,t,n){"use strict";var r=n(742);e.exports=r},function(e,t,n){"use strict";var r=n(86);e.exports=new r({explicit:[n(745),n(746),n(747)]})},function(e,t,n){"use strict";(function(t){var r=n(762),o=n(763),i=/^([a-z][a-z0-9.+-]*:)?(\/\/)?([\S\s]*)/i,a=/^[A-Za-z][A-Za-z0-9+-.]*:\/\//,u=[["#","hash"],["?","query"],function(e){return e.replace("\\","/")},["/","pathname"],["@","auth",1],[NaN,"host",void 0,1,1],[/:(\d+)$/,"port",void 0,1],[NaN,"hostname",void 0,1,1]],s={hash:1,query:1};function l(e){var n,r=t&&t.location||{},o={},i=typeof(e=e||r);if("blob:"===e.protocol)o=new f(unescape(e.pathname),{});else if("string"===i)for(n in o=new f(e,{}),s)delete o[n];else if("object"===i){for(n in e)n in s||(o[n]=e[n]);void 0===o.slashes&&(o.slashes=a.test(e.href))}return o}function c(e){var t=i.exec(e);return{protocol:t[1]?t[1].toLowerCase():"",slashes:!!t[2],rest:t[3]}}function f(e,t,n){if(!(this instanceof f))return new f(e,t,n);var i,a,s,p,d,h,v=u.slice(),m=typeof t,g=this,y=0;for("object"!==m&&"string"!==m&&(n=t,t=null),n&&"function"!=typeof n&&(n=o.parse),t=l(t),i=!(a=c(e||"")).protocol&&!a.slashes,g.slashes=a.slashes||i&&t.slashes,g.protocol=a.protocol||t.protocol||"",e=a.rest,a.slashes||(v[3]=[/(.*)/,"pathname"]);y-1||r("96",e),!l.plugins[n]){t.extractEvents||r("97",e),l.plugins[n]=t;var a=t.eventTypes;for(var s in a)u(a[s],t,s)||r("98",s,e)}}}function u(e,t,n){l.eventNameDispatchConfigs.hasOwnProperty(n)&&r("99",n),l.eventNameDispatchConfigs[n]=e;var o=e.phasedRegistrationNames;if(o){for(var i in o){if(o.hasOwnProperty(i))s(o[i],t,n)}return!0}return!!e.registrationName&&(s(e.registrationName,t,n),!0)}function s(e,t,n){l.registrationNameModules[e]&&r("100",e),l.registrationNameModules[e]=t,l.registrationNameDependencies[e]=t.eventTypes[n].dependencies}var l={plugins:[],eventNameDispatchConfigs:{},registrationNameModules:{},registrationNameDependencies:{},possibleRegistrationNames:null,injectEventPluginOrder:function(e){o&&r("101"),o=Array.prototype.slice.call(e),a()},injectEventPluginsByName:function(e){var t=!1;for(var n in e)if(e.hasOwnProperty(n)){var o=e[n];i.hasOwnProperty(n)&&i[n]===o||(i[n]&&r("102",n),i[n]=o,t=!0)}t&&a()},getPluginModuleForEvent:function(e){var t=e.dispatchConfig;if(t.registrationName)return l.registrationNameModules[t.registrationName]||null;if(void 0!==t.phasedRegistrationNames){var n=t.phasedRegistrationNames;for(var r in n)if(n.hasOwnProperty(r)){var o=l.registrationNameModules[n[r]];if(o)return o}}return null},_resetEventPlugins:function(){for(var e in o=null,i)i.hasOwnProperty(e)&&delete i[e];l.plugins.length=0;var t=l.eventNameDispatchConfigs;for(var n in t)t.hasOwnProperty(n)&&delete t[n];var r=l.registrationNameModules;for(var a in r)r.hasOwnProperty(a)&&delete r[a]}};e.exports=l},function(e,t,n){"use strict";var r,o,i=n(11),a=n(213);n(8),n(10);function u(e,t,n,r){var o=e.type||"unknown-event";e.currentTarget=s.getNodeFromInstance(r),t?a.invokeGuardedCallbackWithCatch(o,n,e):a.invokeGuardedCallback(o,n,e),e.currentTarget=null}var s={isEndish:function(e){return"topMouseUp"===e||"topTouchEnd"===e||"topTouchCancel"===e},isMoveish:function(e){return"topMouseMove"===e||"topTouchMove"===e},isStartish:function(e){return"topMouseDown"===e||"topTouchStart"===e},executeDirectDispatch:function(e){var t=e._dispatchListeners,n=e._dispatchInstances;Array.isArray(t)&&i("103"),e.currentTarget=t?s.getNodeFromInstance(n):null;var r=t?t(e):null;return e.currentTarget=null,e._dispatchListeners=null,e._dispatchInstances=null,r},executeDispatchesInOrder:function(e,t){var n=e._dispatchListeners,r=e._dispatchInstances;if(Array.isArray(n))for(var o=0;o0&&r.length<20?n+" (keys: "+r.join(", ")+")":n}function s(e,t){var n=o.get(e);return n||null}var l={isMounted:function(e){var t=o.get(e);return!!t&&!!t._renderedComponent},enqueueCallback:function(e,t,n){l.validateCallback(t,n);var r=s(e);if(!r)return null;r._pendingCallbacks?r._pendingCallbacks.push(t):r._pendingCallbacks=[t],a(r)},enqueueCallbackInternal:function(e,t){e._pendingCallbacks?e._pendingCallbacks.push(t):e._pendingCallbacks=[t],a(e)},enqueueForceUpdate:function(e){var t=s(e);t&&(t._pendingForceUpdate=!0,a(t))},enqueueReplaceState:function(e,t,n){var r=s(e);r&&(r._pendingStateQueue=[t],r._pendingReplaceState=!0,void 0!==n&&null!==n&&(l.validateCallback(n,"replaceState"),r._pendingCallbacks?r._pendingCallbacks.push(n):r._pendingCallbacks=[n]),a(r))},enqueueSetState:function(e,t){var n=s(e);n&&((n._pendingStateQueue||(n._pendingStateQueue=[])).push(t),a(n))},enqueueElementInternal:function(e,t,n){e._pendingElement=t,e._context=n,a(e)},validateCallback:function(e,t){e&&"function"!=typeof e&&r("122",t,u(e))}};e.exports=l},function(e,t,n){"use strict";n(13);var r=n(34),o=(n(10),r);e.exports=o},function(e,t,n){"use strict";e.exports=function(e){var t,n=e.keyCode;return"charCode"in e?0===(t=e.charCode)&&13===n&&(t=13):t=n,t>=32||13===t?t:0}},function(e,t,n){var r=n(62),o=n(229),i=n(47),a="[object Object]",u=Function.prototype,s=Object.prototype,l=u.toString,c=s.hasOwnProperty,f=l.call(Object);e.exports=function(e){if(!i(e)||r(e)!=a)return!1;var t=o(e);if(null===t)return!0;var n=c.call(t,"constructor")&&t.constructor;return"function"==typeof n&&n instanceof n&&l.call(n)==f}},function(e,t,n){var r=n(298)(Object.getPrototypeOf,Object);e.exports=r},function(e,t,n){var r=n(292);e.exports=function(e){var t=new e.constructor(e.byteLength);return new r(t).set(new r(e)),t}},function(e,t){var n=this&&this.__extends||function(e,t){for(var n in t)t.hasOwnProperty(n)&&(e[n]=t[n]);function r(){this.constructor=e}e.prototype=null===t?Object.create(t):(r.prototype=t.prototype,new r)},r=Object.prototype.hasOwnProperty; +/*! + * https://github.com/Starcounter-Jack/JSON-Patch + * (c) 2017 Joachim Wester + * MIT license + */function o(e,t){return r.call(e,t)}function i(e){if(Array.isArray(e)){for(var t=new Array(e.length),n=0;n=48&&t<=57))return!1;n++}return!0},t.escapePathComponent=a,t.unescapePathComponent=function(e){return e.replace(/~1/g,"/").replace(/~0/g,"~")},t._getPathRecursive=u,t.getPath=function(e,t){if(e===t)return"/";var n=u(e,t);if(""===n)throw new Error("Object not found in root");return"/"+n},t.hasUndefined=function e(t){if(void 0===t)return!0;if(t)if(Array.isArray(t)){for(var n=0,r=t.length;nw;w++)if((p||w in y)&&(m=b(v=y[w],w,g),e))if(n)E[w]=m;else if(m)switch(e){case 3:return!0;case 5:return v;case 6:return w;case 2:E.push(v)}else if(c)return!1;return f?-1:l||c?c:E}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.authorizeRequest=t.authorizeAccessCodeWithBasicAuthentication=t.authorizeAccessCodeWithFormParams=t.authorizeApplication=t.authorizePassword=t.preAuthorizeImplicit=t.CONFIGURE_AUTH=t.VALIDATE=t.AUTHORIZE_OAUTH2=t.PRE_AUTHORIZE_OAUTH2=t.LOGOUT=t.AUTHORIZE=t.SHOW_AUTH_POPUP=void 0;var r=l(n(45)),o=l(n(23)),i=l(n(41));t.showDefinitions=function(e){return{type:c,payload:e}},t.authorize=function(e){return{type:f,payload:e}},t.logout=function(e){return{type:p,payload:e}},t.authorizeOauth2=function(e){return{type:d,payload:e}},t.configureAuth=function(e){return{type:h,payload:e}};var a=l(n(210)),u=l(n(32)),s=n(9);function l(e){return e&&e.__esModule?e:{default:e}}var c=t.SHOW_AUTH_POPUP="show_popup",f=t.AUTHORIZE="authorize",p=t.LOGOUT="logout",d=(t.PRE_AUTHORIZE_OAUTH2="pre_authorize_oauth2",t.AUTHORIZE_OAUTH2="authorize_oauth2"),h=(t.VALIDATE="validate",t.CONFIGURE_AUTH="configure_auth");t.preAuthorizeImplicit=function(e){return function(t){var n=t.authActions,r=t.errActions,o=e.auth,a=e.token,s=e.isValid,l=o.schema,c=o.name,f=l.get("flow");delete u.default.swaggerUIRedirectOauth2,"accessCode"===f||s||r.newAuthErr({authId:c,source:"auth",level:"warning",message:"Authorization may be unsafe, passed state was changed in server Passed state wasn't returned from auth server"}),a.error?r.newAuthErr({authId:c,source:"auth",level:"error",message:(0,i.default)(a)}):n.authorizeOauth2({auth:o,token:a})}};t.authorizePassword=function(e){return function(t){var n=t.authActions,r=e.schema,i=e.name,a=e.username,u=e.password,l=e.passwordType,c=e.clientId,f=e.clientSecret,p={grant_type:"password",scope:e.scopes.join(" "),username:a,password:u},d={};switch(l){case"request-body":!function(e,t,n){t&&(0,o.default)(e,{client_id:t});n&&(0,o.default)(e,{client_secret:n})}(p,c,f);break;case"basic":d.Authorization="Basic "+(0,s.btoa)(c+":"+f);break;default:console.warn("Warning: invalid passwordType "+l+" was passed, not including client id and secret")}return n.authorizeRequest({body:(0,s.buildFormData)(p),url:r.get("tokenUrl"),name:i,headers:d,query:{},auth:e})}};t.authorizeApplication=function(e){return function(t){var n=t.authActions,r=e.schema,o=e.scopes,i=e.name,a=e.clientId,u=e.clientSecret,l={Authorization:"Basic "+(0,s.btoa)(a+":"+u)},c={grant_type:"client_credentials",scope:o.join(" ")};return n.authorizeRequest({body:(0,s.buildFormData)(c),name:i,url:r.get("tokenUrl"),auth:e,headers:l})}},t.authorizeAccessCodeWithFormParams=function(e){var t=e.auth,n=e.redirectUrl;return function(e){var r=e.authActions,o=t.schema,i=t.name,a=t.clientId,u=t.clientSecret,l={grant_type:"authorization_code",code:t.code,client_id:a,client_secret:u,redirect_uri:n};return r.authorizeRequest({body:(0,s.buildFormData)(l),name:i,url:o.get("tokenUrl"),auth:t})}},t.authorizeAccessCodeWithBasicAuthentication=function(e){var t=e.auth,n=e.redirectUrl;return function(e){var r=e.authActions,o=t.schema,i=t.name,a=t.clientId,u=t.clientSecret,l={Authorization:"Basic "+(0,s.btoa)(a+":"+u)},c={grant_type:"authorization_code",code:t.code,client_id:a,redirect_uri:n};return r.authorizeRequest({body:(0,s.buildFormData)(c),name:i,url:o.get("tokenUrl"),auth:t,headers:l})}},t.authorizeRequest=function(e){return function(t){var n=t.fn,u=t.getConfigs,s=t.authActions,l=t.errActions,c=t.oas3Selectors,f=t.specSelectors,p=t.authSelectors,d=e.body,h=e.query,v=void 0===h?{}:h,m=e.headers,g=void 0===m?{}:m,y=e.name,b=e.url,_=e.auth,w=(p.getConfigs()||{}).additionalQueryStringParams,E=void 0;E=f.isOAS3()?(0,a.default)(b,c.selectedServer(),!0):(0,a.default)(b,f.url(),!0),"object"===(void 0===w?"undefined":(0,r.default)(w))&&(E.query=(0,o.default)({},E.query,w));var x=E.toString(),S=(0,o.default)({Accept:"application/json, text/plain, */*","Content-Type":"application/x-www-form-urlencoded","X-Requested-With":"XMLHttpRequest"},g);n.fetch({url:x,method:"post",headers:S,query:v,body:d,requestInterceptor:u().requestInterceptor,responseInterceptor:u().responseInterceptor}).then(function(e){var t=JSON.parse(e.data),n=t&&(t.error||""),r=t&&(t.parseError||"");e.ok?n||r?l.newAuthErr({authId:y,level:"error",source:"auth",message:(0,i.default)(t)}):s.authorizeOauth2({auth:_,token:t}):l.newAuthErr({authId:y,level:"error",source:"auth",message:e.statusText})}).catch(function(e){var t=new Error(e).message;if(e.response&&e.response.data){var n=e.response.data;try{var r="string"==typeof n?JSON.parse(n):n;r.error&&(t+=", error: "+r.error),r.error_description&&(t+=", description: "+r.error_description)}catch(e){}}l.newAuthErr({authId:y,level:"error",source:"auth",message:t})})}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.parseYamlConfig=void 0;var r,o=n(208),i=(r=o)&&r.__esModule?r:{default:r};t.parseYamlConfig=function(e,t){try{return i.default.safeLoad(e)}catch(e){return t&&t.errActions.newThrownErr(new Error(e)),{}}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.loaded=t.TOGGLE_CONFIGS=t.UPDATE_CONFIGS=void 0;var r,o=n(22),i=(r=o)&&r.__esModule?r:{default:r};t.update=function(e,t){return{type:a,payload:(0,i.default)({},e,t)}},t.toggle=function(e){return{type:u,payload:e}};var a=t.UPDATE_CONFIGS="configs_update",u=t.TOGGLE_CONFIGS="configs_toggle";t.loaded=function(){return function(){}}},function(e,t,n){"use strict";function r(e,t,n,r,o){this.src=e,this.env=r,this.options=n,this.parser=t,this.tokens=o,this.pos=0,this.posMax=this.src.length,this.level=0,this.pending="",this.pendingLevel=0,this.cache=[],this.isInLabel=!1,this.linkLevel=0,this.linkContent="",this.labelUnmatchedScopes=0}r.prototype.pushPending=function(){this.tokens.push({type:"text",content:this.pending,level:this.pendingLevel}),this.pending=""},r.prototype.push=function(e){this.pending&&this.pushPending(),this.tokens.push(e),this.pendingLevel=this.level},r.prototype.cacheSet=function(e,t){for(var n=this.cache.length;n<=e;n++)this.cache.push(0);this.cache[e]=t},r.prototype.cacheGet=function(e){return es;)r(u,n=t[s++])&&(~i(l,n)||l.push(n));return l}},function(e,t,n){var r=n(21).document;e.exports=r&&r.documentElement},function(e,t,n){var r=n(52),o=n(72),i=n(162)("IE_PROTO"),a=Object.prototype;e.exports=Object.getPrototypeOf||function(e){return e=o(e),r(e,i)?e[i]:"function"==typeof e.constructor&&e instanceof e.constructor?e.constructor.prototype:e instanceof Object?a:null}},function(e,t,n){var r=n(33),o=r["__core-js_shared__"]||(r["__core-js_shared__"]={});e.exports=function(e){return o[e]||(o[e]={})}},function(e,t){e.exports=function(e,t){return{enumerable:!(1&e),configurable:!(2&e),writable:!(4&e),value:t}}},function(e,t,n){"use strict";var r=n(246)(!0);n(247)(String,"String",function(e){this._t=String(e),this._i=0},function(){var e,t=this._t,n=this._i;return n>=t.length?{value:void 0,done:!0}:(e=r(t,n),this._i+=e.length,{value:e,done:!1})})},function(e,t,n){var r=n(119),o=n(53);e.exports=function(e){return function(t,n){var i,a,u=String(o(t)),s=r(n),l=u.length;return s<0||s>=l?e?"":void 0:(i=u.charCodeAt(s))<55296||i>56319||s+1===l||(a=u.charCodeAt(s+1))<56320||a>57343?e?u.charAt(s):i:e?u.slice(s,s+2):a-56320+(i-55296<<10)+65536}}},function(e,t,n){"use strict";var r=n(248),o=n(29),i=n(73),a=n(59),u=n(102),s=n(462),l=n(171),c=n(468),f=n(18)("iterator"),p=!([].keys&&"next"in[].keys()),d=function(){return this};e.exports=function(e,t,n,h,v,m,g){s(n,t,h);var y,b,_,w=function(e){if(!p&&e in C)return C[e];switch(e){case"keys":case"values":return function(){return new n(this,e)}}return function(){return new n(this,e)}},E=t+" Iterator",x="values"==v,S=!1,C=e.prototype,k=C[f]||C["@@iterator"]||v&&C[v],A=k||w(v),O=v?x?w("entries"):A:void 0,P="Array"==t&&C.entries||k;if(P&&(_=c(P.call(new e)))!==Object.prototype&&_.next&&(l(_,E,!0),r||"function"==typeof _[f]||a(_,f,d)),x&&k&&"values"!==k.name&&(S=!0,A=function(){return k.call(this)}),r&&!g||!p&&!S&&C[f]||a(C,f,A),u[t]=A,u[E]=d,v)if(y={values:x?A:w("values"),keys:m?A:w("keys"),entries:O},g)for(b in y)b in C||i(C,b,y[b]);else o(o.P+o.F*(p||S),t,y);return y}},function(e,t){e.exports=!1},function(e,t,n){var r=n(465),o=n(251);e.exports=Object.keys||function(e){return r(e,o)}},function(e,t,n){var r=n(119),o=Math.max,i=Math.min;e.exports=function(e,t){return(e=r(e))<0?o(e+t,0):i(e,t)}},function(e,t){e.exports="constructor,hasOwnProperty,isPrototypeOf,propertyIsEnumerable,toLocaleString,toString,valueOf".split(",")},function(e,t,n){var r=n(33).document;e.exports=r&&r.documentElement},function(e,t,n){var r=n(60),o=n(121),i=n(18)("species");e.exports=function(e,t){var n,a=r(e).constructor;return void 0===a||void 0==(n=r(a)[i])?t:o(n)}},function(e,t,n){var r,o,i,a=n(120),u=n(480),s=n(252),l=n(169),c=n(33),f=c.process,p=c.setImmediate,d=c.clearImmediate,h=c.MessageChannel,v=c.Dispatch,m=0,g={},y=function(){var e=+this;if(g.hasOwnProperty(e)){var t=g[e];delete g[e],t()}},b=function(e){y.call(e.data)};p&&d||(p=function(e){for(var t=[],n=1;arguments.length>n;)t.push(arguments[n++]);return g[++m]=function(){u("function"==typeof e?e:Function(e),t)},r(m),m},d=function(e){delete g[e]},"process"==n(99)(f)?r=function(e){f.nextTick(a(y,e,1))}:v&&v.now?r=function(e){v.now(a(y,e,1))}:h?(i=(o=new h).port2,o.port1.onmessage=b,r=a(i.postMessage,i,1)):c.addEventListener&&"function"==typeof postMessage&&!c.importScripts?(r=function(e){c.postMessage(e+"","*")},c.addEventListener("message",b,!1)):r="onreadystatechange"in l("script")?function(e){s.appendChild(l("script")).onreadystatechange=function(){s.removeChild(this),y.call(e)}}:function(e){setTimeout(a(y,e,1),0)}),e.exports={set:p,clear:d}},function(e,t){e.exports=function(e){try{return{e:!1,v:e()}}catch(e){return{e:!0,v:e}}}},function(e,t,n){var r=n(60),o=n(74),i=n(172);e.exports=function(e,t){if(r(e),o(t)&&t.constructor===e)return t;var n=i.f(e);return(0,n.resolve)(t),n.promise}},function(e,t,n){var r=n(74),o=n(99),i=n(18)("match");e.exports=function(e){var t;return r(e)&&(void 0!==(t=e[i])?!!t:"RegExp"==o(e))}},function(e,t,n){var r=n(20),o=n(15),i=n(51);e.exports=function(e,t){var n=(o.Object||{})[e]||Object[e],a={};a[e]=t(n),r(r.S+r.F*i(function(){n(1)}),"Object",a)}},function(e,t,n){var r=n(93);e.exports=Array.isArray||function(e){return"Array"==r(e)}},function(e,t,n){var r=n(240),o=n(164).concat("length","prototype");t.f=Object.getOwnPropertyNames||function(e){return r(e,o)}},function(e,t,n){var r=n(125),o=n(95),i=n(71),a=n(158),u=n(52),s=n(239),l=Object.getOwnPropertyDescriptor;t.f=n(44)?l:function(e,t){if(e=i(e),t=a(t,!0),s)try{return l(e,t)}catch(e){}if(u(e,t))return o(!r.f.call(e,t),e[t])}},function(e,t){var n={}.toString;e.exports=Array.isArray||function(e){return"[object Array]"==n.call(e)}},function(e,t,n){e.exports={default:n(532),__esModule:!0}},function(e,t,n){"use strict";var r=n(96),o=n(177),i=n(125),a=n(72),u=n(155),s=Object.assign;e.exports=!s||n(51)(function(){var e={},t={},n=Symbol(),r="abcdefghijklmnopqrst";return e[n]=7,r.split("").forEach(function(e){t[e]=e}),7!=s({},e)[n]||Object.keys(s({},t)).join("")!=r})?function(e,t){for(var n=a(e),s=arguments.length,l=1,c=o.f,f=i.f;s>l;)for(var p,d=u(arguments[l++]),h=c?r(d).concat(c(d)):r(d),v=h.length,m=0;v>m;)f.call(d,p=h[m++])&&(n[p]=d[p]);return n}:s},function(e,t,n){"use strict";var r=n(104),o=n(13),i=n(266),a=(n(267),n(126));n(8),n(536);function u(e,t,n){this.props=e,this.context=t,this.refs=a,this.updater=n||i}function s(e,t,n){this.props=e,this.context=t,this.refs=a,this.updater=n||i}function l(){}u.prototype.isReactComponent={},u.prototype.setState=function(e,t){"object"!=typeof e&&"function"!=typeof e&&null!=e&&r("85"),this.updater.enqueueSetState(this,e),t&&this.updater.enqueueCallback(this,t,"setState")},u.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this),e&&this.updater.enqueueCallback(this,e,"forceUpdate")},l.prototype=u.prototype,s.prototype=new l,s.prototype.constructor=s,o(s.prototype,u.prototype),s.prototype.isPureReactComponent=!0,e.exports={Component:u,PureComponent:s}},function(e,t,n){"use strict";n(10);var r={isMounted:function(e){return!1},enqueueCallback:function(e,t){},enqueueForceUpdate:function(e){},enqueueReplaceState:function(e,t){},enqueueSetState:function(e,t){}};e.exports=r},function(e,t,n){"use strict";var r=!1;e.exports=r},function(e,t,n){"use strict";var r="function"==typeof Symbol&&Symbol.for&&Symbol.for("react.element")||60103;e.exports=r},function(e,t,n){"use strict";var r=n(544);e.exports=function(e){return r(e,!1)}},function(e,t,n){"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=n(272),o=n(561),i=n(562),a=n(563),u=n(276);n(275);n.d(t,"createStore",function(){return r.b}),n.d(t,"combineReducers",function(){return o.a}),n.d(t,"bindActionCreators",function(){return i.a}),n.d(t,"applyMiddleware",function(){return a.a}),n.d(t,"compose",function(){return u.a})},function(e,t,n){"use strict";n.d(t,"a",function(){return i}),t.b=function e(t,n,a){var u;"function"==typeof n&&void 0===a&&(a=n,n=void 0);if(void 0!==a){if("function"!=typeof a)throw new Error("Expected the enhancer to be a function.");return a(e)(t,n)}if("function"!=typeof t)throw new Error("Expected the reducer to be a function.");var s=t;var l=n;var c=[];var f=c;var p=!1;function d(){f===c&&(f=c.slice())}function h(){return l}function v(e){if("function"!=typeof e)throw new Error("Expected listener to be a function.");var t=!0;return d(),f.push(e),function(){if(t){t=!1,d();var n=f.indexOf(e);f.splice(n,1)}}}function m(e){if(!r.a(e))throw new Error("Actions must be plain objects. Use custom middleware for async actions.");if(void 0===e.type)throw new Error('Actions may not have an undefined "type" property. Have you misspelled a constant?');if(p)throw new Error("Reducers may not dispatch actions.");try{p=!0,l=s(l,e)}finally{p=!1}for(var t=c=f,n=0;no?0:o+t),(n=n>o?o:n)<0&&(n+=o),o=t>n?0:n-t>>>0,t>>>=0;for(var i=Array(o);++rp))return!1;var h=c.get(e);if(h&&c.get(t))return h==t;var v=-1,m=!0,g=n&u?new r:void 0;for(c.set(e,t),c.set(t,e);++v0?("string"==typeof t||a.objectMode||Object.getPrototypeOf(t)===l.prototype||(t=function(e){return l.from(e)}(t)),r?a.endEmitted?e.emit("error",new Error("stream.unshift() after end event")):w(e,a,t,!0):a.ended?e.emit("error",new Error("stream.push() after EOF")):(a.reading=!1,a.decoder&&!n?(t=a.decoder.write(t),a.objectMode||0!==t.length?w(e,a,t,!1):k(e,a)):w(e,a,t,!1))):r||(a.reading=!1));return function(e){return!e.ended&&(e.needReadable||e.lengtht.highWaterMark&&(t.highWaterMark=function(e){return e>=E?e=E:(e--,e|=e>>>1,e|=e>>>2,e|=e>>>4,e|=e>>>8,e|=e>>>16,e++),e}(e)),e<=t.length?e:t.ended?t.length:(t.needReadable=!0,0))}function S(e){var t=e._readableState;t.needReadable=!1,t.emittedReadable||(d("emitReadable",t.flowing),t.emittedReadable=!0,t.sync?o.nextTick(C,e):C(e))}function C(e){d("emit readable"),e.emit("readable"),T(e)}function k(e,t){t.readingMore||(t.readingMore=!0,o.nextTick(A,e,t))}function A(e,t){for(var n=t.length;!t.reading&&!t.flowing&&!t.ended&&t.length=t.length?(n=t.decoder?t.buffer.join(""):1===t.buffer.length?t.buffer.head.data:t.buffer.concat(t.length),t.buffer.clear()):n=function(e,t,n){var r;ei.length?i.length:e;if(a===i.length?o+=i:o+=i.slice(0,e),0===(e-=a)){a===i.length?(++r,n.next?t.head=n.next:t.head=t.tail=null):(t.head=n,n.data=i.slice(a));break}++r}return t.length-=r,o}(e,t):function(e,t){var n=l.allocUnsafe(e),r=t.head,o=1;r.data.copy(n),e-=r.data.length;for(;r=r.next;){var i=r.data,a=e>i.length?i.length:e;if(i.copy(n,n.length-e,0,a),0===(e-=a)){a===i.length?(++o,r.next?t.head=r.next:t.head=t.tail=null):(t.head=r,r.data=i.slice(a));break}++o}return t.length-=o,n}(e,t);return r}(e,t.buffer,t.decoder),n);var n}function I(e){var t=e._readableState;if(t.length>0)throw new Error('"endReadable()" called on non-empty stream');t.endEmitted||(t.ended=!0,o.nextTick(j,t,e))}function j(e,t){e.endEmitted||0!==e.length||(e.endEmitted=!0,t.readable=!1,t.emit("end"))}function N(e,t){for(var n=0,r=e.length;n=t.highWaterMark||t.ended))return d("read: emitReadable",t.length,t.ended),0===t.length&&t.ended?I(this):S(this),null;if(0===(e=x(e,t))&&t.ended)return 0===t.length&&I(this),null;var r,o=t.needReadable;return d("need readable",o),(0===t.length||t.length-e0?M(e,t):null)?(t.needReadable=!0,e=0):t.length-=e,0===t.length&&(t.ended||(t.needReadable=!0),n!==e&&t.ended&&I(this)),null!==r&&this.emit("data",r),r},b.prototype._read=function(e){this.emit("error",new Error("_read() is not implemented"))},b.prototype.pipe=function(e,t){var n=this,i=this._readableState;switch(i.pipesCount){case 0:i.pipes=e;break;case 1:i.pipes=[i.pipes,e];break;default:i.pipes.push(e)}i.pipesCount+=1,d("pipe count=%d opts=%j",i.pipesCount,t);var s=(!t||!1!==t.end)&&e!==r.stdout&&e!==r.stderr?c:b;function l(t,r){d("onunpipe"),t===n&&r&&!1===r.hasUnpiped&&(r.hasUnpiped=!0,d("cleanup"),e.removeListener("close",g),e.removeListener("finish",y),e.removeListener("drain",f),e.removeListener("error",m),e.removeListener("unpipe",l),n.removeListener("end",c),n.removeListener("end",b),n.removeListener("data",v),p=!0,!i.awaitDrain||e._writableState&&!e._writableState.needDrain||f())}function c(){d("onend"),e.end()}i.endEmitted?o.nextTick(s):n.once("end",s),e.on("unpipe",l);var f=function(e){return function(){var t=e._readableState;d("pipeOnDrain",t.awaitDrain),t.awaitDrain&&t.awaitDrain--,0===t.awaitDrain&&u(e,"data")&&(t.flowing=!0,T(e))}}(n);e.on("drain",f);var p=!1;var h=!1;function v(t){d("ondata"),h=!1,!1!==e.write(t)||h||((1===i.pipesCount&&i.pipes===e||i.pipesCount>1&&-1!==N(i.pipes,e))&&!p&&(d("false write response, pause",n._readableState.awaitDrain),n._readableState.awaitDrain++,h=!0),n.pause())}function m(t){d("onerror",t),b(),e.removeListener("error",m),0===u(e,"error")&&e.emit("error",t)}function g(){e.removeListener("finish",y),b()}function y(){d("onfinish"),e.removeListener("close",g),b()}function b(){d("unpipe"),n.unpipe(e)}return n.on("data",v),function(e,t,n){if("function"==typeof e.prependListener)return e.prependListener(t,n);e._events&&e._events[t]?a(e._events[t])?e._events[t].unshift(n):e._events[t]=[n,e._events[t]]:e.on(t,n)}(e,"error",m),e.once("close",g),e.once("finish",y),e.emit("pipe",n),i.flowing||(d("pipe resume"),n.resume()),e},b.prototype.unpipe=function(e){var t=this._readableState,n={hasUnpiped:!1};if(0===t.pipesCount)return this;if(1===t.pipesCount)return e&&e!==t.pipes?this:(e||(e=t.pipes),t.pipes=null,t.pipesCount=0,t.flowing=!1,e&&e.emit("unpipe",this,n),this);if(!e){var r=t.pipes,o=t.pipesCount;t.pipes=null,t.pipesCount=0,t.flowing=!1;for(var i=0;i=0&&(e._idleTimeoutId=setTimeout(function(){e._onTimeout&&e._onTimeout()},t))},n(663),t.setImmediate="undefined"!=typeof self&&self.setImmediate||void 0!==e&&e.setImmediate||this&&this.setImmediate,t.clearImmediate="undefined"!=typeof self&&self.clearImmediate||void 0!==e&&e.clearImmediate||this&&this.clearImmediate}).call(t,n(31))},function(e,t,n){"use strict";var r=n(141).Buffer,o=r.isEncoding||function(e){switch((e=""+e)&&e.toLowerCase()){case"hex":case"utf8":case"utf-8":case"ascii":case"binary":case"base64":case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":case"raw":return!0;default:return!1}};function i(e){var t;switch(this.encoding=function(e){var t=function(e){if(!e)return"utf8";for(var t;;)switch(e){case"utf8":case"utf-8":return"utf8";case"ucs2":case"ucs-2":case"utf16le":case"utf-16le":return"utf16le";case"latin1":case"binary":return"latin1";case"base64":case"ascii":case"hex":return e;default:if(t)return;e=(""+e).toLowerCase(),t=!0}}(e);if("string"!=typeof t&&(r.isEncoding===o||!o(e)))throw new Error("Unknown encoding: "+e);return t||e}(e),this.encoding){case"utf16le":this.text=s,this.end=l,t=4;break;case"utf8":this.fillLast=u,t=4;break;case"base64":this.text=c,this.end=f,t=3;break;default:return this.write=p,void(this.end=d)}this.lastNeed=0,this.lastTotal=0,this.lastChar=r.allocUnsafe(t)}function a(e){return e<=127?0:e>>5==6?2:e>>4==14?3:e>>3==30?4:e>>6==2?-1:-2}function u(e){var t=this.lastTotal-this.lastNeed,n=function(e,t,n){if(128!=(192&t[0]))return e.lastNeed=0,"�";if(e.lastNeed>1&&t.length>1){if(128!=(192&t[1]))return e.lastNeed=1,"�";if(e.lastNeed>2&&t.length>2&&128!=(192&t[2]))return e.lastNeed=2,"�"}}(this,e);return void 0!==n?n:this.lastNeed<=e.length?(e.copy(this.lastChar,t,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal)):(e.copy(this.lastChar,t,0,e.length),void(this.lastNeed-=e.length))}function s(e,t){if((e.length-t)%2==0){var n=e.toString("utf16le",t);if(n){var r=n.charCodeAt(n.length-1);if(r>=55296&&r<=56319)return this.lastNeed=2,this.lastTotal=4,this.lastChar[0]=e[e.length-2],this.lastChar[1]=e[e.length-1],n.slice(0,-1)}return n}return this.lastNeed=1,this.lastTotal=2,this.lastChar[0]=e[e.length-1],e.toString("utf16le",t,e.length-1)}function l(e){var t=e&&e.length?this.write(e):"";if(this.lastNeed){var n=this.lastTotal-this.lastNeed;return t+this.lastChar.toString("utf16le",0,n)}return t}function c(e,t){var n=(e.length-t)%3;return 0===n?e.toString("base64",t):(this.lastNeed=3-n,this.lastTotal=3,1===n?this.lastChar[0]=e[e.length-1]:(this.lastChar[0]=e[e.length-2],this.lastChar[1]=e[e.length-1]),e.toString("base64",t,e.length-n))}function f(e){var t=e&&e.length?this.write(e):"";return this.lastNeed?t+this.lastChar.toString("base64",0,3-this.lastNeed):t}function p(e){return e.toString(this.encoding)}function d(e){return e&&e.length?this.write(e):""}t.StringDecoder=i,i.prototype.write=function(e){if(0===e.length)return"";var t,n;if(this.lastNeed){if(void 0===(t=this.fillLast(e)))return"";n=this.lastNeed,this.lastNeed=0}else n=0;return n=0)return o>0&&(e.lastNeed=o-1),o;if(--r=0)return o>0&&(e.lastNeed=o-2),o;if(--r=0)return o>0&&(2===o?o=0:e.lastNeed=o-3),o;return 0}(this,e,t);if(!this.lastNeed)return e.toString("utf8",t);this.lastTotal=n;var r=e.length-(n-this.lastNeed);return e.copy(this.lastChar,0,r),e.toString("utf8",t,r)},i.prototype.fillLast=function(e){if(this.lastNeed<=e.length)return e.copy(this.lastChar,this.lastTotal-this.lastNeed,0,this.lastNeed),this.lastChar.toString(this.encoding,0,this.lastTotal);e.copy(this.lastChar,this.lastTotal-this.lastNeed,0,e.length),this.lastNeed-=e.length}},function(e,t,n){"use strict";e.exports=i;var r=n(65),o=n(106);function i(e){if(!(this instanceof i))return new i(e);r.call(this,e),this._transformState={afterTransform:function(e,t){var n=this._transformState;n.transforming=!1;var r=n.writecb;if(!r)return this.emit("error",new Error("write callback called multiple times"));n.writechunk=null,n.writecb=null,null!=t&&this.push(t),r(e);var o=this._readableState;o.reading=!1,(o.needReadable||o.length=0?n&&o?o-1:o:1:!1!==e&&r(e)}},function(e,t,n){"use strict";e.exports=n(679)()?Object.assign:n(680)},function(e,t,n){"use strict";var r,o,i,a,u,s=n(67),l=function(e,t){return t};try{Object.defineProperty(l,"length",{configurable:!0,writable:!1,enumerable:!1,value:1})}catch(e){}1===l.length?(r={configurable:!0,writable:!1,enumerable:!1},o=Object.defineProperty,e.exports=function(e,t){return t=s(t),e.length===t?e:(r.value=t,o(e,"length",r))}):(a=n(317),u=[],i=function(e){var t,n=0;if(u[e])return u[e];for(t=[];e--;)t.push("a"+(++n).toString(36));return new Function("fn","return function ("+t.join(", ")+") { return fn.apply(this, arguments); };")},e.exports=function(e,t){var n;if(t=s(t),e.length===t)return e;n=i(t)(e);try{a(n,e)}catch(e){}return n})},function(e,t,n){"use strict";var r=n(82),o=Object.defineProperty,i=Object.getOwnPropertyDescriptor,a=Object.getOwnPropertyNames,u=Object.getOwnPropertySymbols;e.exports=function(e,t){var n,s=Object(r(t));if(e=Object(r(e)),a(s).forEach(function(r){try{o(e,r,i(t,r))}catch(e){n=e}}),"function"==typeof u&&u(s).forEach(function(r){try{o(e,r,i(t,r))}catch(e){n=e}}),void 0!==n)throw n;return e}},function(e,t,n){"use strict";var r=n(57),o=n(142),i=Function.prototype.call;e.exports=function(e,t){var n={},a=arguments[2];return r(t),o(e,function(e,r,o,u){n[r]=i.call(t,a,e,r,o,u)}),n}},function(e,t){e.exports=function(e){return!!e&&("object"==typeof e||"function"==typeof e)&&"function"==typeof e.then}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e){return{statePlugins:{err:{reducers:(0,i.default)(e),actions:a,selectors:u}}}};var r,o=n(321),i=(r=o)&&r.__esModule?r:{default:r},a=s(n(127)),u=s(n(325));function s(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=s(n(22)),o=s(n(23));t.default=function(e){var t;return t={},(0,r.default)(t,i.NEW_THROWN_ERR,function(t,n){var r=n.payload,i=(0,o.default)(l,r,{type:"thrown"});return t.update("errors",function(e){return(e||(0,a.List)()).push((0,a.fromJS)(i))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_THROWN_ERR_BATCH,function(t,n){var r=n.payload;return r=r.map(function(e){return(0,a.fromJS)((0,o.default)(l,e,{type:"thrown"}))}),t.update("errors",function(e){return(e||(0,a.List)()).concat((0,a.fromJS)(r))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_SPEC_ERR,function(t,n){var r=n.payload,o=(0,a.fromJS)(r);return o=o.set("type","spec"),t.update("errors",function(e){return(e||(0,a.List)()).push((0,a.fromJS)(o)).sortBy(function(e){return e.get("line")})}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_SPEC_ERR_BATCH,function(t,n){var r=n.payload;return r=r.map(function(e){return(0,a.fromJS)((0,o.default)(l,e,{type:"spec"}))}),t.update("errors",function(e){return(e||(0,a.List)()).concat((0,a.fromJS)(r))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.NEW_AUTH_ERR,function(t,n){var r=n.payload,i=(0,a.fromJS)((0,o.default)({},r));return i=i.set("type","auth"),t.update("errors",function(e){return(e||(0,a.List)()).push((0,a.fromJS)(i))}).update("errors",function(t){return(0,u.default)(t,e.getSystem())})}),(0,r.default)(t,i.CLEAR,function(e,t){var n=t.payload;if(!n||!e.get("errors"))return e;var r=e.get("errors").filter(function(e){return e.keySeq().every(function(t){var r=e.get(t),o=n[t];return!o||r!==o})});return e.merge({errors:r})}),(0,r.default)(t,i.CLEAR_BY,function(e,t){var n=t.payload;if(!n||"function"!=typeof n)return e;var r=e.get("errors").filter(function(e){return n(e)});return e.merge({errors:r})}),t};var i=n(127),a=n(7),u=s(n(322));function s(e){return e&&e.__esModule?e:{default:e}}var l={line:0,level:"error",message:"Unknown error"}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){var n={jsSpec:t.specSelectors.specJson().toJS()};return(0,i.default)(u,function(e,t){try{var r=t.transform(e,n);return r.filter(function(e){return!!e})}catch(t){return console.error("Transformer error:",t),e}},e).filter(function(e){return!!e}).map(function(e){return!e.get("line")&&e.get("path"),e})};var r,o=n(727),i=(r=o)&&r.__esModule?r:{default:r};function a(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}var u=[a(n(323)),a(n(324))]},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.transform=function(e){return e.map(function(e){var t=e.get("message").indexOf("is not of a type(s)");if(t>-1){var n=e.get("message").slice(t+"is not of a type(s)".length).split(",");return e.set("message",e.get("message").slice(0,t)+function(e){return e.reduce(function(e,t,n,r){return n===r.length-1&&r.length>1?e+"or "+t:r[n+1]&&r.length>2?e+t+", ":r[n+1]?e+t+" ":e+t},"should be a")}(n))}return e})}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.transform=function(e,t){t.jsSpec;return e};var r,o=n(138);(r=o)&&r.__esModule,n(7)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.lastError=t.allErrors=void 0;var r=n(7),o=n(58),i=t.allErrors=(0,o.createSelector)(function(e){return e},function(e){return e.get("errors",(0,r.List)())});t.lastError=(0,o.createSelector)(i,function(e){return e.last()})},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{statePlugins:{layout:{reducers:i.default,actions:a,selectors:u}}}};var r,o=n(327),i=(r=o)&&r.__esModule?r:{default:r},a=s(n(202)),u=s(n(328));function s(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r,o,i=n(22),a=(r=i)&&r.__esModule?r:{default:r},u=n(7),s=n(202);t.default=(o={},(0,a.default)(o,s.UPDATE_LAYOUT,function(e,t){return e.set("layout",t.payload)}),(0,a.default)(o,s.UPDATE_FILTER,function(e,t){return e.set("filter",t.payload)}),(0,a.default)(o,s.SHOW,function(e,t){var n=t.payload.shown,r=(0,u.fromJS)(t.payload.thing);return e.update("shown",(0,u.fromJS)({}),function(e){return e.set(r,n)})}),(0,a.default)(o,s.UPDATE_MODE,function(e,t){var n=t.payload.thing,r=t.payload.mode;return e.setIn(["modes"].concat(n),(r||"")+"")}),o)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.showSummary=t.whatMode=t.isShown=t.currentFilter=t.current=void 0;var r,o=n(83),i=(r=o)&&r.__esModule?r:{default:r},a=n(58),u=n(9),s=n(7);t.current=function(e){return e.get("layout")},t.currentFilter=function(e){return e.get("filter")};var l=t.isShown=function(e,t,n){return t=(0,u.normalizeArray)(t),e.get("shown",(0,s.fromJS)({})).get((0,s.fromJS)(t),n)};t.whatMode=function(e,t){var n=arguments.length>2&&void 0!==arguments[2]?arguments[2]:"";return t=(0,u.normalizeArray)(t),e.getIn(["modes"].concat((0,i.default)(t)),n)},t.showSummary=(0,a.createSelector)(function(e){return e},function(e){return!l(e,"editor")})},function(e,t,n){var r=n(36);e.exports=function(e,t,n,o){try{return o?t(r(n)[0],n[1]):t(n)}catch(t){var i=e.return;throw void 0!==i&&r(i.call(e)),t}}},function(e,t,n){var r=n(70),o=n(19)("iterator"),i=Array.prototype;e.exports=function(e){return void 0!==e&&(r.Array===e||i[o]===e)}},function(e,t,n){var r=n(19)("iterator"),o=!1;try{var i=[7][r]();i.return=function(){o=!0},Array.from(i,function(){throw 2})}catch(e){}e.exports=function(e,t){if(!t&&!o)return!1;var n=!1;try{var i=[7],a=i[r]();a.next=function(){return{done:n=!0}},i[r]=function(){return a},e(i)}catch(e){}return n}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{statePlugins:{spec:{wrapActions:s,reducers:i.default,actions:a,selectors:u}}}};var r,o=n(333),i=(r=o)&&r.__esModule?r:{default:r},a=l(n(203)),u=l(n(144)),s=l(n(346));function l(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r,o=p(n(22)),i=p(n(23)),a=p(n(83)),u=n(7),s=n(9),l=p(n(32)),c=n(144),f=n(203);function p(e){return e&&e.__esModule?e:{default:e}}t.default=(r={},(0,o.default)(r,f.UPDATE_SPEC,function(e,t){return"string"==typeof t.payload?e.set("spec",t.payload):e}),(0,o.default)(r,f.UPDATE_URL,function(e,t){return e.set("url",t.payload+"")}),(0,o.default)(r,f.UPDATE_JSON,function(e,t){return e.set("json",(0,s.fromJSOrdered)(t.payload))}),(0,o.default)(r,f.UPDATE_RESOLVED,function(e,t){return e.setIn(["resolved"],(0,s.fromJSOrdered)(t.payload))}),(0,o.default)(r,f.UPDATE_RESOLVED_SUBTREE,function(e,t){var n=t.payload,r=n.value,o=n.path;return e.setIn(["resolvedSubtrees"].concat((0,a.default)(o)),(0,s.fromJSOrdered)(r))}),(0,o.default)(r,f.UPDATE_PARAM,function(e,t){var n=t.payload,r=n.path,o=n.paramName,i=n.paramIn,u=n.param,l=n.value,c=n.isXml,f=u?(0,s.paramToIdentifier)(u):i+"."+o,p=c?"value_xml":"value";return e.setIn(["meta","paths"].concat((0,a.default)(r),["parameters",f,p]),l)}),(0,o.default)(r,f.UPDATE_EMPTY_PARAM_INCLUSION,function(e,t){var n=t.payload,r=n.pathMethod,o=n.paramName,i=n.paramIn,u=n.includeEmptyValue;if(!o||!i)return console.warn("Warning: UPDATE_EMPTY_PARAM_INCLUSION could not generate a paramKey."),e;var s=i+"."+o;return e.setIn(["meta","paths"].concat((0,a.default)(r),["parameter_inclusions",s]),u)}),(0,o.default)(r,f.VALIDATE_PARAMS,function(e,t){var n=t.payload,r=n.pathMethod,o=n.isOAS3,i=(0,c.specJsonWithResolvedSubtrees)(e).getIn(["paths"].concat((0,a.default)(r))),l=(0,c.parameterValues)(e,r).toJS();return e.updateIn(["meta","paths"].concat((0,a.default)(r),["parameters"]),(0,u.fromJS)({}),function(t){return i.get("parameters",(0,u.List)()).reduce(function(t,n){var i=(0,s.paramToValue)(n,l),a=(0,c.parameterInclusionSettingFor)(e,r,n.get("name"),n.get("in")),f=(0,s.validateParam)(n,i,{bypassRequiredCheck:a,isOAS3:o});return t.setIn([(0,s.paramToIdentifier)(n),"errors"],(0,u.fromJS)(f))},t)})}),(0,o.default)(r,f.CLEAR_VALIDATE_PARAMS,function(e,t){var n=t.payload.pathMethod;return e.updateIn(["meta","paths"].concat((0,a.default)(n),["parameters"]),(0,u.fromJS)([]),function(e){return e.map(function(e){return e.set("errors",(0,u.fromJS)([]))})})}),(0,o.default)(r,f.SET_RESPONSE,function(e,t){var n=t.payload,r=n.res,o=n.path,a=n.method,u=void 0;(u=r.error?(0,i.default)({error:!0,name:r.err.name,message:r.err.message,statusCode:r.err.statusCode},r.err.response):r).headers=u.headers||{};var c=e.setIn(["responses",o,a],(0,s.fromJSOrdered)(u));return l.default.Blob&&r.data instanceof l.default.Blob&&(c=c.setIn(["responses",o,a,"text"],r.data)),c}),(0,o.default)(r,f.SET_REQUEST,function(e,t){var n=t.payload,r=n.req,o=n.path,i=n.method;return e.setIn(["requests",o,i],(0,s.fromJSOrdered)(r))}),(0,o.default)(r,f.SET_MUTATED_REQUEST,function(e,t){var n=t.payload,r=n.req,o=n.path,i=n.method;return e.setIn(["mutatedRequests",o,i],(0,s.fromJSOrdered)(r))}),(0,o.default)(r,f.UPDATE_OPERATION_META_VALUE,function(e,t){var n=t.payload,r=n.path,o=n.value,i=n.key,s=["paths"].concat((0,a.default)(r)),l=["meta","paths"].concat((0,a.default)(r));return e.getIn(["json"].concat((0,a.default)(s)))||e.getIn(["resolved"].concat((0,a.default)(s)))||e.getIn(["resolvedSubtrees"].concat((0,a.default)(s)))?e.setIn([].concat((0,a.default)(l),[i]),(0,u.fromJS)(o)):e}),(0,o.default)(r,f.CLEAR_RESPONSE,function(e,t){var n=t.payload,r=n.path,o=n.method;return e.deleteIn(["responses",r,o])}),(0,o.default)(r,f.CLEAR_REQUEST,function(e,t){var n=t.payload,r=n.path,o=n.method;return e.deleteIn(["requests",r,o])}),(0,o.default)(r,f.SET_SCHEME,function(e,t){var n=t.payload,r=n.scheme,o=n.path,i=n.method;return o&&i?e.setIn(["scheme",o,i],r):o||i?void 0:e.setIn(["scheme","_defaultScheme"],r)}),r)},function(e,t,n){var r=n(36),o=n(94),i=n(19)("species");e.exports=function(e,t){var n,a=r(e).constructor;return void 0===a||void 0==(n=r(a)[i])?t:o(n)}},function(e,t,n){var r,o,i,a=n(49),u=n(735),s=n(241),l=n(157),c=n(21),f=c.process,p=c.setImmediate,d=c.clearImmediate,h=c.MessageChannel,v=c.Dispatch,m=0,g={},y=function(){var e=+this;if(g.hasOwnProperty(e)){var t=g[e];delete g[e],t()}},b=function(e){y.call(e.data)};p&&d||(p=function(e){for(var t=[],n=1;arguments.length>n;)t.push(arguments[n++]);return g[++m]=function(){u("function"==typeof e?e:Function(e),t)},r(m),m},d=function(e){delete g[e]},"process"==n(93)(f)?r=function(e){f.nextTick(a(y,e,1))}:v&&v.now?r=function(e){v.now(a(y,e,1))}:h?(i=(o=new h).port2,o.port1.onmessage=b,r=a(i.postMessage,i,1)):c.addEventListener&&"function"==typeof postMessage&&!c.importScripts?(r=function(e){c.postMessage(e+"","*")},c.addEventListener("message",b,!1)):r="onreadystatechange"in l("script")?function(e){s.appendChild(l("script")).onreadystatechange=function(){s.removeChild(this),y.call(e)}}:function(e){setTimeout(a(y,e,1),0)}),e.exports={set:p,clear:d}},function(e,t){e.exports=function(e){try{return{e:!1,v:e()}}catch(e){return{e:!0,v:e}}}},function(e,t,n){var r=n(36),o=n(28),i=n(206);e.exports=function(e,t){if(r(e),o(t)&&t.constructor===e)return t;var n=i.f(e);return(0,n.resolve)(t),n.promise}},function(e,t,n){e.exports=n(740)},function(e,t,n){"use strict";t.__esModule=!0;var r,o=n(204),i=(r=o)&&r.__esModule?r:{default:r};t.default=function(e){return function(){var t=e.apply(this,arguments);return new i.default(function(e,n){return function r(o,a){try{var u=t[o](a),s=u.value}catch(e){return void n(e)}if(!u.done)return i.default.resolve(s).then(function(e){r("next",e)},function(e){r("throw",e)});e(s)}("next")})}}},function(e,t,n){"use strict";var r=n(86);e.exports=new r({include:[n(341)]})},function(e,t,n){"use strict";var r=n(86);e.exports=new r({include:[n(209)],implicit:[n(748),n(749),n(750),n(751)]})},function(e,t,n){var r=n(62),o=n(24),i=n(47),a="[object String]";e.exports=function(e){return"string"==typeof e||!o(e)&&i(e)&&r(e)==a}},function(e,t,n){var r=n(147),o=n(79),i=n(135),a=n(38),u=n(80);e.exports=function(e,t,n,s){if(!a(e))return e;for(var l=-1,c=(t=o(t,e)).length,f=c-1,p=e;null!=p&&++l.":"function"==typeof t?" Instead of passing a class like Foo, pass React.createElement(Foo) or .":null!=t&&void 0!==t.props?" This may be caused by unintentionally loading two independent copies of React.":"");var i,u=a.createElement(D,{child:t});if(e){var s=p.get(e);i=s._processChildContext(s._context)}else i=g;var l=N(n);if(l){var c=l._currentElement.props.child;if(_(c,t)){var f=l._renderedComponent.getPublicInstance(),d=o&&function(){o.call(f)};return L._updateRootComponent(l,u,i,n,d),f}L.unmountComponentAtNode(n)}var h=A(n),m=h&&!!O(h),y=I(n),b=m&&!l&&!y,w=L._renderNewRootComponent(u,n,b,i)._renderedComponent.getPublicInstance();return o&&o.call(w),w},render:function(e,t,n){return L._renderSubtreeIntoContainer(null,e,t,n)},unmountComponentAtNode:function(e){j(e)||r("40");var t=N(e);if(!t){I(e),1===e.nodeType&&e.hasAttribute(E);return!1}return delete k[t._instance.rootID],m.batchedUpdates(M,t,e,!1),!0},_mountImageIntoNode:function(e,t,n,i,a){if(j(t)||r("41"),i){var u=A(t);if(d.canReuseMarkup(e,u))return void s.precacheNode(n,u);var l=u.getAttribute(d.CHECKSUM_ATTR_NAME);u.removeAttribute(d.CHECKSUM_ATTR_NAME);var c=u.outerHTML;u.setAttribute(d.CHECKSUM_ATTR_NAME,l);var f=e,p=function(e,t){for(var n=Math.min(e.length,t.length),r=0;r1?r-1:0),a=1;a=o&&(t=console)[e].apply(t,i)}return i.warn=i.bind(null,"warn"),i.error=i.bind(null,"error"),i.info=i.bind(null,"info"),i.debug=i.bind(null,"debug"),{rootInjects:{log:i}}}},function(e,t,n){"use strict";var r,o=n(387),i=(r=o)&&r.__esModule?r:{default:r},a=function(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}(n(393));e.exports=function(e){var t=e.configs,n=e.getConfigs;return{fn:{fetch:i.default.makeHttp(t.preFetch,t.postFetch),buildRequest:i.default.buildRequest,execute:i.default.execute,resolve:i.default.resolve,resolveSubtree:function(e,t,r){for(var o=arguments.length,a=Array(o>3?o-3:0),u=3;u2&&void 0!==arguments[2]?arguments[2]:"",r=(arguments.length>3&&void 0!==arguments[3]?arguments[3]:{}).v2OperationIdCompatibilityMode;return e&&"object"===(void 0===e?"undefined":(0,c.default)(e))?(e.operationId||"").replace(/\s/g,"").length?h(e.operationId):i(t,n,{v2OperationIdCompatibilityMode:r}):null}function i(e,t){if((arguments.length>2&&void 0!==arguments[2]?arguments[2]:{}).v2OperationIdCompatibilityMode){var n=(t.toLowerCase()+"_"+e).replace(/[\s!@#$%^&*()_+=[{\]};:<>|.\/?,\\'""-]/g,"_");return(n=n||e.substring(1)+"_"+t).replace(/((_){2,})/g,"_").replace(/^(_)*/g,"").replace(/([_])*$/g,"")}return""+d(t)+h(e)}function a(e,t){return d(t)+"-"+e}function u(e,t){return s(e,t,!0)||null}function s(e,t,n){if(!e||"object"!==(void 0===e?"undefined":(0,c.default)(e))||!e.paths||"object"!==(0,c.default)(e.paths))return null;var r=e.paths;for(var o in r)for(var i in r[o])if("PARAMETERS"!==i.toUpperCase()){var a=r[o][i];if(a&&"object"===(void 0===a?"undefined":(0,c.default)(a))){var u={spec:e,pathName:o,method:i.toUpperCase(),operation:a},s=t(u);if(n&&s)return u}}}Object.defineProperty(t,"__esModule",{value:!0});var l=r(n(18)),c=r(n(1));t.isOAS3=function(e){var t=e.openapi;return!!t&&(0,p.default)(t,"3")},t.isSwagger2=function(e){var t=e.swagger;return!!t&&(0,p.default)(t,"2")},t.opId=o,t.idFromPathMethod=i,t.legacyIdFromPathMethod=a,t.getOperationRaw=function(e,t){return e&&e.paths?u(e,function(e){var n=e.pathName,r=e.method,i=e.operation;if(!i||"object"!==(void 0===i?"undefined":(0,c.default)(i)))return!1;var u=i.operationId;return[o(i,n,r),a(n,r),u].some(function(e){return e&&e===t})}):null},t.findOperation=u,t.eachOperation=s,t.normalizeSwagger=function(e){var t=e.spec,n=t.paths,r={};if(!n||t.$$normalized)return e;for(var i in n){var a=n[i];if((0,f.default)(a)){var u=a.parameters;for(var s in a)!function(e){var n=a[e];if(!(0,f.default)(n))return"continue";var s=o(n,i,e);if(s){r[s]?r[s].push(n):r[s]=[n];var c=r[s];if(c.length>1)c.forEach(function(e,t){e.__originalOperationId=e.__originalOperationId||e.operationId,e.operationId=""+s+(t+1)});else if(void 0!==n.operationId){var p=c[0];p.__originalOperationId=p.__originalOperationId||n.operationId,p.operationId=s}}if("parameters"!==e){var d=[],h={};for(var v in t)"produces"!==v&&"consumes"!==v&&"security"!==v||(h[v]=t[v],d.push(h));if(u&&(h.parameters=u,d.push(h)),d.length){var m=!0,g=!1,y=void 0;try{for(var b,_=(0,l.default)(d);!(m=(b=_.next()).done);m=!0){var w=b.value;for(var E in w)if(n[E]){if("parameters"===E){var x=!0,S=!1,C=void 0;try{for(var k,A=(0,l.default)(w[E]);!(x=(k=A.next()).done);x=!0)!function(){var e=k.value;n[E].some(function(t){return t.name&&t.name===e.name||t.$ref&&t.$ref===e.$ref||t.$$ref&&t.$$ref===e.$$ref||t===e})||n[E].push(e)}()}catch(e){S=!0,C=e}finally{try{!x&&A.return&&A.return()}finally{if(S)throw C}}}}else n[E]=w[E]}}catch(e){g=!0,y=e}finally{try{!m&&_.return&&_.return()}finally{if(g)throw y}}}}}(s)}}return t.$$normalized=!0,e};var f=r(n(47)),p=r(n(14)),d=function(e){return String.prototype.toLowerCase.call(e)},h=function(e){return e.replace(/[^\w]/gi,"_")}},function(e,t){e.exports=n(893)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e,t){var n=(arguments.length>2&&void 0!==arguments[2]?arguments[2]:{}).loadSpec,r=void 0!==n&&n,o={ok:e.ok,url:e.url||t,status:e.status,statusText:e.statusText,headers:i(e.headers)},a=o.headers["content-type"],u=r||_(a);return(u?e.text:e.blob||e.buffer).call(e).then(function(e){if(o.text=e,o.data=e,u)try{var t=function(e,t){return t&&(0===t.indexOf("application/json")||t.indexOf("+json")>0)?JSON.parse(e):g.default.safeLoad(e)}(e,a);o.body=t,o.obj=t}catch(e){o.parseError=e}return o})}function i(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t={};return"function"==typeof e.forEach?(e.forEach(function(e,n){void 0!==t[n]?(t[n]=Array.isArray(t[n])?t[n]:[t[n]],t[n].push(e)):t[n]=e}),t):t}function a(e,t){return t||"undefined"==typeof navigator||(t=navigator),t&&"ReactNative"===t.product?!(!e||"object"!==(void 0===e?"undefined":(0,h.default)(e))||"string"!=typeof e.uri):"undefined"!=typeof File?e instanceof File:null!==e&&"object"===(void 0===e?"undefined":(0,h.default)(e))&&"function"==typeof e.pipe}function u(e,t){var n=e.collectionFormat,r=e.allowEmptyValue,o="object"===(void 0===e?"undefined":(0,h.default)(e))?e.value:e;if(void 0===o&&r)return"";if(a(o)||"boolean"==typeof o)return o;var i=encodeURIComponent;return t&&(i=(0,y.default)(o)?function(e){return e}:function(e){return(0,p.default)(e)}),"object"!==(void 0===o?"undefined":(0,h.default)(o))||Array.isArray(o)?Array.isArray(o)?Array.isArray(o)&&!n?o.map(i).join(","):"multi"===n?o.map(i):o.map(i).join({csv:",",ssv:"%20",tsv:"%09",pipes:"|"}[n]):i(o):""}function s(e){var t=(0,f.default)(e).reduce(function(t,n){var r=e[n],o=!!r.skipEncoding,i=o?n:encodeURIComponent(n),a=function(e){return e&&"object"===(void 0===e?"undefined":(0,h.default)(e))}(r)&&!Array.isArray(r);return t[i]=u(a?r:{value:r},o),t},{});return m.default.stringify(t,{encode:!1,indices:!1})||""}function l(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=e.url,r=void 0===t?"":t,o=e.query,i=e.form;if(i){var l=(0,f.default)(i).some(function(e){return a(i[e].value)}),p=e.headers["content-type"]||e.headers["Content-Type"];if(l||/multipart\/form-data/i.test(p)){var d=n(30);e.body=new d,(0,f.default)(i).forEach(function(t){e.body.append(t,u(i[t],!0))})}else e.body=s(i);delete e.form}if(o){var h=r.split("?"),v=(0,c.default)(h,2),g=v[0],y=v[1],b="";if(y){var _=m.default.parse(y);(0,f.default)(o).forEach(function(e){return delete _[e]}),b=m.default.stringify(_,{encode:!0})}var w=function(){for(var e=arguments.length,t=Array(e),n=0;n1&&void 0!==arguments[1]?arguments[1]:{};return d.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:if("object"===(void 0===t?"undefined":(0,h.default)(t))&&(t=(a=t).url),a.headers=a.headers||{},b.mergeInQueryOrForm(a),!a.requestInterceptor){e.next=10;break}return e.next=6,a.requestInterceptor(a);case 6:if(e.t0=e.sent,e.t0){e.next=9;break}e.t0=a;case 9:a=e.t0;case 10:return n=a.headers["content-type"]||a.headers["Content-Type"],/multipart\/form-data/i.test(n)&&(delete a.headers["content-type"],delete a.headers["Content-Type"]),r=void 0,e.prev=13,e.next=16,(a.userFetch||fetch)(a.url,a);case 16:return r=e.sent,e.next=19,b.serializeRes(r,t,a);case 19:if(r=e.sent,!a.responseInterceptor){e.next=27;break}return e.next=23,a.responseInterceptor(r);case 23:if(e.t1=e.sent,e.t1){e.next=26;break}e.t1=r;case 26:r=e.t1;case 27:e.next=37;break;case 29:if(e.prev=29,e.t2=e.catch(13),r){e.next=33;break}throw e.t2;case 33:throw(o=new Error(r.statusText)).statusCode=o.status=r.status,o.responseError=e.t2,o;case 37:if(r.ok){e.next=42;break}throw(i=new Error(r.statusText)).statusCode=i.status=r.status,i.response=r,i;case 42:return e.abrupt("return",r);case 43:case"end":return e.stop()}},e,this,[[13,29]])}));return function(t){return e.apply(this,arguments)}}();var _=t.shouldDownloadAsText=function(){return/(json|xml|yaml|text)\b/.test(arguments.length>0&&void 0!==arguments[0]?arguments[0]:"")}},function(e,t){e.exports=n(41)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){return Array.isArray(e)?e.length<1?"":"/"+e.map(function(e){return(e+"").replace(/~/g,"~0").replace(/\//g,"~1")}).join("/"):e}function i(e,t,n){return{op:"replace",path:e,value:t,meta:n}}function a(e,t,n){return f(c(e.filter(m).map(function(e){return t(e.value,n,e.path)})||[]))}function u(e,t,n){return n=n||[],Array.isArray(e)?e.map(function(e,r){return u(e,t,n.concat(r))}):p(e)?(0,w.default)(e).map(function(r){return u(e[r],t,n.concat(r))}):t(e,n[n.length-1],n)}function s(e,t,n){var r=[];if((n=n||[]).length>0){var o=t(e,n[n.length-1],n);o&&(r=r.concat(o))}if(Array.isArray(e)){var i=e.map(function(e,r){return s(e,t,n.concat(r))});i&&(r=r.concat(i))}else if(p(e)){var a=(0,w.default)(e).map(function(r){return s(e[r],t,n.concat(r))});a&&(r=r.concat(a))}return c(r)}function l(e){return Array.isArray(e)?e:[e]}function c(e){var t;return(t=[]).concat.apply(t,(0,_.default)(e.map(function(e){return Array.isArray(e)?c(e):e})))}function f(e){return e.filter(function(e){return void 0!==e})}function p(e){return e&&"object"===(void 0===e?"undefined":(0,b.default)(e))}function d(e){return e&&"function"==typeof e}function h(e){if(g(e)){var t=e.op;return"add"===t||"remove"===t||"replace"===t}return!1}function v(e){return h(e)||g(e)&&"mutation"===e.type}function m(e){return v(e)&&("add"===e.op||"replace"===e.op||"merge"===e.op||"mergeDeep"===e.op)}function g(e){return e&&"object"===(void 0===e?"undefined":(0,b.default)(e))}function y(e,t){try{return S.default.getValueByPointer(e,t)}catch(e){return console.error(e),{}}}Object.defineProperty(t,"__esModule",{value:!0});var b=r(n(1)),_=r(n(34)),w=r(n(0)),E=r(n(35)),x=r(n(2)),S=r(n(36)),C=r(n(4)),k=r(n(37)),A=r(n(38));t.default={add:function(e,t){return{op:"add",path:e,value:t}},replace:i,remove:function(e,t){return{op:"remove",path:e}},merge:function(e,t){return{type:"mutation",op:"merge",path:e,value:t}},mergeDeep:function(e,t){return{type:"mutation",op:"mergeDeep",path:e,value:t}},context:function(e,t){return{type:"context",path:e,value:t}},getIn:function(e,t){return t.reduce(function(e,t){return void 0!==t&&e?e[t]:e},e)},applyPatch:function(e,t,n){if(n=n||{},"merge"===(t=(0,x.default)({},t,{path:t.path&&o(t.path)})).op){var r=y(e,t.path);(0,x.default)(r,t.value),S.default.applyPatch(e,[i(t.path,r)])}else if("mergeDeep"===t.op){var a=y(e,t.path);for(var u in t.value){var s=t.value[u],l=Array.isArray(s);if(l){var c=a[u]||[];a[u]=c.concat(s)}else if(p(s)&&!l){var f=(0,x.default)({},a[u]);for(var d in s){if(Object.prototype.hasOwnProperty.call(f,d)){f=(0,k.default)((0,A.default)({},f),s);break}(0,x.default)(f,(0,E.default)({},d,s[d]))}a[u]=f}else a[u]=s}}else if("add"===t.op&&""===t.path&&p(t.value)){var h=(0,w.default)(t.value).reduce(function(e,n){return e.push({op:"add",path:"/"+o(n),value:t.value[n]}),e},[]);S.default.applyPatch(e,h)}else if("replace"===t.op&&""===t.path){var v=t.value;n.allowMetaPatches&&t.meta&&m(t)&&(Array.isArray(t.value)||p(t.value))&&(v=(0,x.default)({},v,t.meta)),e=v}else if(S.default.applyPatch(e,[t]),n.allowMetaPatches&&t.meta&&m(t)&&(Array.isArray(t.value)||p(t.value))){var g=y(e,t.path),b=(0,x.default)({},g,t.meta);S.default.applyPatch(e,[i(t.path,b)])}return e},parentPathMatch:function(e,t){if(!Array.isArray(t))return!1;for(var n=0,r=t.length;n1&&void 0!==arguments[1]?arguments[1]:{},n=t.requestInterceptor,r=t.responseInterceptor,o=e.withCredentials?"include":"same-origin";return function(t){return e({url:t,loadSpec:!0,requestInterceptor:n,responseInterceptor:r,headers:{Accept:"application/json"},credentials:o}).then(function(e){return e.body})}}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(4)),a=r(n(11));t.makeFetchJSON=o,t.clearCache=function(){s.plugins.refs.clearCache()},t.default=function(e){function t(e){var t=this;E&&(s.plugins.refs.docCache[E]=e),s.plugins.refs.fetchJSON=o(w,{requestInterceptor:y,responseInterceptor:b});var n=[s.plugins.refs];return"function"==typeof g&&n.push(s.plugins.parameters),"function"==typeof m&&n.push(s.plugins.properties),"strict"!==p&&n.push(s.plugins.allOf),(0,l.default)({spec:e,context:{baseDoc:E},plugins:n,allowMetaPatches:h,pathDiscriminator:v,parameterMacro:g,modelPropertyMacro:m}).then(_?function(){var e=(0,a.default)(i.default.mark(function e(n){return i.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:return e.abrupt("return",n);case 1:case"end":return e.stop()}},e,t)}));return function(t){return e.apply(this,arguments)}}():c.normalizeSwagger)}var n=e.fetch,r=e.spec,f=e.url,p=e.mode,d=e.allowMetaPatches,h=void 0===d||d,v=e.pathDiscriminator,m=e.modelPropertyMacro,g=e.parameterMacro,y=e.requestInterceptor,b=e.responseInterceptor,_=e.skipNormalization,w=e.http,E=e.baseDoc;return E=E||f,w=n||w||u.default,r?t(r):o(w,{requestInterceptor:y,responseInterceptor:b})(E).then(t)};var u=r(n(7)),s=n(31),l=r(s),c=n(5)},function(e,t){e.exports=n(204)},function(e,t){e.exports=n(91)},function(e,t){e.exports=n(2)},function(e,t){e.exports=n(3)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){function n(){Error.captureStackTrace?Error.captureStackTrace(this,this.constructor):this.stack=(new Error).stack;for(var e=arguments.length,n=Array(e),r=0;r-1&&-1===o.indexOf(n)||i.indexOf(u)>-1||a.some(function(e){return u.indexOf(e)>-1})};var r=["properties"],o=["properties"],i=["definitions","parameters","responses","securityDefinitions","components/schemas","components/responses","components/parameters","components/securitySchemes"],a=["schema/example","items/example"]},function(e,t,n){e.exports=n(24)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=this,n=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};if("string"==typeof e?n.url=e:n=e,!(this instanceof o))return new o(n);(0,a.default)(this,n);var r=this.resolve().then(function(){return t.disableInterfaces||(0,a.default)(t,o.makeApisTagOperation(t)),t});return r.client=this,r}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(3)),a=r((r(n(25)),n(6))),u=r(n(14)),s=r(n(10)),l=n(7),c=r(l),f=n(16),p=r(f),d=r(n(48)),h=n(49),v=n(51),m=n(5);o.http=c.default,o.makeHttp=l.makeHttp.bind(null,o.http),o.resolve=p.default,o.resolveSubtree=d.default,o.execute=v.execute,o.serializeRes=l.serializeRes,o.serializeHeaders=l.serializeHeaders,o.clearCache=f.clearCache,o.parameterBuilders=v.PARAMETER_BUILDERS,o.makeApisTagOperation=h.makeApisTagOperation,o.buildRequest=v.buildRequest,o.helpers={opId:m.opId},o.prototype={http:c.default,execute:function(e){return this.applyDefaults(),o.execute((0,i.default)({spec:this.spec,http:this.http,securities:{authorized:this.authorizations},contextUrl:"string"==typeof this.url?this.url:void 0},e))},resolve:function(){var e=this;return o.resolve({spec:this.spec,url:this.url,allowMetaPatches:this.allowMetaPatches,requestInterceptor:this.requestInterceptor||null,responseInterceptor:this.responseInterceptor||null}).then(function(t){return e.originalSpec=e.spec,e.spec=t.spec,e.errors=t.errors,e})}},o.prototype.applyDefaults=function(){var e=this.spec,t=this.url;if(t&&(0,u.default)(t,"http")){var n=s.default.parse(t);e.host||(e.host=n.host),e.schemes||(e.schemes=[n.protocol.replace(":","")]),e.basePath||(e.basePath="/")}},t.default=o,e.exports=t.default},function(e,t){e.exports=n(905)},function(e,t){e.exports=n(17)},function(e,t){e.exports=n(906)},function(e,t){e.exports=n(907)},function(e,t){e.exports=n(342)},function(e,t){e.exports=n(910)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0}),t.plugins=t.SpecMap=void 0;var o=r(n(8)),i=r(n(1)),a=r(n(17)),u=r(n(4)),s=r(n(0)),l=r(n(18)),c=r(n(32)),f=r(n(2)),p=r(n(19)),d=r(n(20));t.default=function(e){return new w(e).dispatch()};var h=r(n(33)),v=r(n(9)),m=r(n(39)),g=r(n(43)),y=r(n(44)),b=r(n(45)),_=r(n(46)),w=function(){function e(t){(0,p.default)(this,e),(0,f.default)(this,{spec:"",debugLevel:"info",plugins:[],pluginHistory:{},errors:[],mutations:[],promisedPatches:[],state:{},patches:[],context:{},contextTree:new _.default,showDebug:!1,allPatches:[],pluginProp:"specMap",libMethods:(0,f.default)((0,c.default)(this),v.default),allowMetaPatches:!1},t),this.get=this._get.bind(this),this.getContext=this._getContext.bind(this),this.hasRun=this._hasRun.bind(this),this.wrappedPlugins=this.plugins.map(this.wrapPlugin.bind(this)).filter(v.default.isFunction),this.patches.push(v.default.add([],this.spec)),this.patches.push(v.default.context([],this.context)),this.updatePatches(this.patches)}return(0,d.default)(e,[{key:"debug",value:function(e){if(this.debugLevel===e){for(var t,n=arguments.length,r=Array(n>1?n-1:0),o=1;o1?n-1:0),o=1;o0})}},{key:"nextPromisedPatch",value:function(){if(this.promisedPatches.length>0)return a.default.race(this.promisedPatches.map(function(e){return e.value}))}},{key:"getPluginHistory",value:function(e){var t=this.getPluginName(e);return this.pluginHistory[t]||[]}},{key:"getPluginRunCount",value:function(e){return this.getPluginHistory(e).length}},{key:"getPluginHistoryTip",value:function(e){var t=this.getPluginHistory(e);return t&&t[t.length-1]||{}}},{key:"getPluginMutationIndex",value:function(e){var t=this.getPluginHistoryTip(e).mutationIndex;return"number"!=typeof t?-1:t}},{key:"getPluginName",value:function(e){return e.pluginName}},{key:"updatePluginHistory",value:function(e,t){var n=this.getPluginName(e);(this.pluginHistory[n]=this.pluginHistory[n]||[]).push(t)}},{key:"updatePatches",value:function(e,t){var n=this;v.default.normalizeArray(e).forEach(function(e){if(e instanceof Error)n.errors.push(e);else try{if(!v.default.isObject(e))return void n.debug("updatePatches","Got a non-object patch",e);if(n.showDebug&&n.allPatches.push(e),v.default.isPromise(e.value))return n.promisedPatches.push(e),void n.promisedPatchThen(e);if(v.default.isContextPatch(e))return void n.setContext(e.path,e.value);if(v.default.isMutation(e))return void n.updateMutations(e)}catch(e){console.error(e),n.errors.push(e)}})}},{key:"updateMutations",value:function(e){"object"===(0,i.default)(e.value)&&!Array.isArray(e.value)&&this.allowMetaPatches&&(e.value=(0,f.default)({},e.value));var t=v.default.applyPatch(this.state,e,{allowMetaPatches:this.allowMetaPatches});t&&(this.mutations.push(e),this.state=t)}},{key:"removePromisedPatch",value:function(e){var t=this.promisedPatches.indexOf(e);t<0?this.debug("Tried to remove a promisedPatch that isn't there!"):this.promisedPatches.splice(t,1)}},{key:"promisedPatchThen",value:function(e){var t=this;return e.value=e.value.then(function(n){var r=(0,f.default)({},e,{value:n});t.removePromisedPatch(e),t.updatePatches(r)}).catch(function(n){t.removePromisedPatch(e),t.updatePatches(n)})}},{key:"getMutations",value:function(e,t){return e=e||0,"number"!=typeof t&&(t=this.mutations.length),this.mutations.slice(e,t)}},{key:"getCurrentMutations",value:function(){return this.getMutationsForPlugin(this.getCurrentPlugin())}},{key:"getMutationsForPlugin",value:function(e){var t=this.getPluginMutationIndex(e);return this.getMutations(t+1)}},{key:"getCurrentPlugin",value:function(){return this.currentPlugin}},{key:"getPatchesOfType",value:function(e,t){return e.filter(t)}},{key:"getLib",value:function(){return this.libMethods}},{key:"_get",value:function(e){return v.default.getIn(this.state,e)}},{key:"_getContext",value:function(e){return this.contextTree.get(e)}},{key:"setContext",value:function(e,t){return this.contextTree.set(e,t)}},{key:"_hasRun",value:function(e){return this.getPluginRunCount(this.getCurrentPlugin())>(e||0)}},{key:"_clone",value:function(e){return JSON.parse((0,o.default)(e))}},{key:"dispatch",value:function(){function e(e){e&&(e=v.default.fullyNormalizeArray(e),n.updatePatches(e,r))}var t=this,n=this,r=this.nextPlugin();if(!r){var o=this.nextPromisedPatch();if(o)return o.then(function(){return t.dispatch()}).catch(function(){return t.dispatch()});var i={spec:this.state,errors:this.errors};return this.showDebug&&(i.patches=this.allPatches),a.default.resolve(i)}if(n.pluginCount=n.pluginCount||{},n.pluginCount[r]=(n.pluginCount[r]||0)+1,n.pluginCount[r]>100)return a.default.resolve({spec:n.state,errors:n.errors.concat(new Error("We've reached a hard limit of 100 plugin runs"))});if(r!==this.currentPlugin&&this.promisedPatches.length){var u=this.promisedPatches.map(function(e){return e.value});return a.default.all(u.map(function(e){return e.then(Function,Function)})).then(function(){return t.dispatch()})}return function(){n.currentPlugin=r;var t=n.getCurrentMutations(),o=n.mutations.length-1;try{if(r.isGenerator){var i=!0,a=!1,u=void 0;try{for(var s,p=(0,l.default)(r(t,n.getLib()));!(i=(s=p.next()).done);i=!0)e(s.value)}catch(e){a=!0,u=e}finally{try{!i&&p.return&&p.return()}finally{if(a)throw u}}}else e(r(t,n.getLib()))}catch(t){console.error(t),e([(0,f.default)((0,c.default)(t),{plugin:r})])}finally{n.updatePluginHistory(r,{mutationIndex:o})}return n.dispatch()}()}}]),e}(),E={refs:m.default,allOf:g.default,parameters:y.default,properties:b.default};t.SpecMap=w,t.plugins=E},function(e,t){e.exports=n(349)},function(e,t){e.exports=n(288)},function(e,t){e.exports=n(83)},function(e,t){e.exports=n(22)},function(e,t){e.exports=n(911)},function(e,t){e.exports=n(179)},function(e,t){e.exports=n(181)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e,t){if(!A.test(e)){if(!t)throw new O("Tried to resolve a relative URL, without having a basePath. path: '"+e+"' basePath: '"+t+"'");return x.default.resolve(t,e)}return e}function i(e,t){var n;return n=e&&e.response&&e.response.body?e.response.body.code+" "+e.response.body.message:e.message,new O("Could not resolve reference: "+n,t,e)}function a(e){return(e+"").split("#")}function u(e,t){var n=P[e];if(n&&!S.default.isPromise(n))try{var r=l(t,n);return(0,b.default)(g.default.resolve(r),{__value:r})}catch(e){return g.default.reject(e)}return s(e).then(function(e){return l(t,e)})}function s(e){var t=P[e];return t?S.default.isPromise(t)?t:g.default.resolve(t):(P[e]=I.fetchJSON(e).then(function(t){return P[e]=t,t}),P[e])}function l(e,t){var n=c(e);if(n.length<1)return t;var r=S.default.getIn(t,n);if(void 0===r)throw new O("Could not resolve pointer: "+e+" does not exist in document",{pointer:e});return r}function c(e){if("string"!=typeof e)throw new TypeError("Expected a string, got a "+(void 0===e?"undefined":(0,v.default)(e)));return"/"===e[0]&&(e=e.substr(1)),""===e?[]:e.split("/").map(f)}function f(e){return"string"!=typeof e?e:E.default.unescape(e.replace(/~1/g,"/").replace(/~0/g,"~"))}function p(e){return E.default.escape(e.replace(/~/g,"~0").replace(/\//g,"~1"))}function d(e,t){if(j(t))return!0;var n=e.charAt(t.length),r=t.slice(-1);return 0===e.indexOf(t)&&(!n||"/"===n||"#"===n)&&"#"!==r}function h(e,t,n,r){var o=T.get(r);o||(o={},T.set(r,o));var i=function(e){return 0===e.length?"":"/"+e.map(p).join("/")}(n),a=(t||"")+"#"+e;if(t==r.contextTree.get([]).baseDoc&&d(i,e))return!0;var u="";if(n.some(function(e){return u=u+"/"+p(e),o[u]&&o[u].some(function(e){return d(e,a)||d(a,e)})}))return!0;o[i]=(o[i]||[]).concat(a)}Object.defineProperty(t,"__esModule",{value:!0});var v=r(n(1)),m=r(n(0)),g=r(n(17)),y=r(n(40)),b=r(n(2)),_=n(41),w=r(n(15)),E=r(n(42)),x=r(n(10)),S=r(n(9)),C=r(n(21)),k=n(22),A=new RegExp("^([a-z]+://|//)","i"),O=(0,C.default)("JSONRefError",function(e,t,n){this.originalError=n,(0,b.default)(this,t||{})}),P={},T=new y.default,M={key:"$ref",plugin:function(e,t,n,r){var s=n.slice(0,-1);if(!(0,k.isFreelyNamed)(s)){var l=r.getContext(n).baseDoc;if("string"!=typeof e)return new O("$ref: must be a string (JSON-Ref)",{$ref:e,baseDoc:l,fullPath:n});var f=a(e),p=f[0],d=f[1]||"",v=void 0;try{v=l||p?o(p,l):null}catch(t){return i(t,{pointer:d,$ref:e,basePath:v,fullPath:n})}var g=void 0,y=void 0;if(!h(d,v,s,r)){if(null==v?(y=c(d),void 0===(g=r.get(y))&&(g=new O("Could not resolve reference: "+e,{pointer:d,$ref:e,baseDoc:l,fullPath:n}))):g=null!=(g=u(v,d)).__value?g.__value:g.catch(function(t){throw i(t,{pointer:d,$ref:e,baseDoc:l,fullPath:n})}),g instanceof Error)return[S.default.remove(n),g];var b=S.default.replace(s,g,{$$ref:e});if(v&&v!==l)return[b,S.default.context(s,{baseDoc:v})];try{if(!function(e,t){var n=[e];return t.path.reduce(function(e,t){return n.push(e[t]),e[t]},e),function e(t){return S.default.isObject(t)&&(n.indexOf(t)>=0||(0,m.default)(t).some(function(n){return e(t[n])}))}(t.value)}(r.state,b))return b}catch(e){return null}}}}},I=(0,b.default)(M,{docCache:P,absoluteify:o,clearCache:function(e){void 0!==e?delete P[e]:(0,m.default)(P).forEach(function(e){delete P[e]})},JSONRefError:O,wrapError:i,getDoc:s,split:a,extractFromDoc:u,fetchJSON:function(e){return(0,_.fetch)(e,{headers:{Accept:"application/json, application/yaml"},loadSpec:!0}).then(function(e){return e.text()}).then(function(e){return w.default.safeLoad(e)})},extract:l,jsonPointerToArray:c,unescapeJsonPointerToken:f});t.default=I;var j=function(e){return!e||"/"===e||"#"===e};e.exports=t.default},function(e,t){e.exports=n(914)},function(e,t){e.exports=n(925)},function(e,t){e.exports=n(926)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=function(e){return e&&e.__esModule?e:{default:e}}(n(2)),o=n(22);t.default={key:"allOf",plugin:function(e,t,n,i,a){if(!a.meta||!a.meta.$$ref){var u=n.slice(0,-1);if(!(0,o.isFreelyNamed)(u)){if(!Array.isArray(e)){var s=new TypeError("allOf must be an array");return s.fullPath=n,s}var l=!1,c=a.value;u.forEach(function(e){c&&(c=c[e])}),delete(c=(0,r.default)({},c)).allOf;var f=[i.replace(u,{})].concat(e.map(function(e,t){if(!i.isObject(e)){if(l)return null;l=!0;var r=new TypeError("Elements in allOf must be objects");return r.fullPath=n,r}return i.mergeDeep(u,e)}));return f.push(i.mergeDeep(u,c)),c.$$ref||f.push(i.remove([].concat(u,"$$ref"))),f}}}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=r(n(2)),i=r(n(9));t.default={key:"parameters",plugin:function(e,t,n,r,a){if(Array.isArray(e)&&e.length){var u=(0,o.default)([],e),s=n.slice(0,-1),l=(0,o.default)({},i.default.getIn(r.spec,s));return e.forEach(function(e,t){try{u[t].default=r.parameterMacro(l,e)}catch(e){var o=new Error(e);return o.fullPath=n,o}}),i.default.replace(n,u)}return i.default.replace(n,e)}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=r(n(2)),i=r(n(9));t.default={key:"properties",plugin:function(e,t,n,r){var a=(0,o.default)({},e);for(var u in e)try{a[u].default=r.modelPropertyMacro(a[u])}catch(e){var s=new Error(e);return s.fullPath=n,s}return i.default.replace(n,a)}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e,t){return i({children:{}},e,t)}function i(e,t,n){return e.value=t||{},e.protoValue=n?(0,u.default)({},n.protoValue,e.value):e.value,(0,a.default)(e.children).forEach(function(t){var n=e.children[t];e.children[t]=i(n,n.value,e)}),e}Object.defineProperty(t,"__esModule",{value:!0});var a=r(n(0)),u=r(n(3)),s=r(n(19)),l=r(n(20)),c=function(){function e(t){(0,s.default)(this,e),this.root=o(t||{})}return(0,l.default)(e,[{key:"set",value:function(e,t){var n=this.getParent(e,!0);if(n){var r=e[e.length-1],a=n.children;a[r]?i(a[r],t,n):a[r]=o(t,n)}else i(this.root,t,null)}},{key:"get",value:function(e){if((e=e||[]).length<1)return this.root.value;for(var t=this.root,n=void 0,r=void 0,o=0;o2&&void 0!==arguments[2]?arguments[2]:{};return o.default.wrap(function(e){for(;;)switch(e.prev=e.next){case 0:return r=y.returnEntireTree,a=y.baseDoc,c=y.requestInterceptor,f=y.responseInterceptor,p=y.parameterMacro,d=y.modelPropertyMacro,h={pathDiscriminator:n,baseDoc:a,requestInterceptor:c,responseInterceptor:f,parameterMacro:p,modelPropertyMacro:d},v=(0,l.normalizeSwagger)({spec:t}),m=v.spec,e.next=5,(0,s.default)((0,i.default)({},h,{spec:m,allowMetaPatches:!0,skipNormalization:!0}));case 5:return g=e.sent,!r&&Array.isArray(n)&&n.length&&(g.spec=(0,u.default)(g.spec,n)||null),e.abrupt("return",g);case 8:case"end":return e.stop()}},e,this)}));return function(t,n){return e.apply(this,arguments)}}(),e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{};return function(t){var n=t.pathName,r=t.method,o=t.operationId;return function(t){var i=arguments.length>1&&void 0!==arguments[1]?arguments[1]:{};return e.execute((0,a.default)({spec:e.spec},(0,u.default)(e,"requestInterceptor","responseInterceptor","userFetch"),{pathName:n,method:r,parameters:t,operationId:o},i))}}}function i(e){var t=e.spec,n=e.cb,r=void 0===n?l:n,o=e.defaultTag,i=void 0===o?"default":o,a=e.v2OperationIdCompatibilityMode,u={},f={};return(0,s.eachOperation)(t,function(e){var n=e.pathName,o=e.method,l=e.operation;(l.tags?c(l.tags):[i]).forEach(function(e){if("string"==typeof e){var i=f[e]=f[e]||{},c=(0,s.opId)(l,n,o,{v2OperationIdCompatibilityMode:a}),p=r({spec:t,pathName:n,method:o,operation:l,operationId:c});if(u[c])u[c]++,i[""+c+u[c]]=p;else if(void 0!==i[c]){var d=u[c]||1;u[c]=d+1,i[""+c+u[c]]=p;var h=i[c];delete i[c],i[""+c+d]=h}else i[c]=p}})}),f}Object.defineProperty(t,"__esModule",{value:!0}),t.self=void 0;var a=r(n(3));t.makeExecute=o,t.makeApisTagOperationsOperationExecute=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=f.makeExecute(e),n=f.mapTagOperations({v2OperationIdCompatibilityMode:e.v2OperationIdCompatibilityMode,spec:e.spec,cb:t}),r={};for(var o in n)for(var i in r[o]={operations:{}},n[o])r[o].operations[i]={execute:n[o][i]};return{apis:r}},t.makeApisTagOperation=function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:{},t=f.makeExecute(e);return{apis:f.mapTagOperations({v2OperationIdCompatibilityMode:e.v2OperationIdCompatibilityMode,spec:e.spec,cb:t})}},t.mapTagOperations=i;var u=r(n(50)),s=n(5),l=function(){return null},c=function(e){return Array.isArray(e)?e:[e]},f=t.self={mapTagOperations:i,makeExecute:o}},function(e,t){e.exports=n(927)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=e.spec,n=e.operationId,r=(e.securities,e.requestContentType,e.responseContentType),o=e.scheme,a=e.requestInterceptor,s=e.responseInterceptor,c=e.contextUrl,f=e.userFetch,p=(e.requestBody,e.server),d=e.serverVariables,h=e.http,g=e.parameters,y=e.parameterBuilders,O=(0,x.isOAS3)(t);y||(y=O?_.default:b.default);var P={url:"",credentials:h&&h.withCredentials?"include":"same-origin",headers:{},cookies:{}};a&&(P.requestInterceptor=a),s&&(P.responseInterceptor=s),f&&(P.userFetch=f);var T=(0,x.getOperationRaw)(t,n);if(!T)throw new C("Operation "+n+" not found");var M=T.operation,I=void 0===M?{}:M,j=T.method,N=T.pathName;if(P.url+=i({spec:t,scheme:o,contextUrl:c,server:p,serverVariables:d,pathName:N,method:j}),!n)return delete P.cookies,P;P.url+=N,P.method=(""+j).toUpperCase(),g=g||{};var R=t.paths[N]||{};r&&(P.headers.accept=r);var D=A([].concat(S(I.parameters)).concat(S(R.parameters)));D.forEach(function(e){var n=y[e.in],r=void 0;if("body"===e.in&&e.schema&&e.schema.properties&&(r=g),void 0===(r=e&&e.name&&g[e.name])?r=e&&e.name&&g[e.in+"."+e.name]:k(e.name,D).length>1&&console.warn("Parameter '"+e.name+"' is ambiguous because the defined spec has more than one parameter with the name: '"+e.name+"' and the passed-in parameter values did not define an 'in' value."),null!==r){if(void 0!==e.default&&void 0===r&&(r=e.default),void 0===r&&e.required&&!e.allowEmptyValue)throw new Error("Required parameter "+e.name+" is not provided");if(O&&e.schema&&"object"===e.schema.type&&"string"==typeof r)try{r=JSON.parse(r)}catch(e){throw new Error("Could not parse object parameter value string as JSON")}n&&n({req:P,parameter:e,value:r,operation:I,spec:t})}});var L=(0,u.default)({},e,{operation:I});if((P=O?(0,w.default)(L,P):(0,E.default)(L,P)).cookies&&(0,l.default)(P.cookies).length){var U=(0,l.default)(P.cookies).reduce(function(e,t){var n=P.cookies[t];return e+(e?"&":"")+v.default.serialize(t,n)},"");P.headers.Cookie=U}return P.cookies&&delete P.cookies,(0,m.mergeInQueryOrForm)(P),P}function i(e){return(0,x.isOAS3)(e.spec)?function(e){var t=e.spec,n=e.pathName,r=e.method,o=e.server,i=e.contextUrl,a=e.serverVariables,u=void 0===a?{}:a,s=(0,f.default)(t,["paths",n,(r||"").toLowerCase(),"servers"])||(0,f.default)(t,["paths",n,"servers"])||(0,f.default)(t,["servers"]),l="",c=null;if(o&&s&&s.length){var p=s.map(function(e){return e.url});p.indexOf(o)>-1&&(l=o,c=s[p.indexOf(o)])}!l&&s&&s.length&&(l=s[0].url,c=s[0]),l.indexOf("{")>-1&&function(e){for(var t=[],n=/{([^}]+)}/g,r=void 0;r=n.exec(e);)t.push(r[1]);return t}(l).forEach(function(e){if(c.variables&&c.variables[e]){var t=c.variables[e],n=u[e]||t.default,r=new RegExp("{"+e+"}","g");l=l.replace(r,n)}});return function(){var e=arguments.length>0&&void 0!==arguments[0]?arguments[0]:"",t=arguments.length>1&&void 0!==arguments[1]?arguments[1]:"",n=h.default.parse(e),r=h.default.parse(t),o=P(n.protocol)||P(r.protocol)||"",i=n.host||r.host,a=n.pathname||"",u=void 0;return"/"===(u=o&&i?o+"://"+(i+a):a)[u.length-1]?u.slice(0,-1):u}(l,i)}(e):function(e){var t=e.spec,n=e.scheme,r=e.contextUrl,o=void 0===r?"":r,i=h.default.parse(o),a=Array.isArray(t.schemes)?t.schemes[0]:null,u=n||a||P(i.protocol)||"http",s=t.host||i.host||"",l=t.basePath||"",c=void 0;return"/"===(c=u&&s?u+"://"+(s+l):l)[c.length-1]?c.slice(0,-1):c}(e)}Object.defineProperty(t,"__esModule",{value:!0}),t.self=void 0;var a=r(n(8)),u=r(n(3)),s=r(n(52)),l=r(n(0)),c=r(n(2));t.execute=function(e){var t=e.http,n=e.fetch,r=e.spec,o=e.operationId,i=e.pathName,l=e.method,c=e.parameters,f=e.securities,h=(0,s.default)(e,["http","fetch","spec","operationId","pathName","method","parameters","securities"]),v=t||n||g.default;i&&l&&!o&&(o=(0,x.legacyIdFromPathMethod)(i,l));var m=O.buildRequest((0,u.default)({spec:r,operationId:o,parameters:c,securities:f,http:v},h));return m.body&&((0,p.default)(m.body)||(0,d.default)(m.body))&&(m.body=(0,a.default)(m.body)),v(m)},t.buildRequest=o,t.baseUrl=i;var f=r((r(n(6)),n(12))),p=r(n(53)),d=r(n(54)),h=r((r(n(13)),n(10))),v=r(n(55)),m=n(7),g=r(m),y=r(n(21)),b=r(n(56)),_=r(n(57)),w=r(n(62)),E=r(n(64)),x=n(5),S=function(e){return Array.isArray(e)?e:[]},C=(0,y.default)("OperationNotFoundError",function(e,t,n){this.originalError=n,(0,c.default)(this,t||{})}),k=function(e,t){return t.filter(function(t){return t.name===e})},A=function(e){var t={};e.forEach(function(e){t[e.in]||(t[e.in]={}),t[e.in][e.name]=e});var n=[];return(0,l.default)(t).forEach(function(e){(0,l.default)(t[e]).forEach(function(r){n.push(t[e][r])})}),n},O=t.self={buildRequest:o},P=function(e){return e?e.replace(/\W/g,""):null}},function(e,t){e.exports=n(84)},function(e,t){e.exports=n(228)},function(e,t){e.exports=n(24)},function(e,t){e.exports=n(930)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default={body:function(e){var t=e.req,n=e.value;t.body=n},header:function(e){var t=e.req,n=e.parameter,r=e.value;t.headers=t.headers||{},void 0!==r&&(t.headers[n.name]=r)},query:function(e){var t=e.req,n=e.value,r=e.parameter;if(t.query=t.query||{},!1===n&&"boolean"===r.type&&(n="false"),0===n&&["number","integer"].indexOf(r.type)>-1&&(n="0"),n)t.query[r.name]={collectionFormat:r.collectionFormat,value:n};else if(r.allowEmptyValue&&void 0!==n){var o=r.name;t.query[o]=t.query[o]||{},t.query[o].allowEmptyValue=!0}},path:function(e){var t=e.req,n=e.value,r=e.parameter;t.url=t.url.split("{"+r.name+"}").join(encodeURIComponent(n))},formData:function(e){var t=e.req,n=e.value,r=e.parameter;(n||r.allowEmptyValue)&&(t.form=t.form||{},t.form[r.name]={value:n,allowEmptyValue:r.allowEmptyValue,collectionFormat:r.collectionFormat})}},e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}Object.defineProperty(t,"__esModule",{value:!0});var o=r(n(0)),i=r(n(1)),a=r(n(58));t.default={path:function(e){var t=e.req,n=e.value,r=e.parameter,o=r.name,i=r.style,u=r.explode,s=(0,a.default)({key:r.name,value:n,style:i||"simple",explode:u||!1,escape:!0});t.url=t.url.split("{"+o+"}").join(s)},query:function(e){var t=e.req,n=e.value,r=e.parameter;if(t.query=t.query||{},!1===n&&(n="false"),0===n&&(n="0"),n){var u=void 0===n?"undefined":(0,i.default)(n);"deepObject"===r.style?(0,o.default)(n).forEach(function(e){var o=n[e];t.query[r.name+"["+e+"]"]={value:(0,a.default)({key:e,value:o,style:"deepObject",escape:r.allowReserved?"unsafe":"reserved"}),skipEncoding:!0}}):"object"!==u||Array.isArray(n)||"form"!==r.style&&r.style||!r.explode&&void 0!==r.explode?t.query[r.name]={value:(0,a.default)({key:r.name,value:n,style:r.style||"form",explode:void 0===r.explode||r.explode,escape:r.allowReserved?"unsafe":"reserved"}),skipEncoding:!0}:(0,o.default)(n).forEach(function(e){var o=n[e];t.query[e]={value:(0,a.default)({key:e,value:o,style:r.style||"form",escape:r.allowReserved?"unsafe":"reserved"}),skipEncoding:!0}})}else if(r.allowEmptyValue&&void 0!==n){var s=r.name;t.query[s]=t.query[s]||{},t.query[s].allowEmptyValue=!0}},header:function(e){var t=e.req,n=e.parameter,r=e.value;t.headers=t.headers||{},u.indexOf(n.name.toLowerCase())>-1||void 0!==r&&(t.headers[n.name]=(0,a.default)({key:n.name,value:r,style:n.style||"simple",explode:void 0!==n.explode&&n.explode,escape:!1}))},cookie:function(e){var t=e.req,n=e.parameter,r=e.value;t.headers=t.headers||{};var o=void 0===r?"undefined":(0,i.default)(r);if("undefined"!==o){var u="object"===o&&!Array.isArray(r)&&n.explode?"":n.name+"=";t.headers.Cookie=u+(0,a.default)({key:n.name,value:r,escape:!1,style:n.style||"form",explode:void 0!==n.explode&&n.explode})}}};var u=["accept","authorization","content-type"];e.exports=t.default},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=(arguments.length>1&&void 0!==arguments[1]?arguments[1]:{}).escape,n=arguments[2];return"number"==typeof e&&(e=e.toString()),"string"==typeof e&&e.length&&t?n?JSON.parse(e):(0,s.stringToCharArray)(e).map(function(e){return c(e)?e:l(e)&&"unsafe"===t?e:((0,u.default)(e)||[]).map(function(e){return("0"+e.toString(16).toUpperCase()).slice(-2)}).map(function(e){return"%"+e}).join("")}).join(""):e}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(0)),a=r(n(1));t.encodeDisallowedCharacters=o,t.default=function(e){var t=e.value;return Array.isArray(t)?function(e){var t=e.key,n=e.value,r=e.style,i=e.explode,a=e.escape,u=function(e){return o(e,{escape:a})};if("simple"===r)return n.map(function(e){return u(e)}).join(",");if("label"===r)return"."+n.map(function(e){return u(e)}).join(".");if("matrix"===r)return n.map(function(e){return u(e)}).reduce(function(e,n){return!e||i?(e||"")+";"+t+"="+n:e+","+n},"");if("form"===r){var s=i?"&"+t+"=":",";return n.map(function(e){return u(e)}).join(s)}if("spaceDelimited"===r){var l=i?t+"=":"";return n.map(function(e){return u(e)}).join(" "+l)}if("pipeDelimited"===r){var c=i?t+"=":"";return n.map(function(e){return u(e)}).join("|"+c)}}(e):"object"===(void 0===t?"undefined":(0,a.default)(t))?function(e){var t=e.key,n=e.value,r=e.style,a=e.explode,u=e.escape,s=function(e){return o(e,{escape:u})},l=(0,i.default)(n);return"simple"===r?l.reduce(function(e,t){var r=s(n[t]);return(e?e+",":"")+t+(a?"=":",")+r},""):"label"===r?l.reduce(function(e,t){var r=s(n[t]);return(e?e+".":".")+t+(a?"=":".")+r},""):"matrix"===r&&a?l.reduce(function(e,t){var r=s(n[t]);return(e?e+";":";")+t+"="+r},""):"matrix"===r?l.reduce(function(e,r){var o=s(n[r]);return(e?e+",":";"+t+"=")+r+","+o},""):"form"===r?l.reduce(function(e,t){var r=s(n[t]);return(e?e+(a?"&":","):"")+t+(a?"=":",")+r},""):void 0}(e):function(e){var t=e.key,n=e.value,r=e.style,i=e.escape,a=function(e){return o(e,{escape:i})};return"simple"===r?a(n):"label"===r?"."+a(n):"matrix"===r?";"+t+"="+a(n):"form"===r?a(n):"deepObject"===r?a(n):void 0}(e)};var u=r((r(n(59)),n(60))),s=n(61),l=function(e){return":/?#[]@!$&'()*+,;=".indexOf(e)>-1},c=function(e){return/^[a-z0-9\-._~]+$/i.test(e)}},function(e,t){e.exports=n(931)},function(e,t){e.exports=n(932)},function(e,t){e.exports=n(933)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=e.request,n=e.securities,r=void 0===n?{}:n,o=e.operation,i=void 0===o?{}:o,a=e.spec,f=(0,s.default)({},t),p=r.authorized,d=void 0===p?{}:p,h=i.security||a.security||[],v=d&&!!(0,u.default)(d).length,m=(0,l.default)(a,["components","securitySchemes"])||{};return f.headers=f.headers||{},f.query=f.query||{},(0,u.default)(r).length&&v&&h&&(!Array.isArray(i.security)||i.security.length)?(h.forEach(function(e,t){for(var n in e){var r=d[n],o=m[n];if(r){var i=r.value||r,a=o.type;if(r)if("apiKey"===a)"query"===o.in&&(f.query[o.name]=i),"header"===o.in&&(f.headers[o.name]=i),"cookie"===o.in&&(f.cookies[o.name]=i);else if("http"===a){if("basic"===o.scheme){var u=i.username,s=i.password,l=(0,c.default)(u+":"+s);f.headers.Authorization="Basic "+l}"bearer"===o.scheme&&(f.headers.Authorization="Bearer "+i)}else if("oauth2"===a){var p=r.token||{},h=p.access_token,v=p.token_type;v&&"bearer"!==v.toLowerCase()||(v="Bearer"),f.headers.Authorization=v+" "+h}}}}),f):t}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(8)),a=r(n(1)),u=r(n(0));t.default=function(e,t){var n=e.operation,r=e.requestBody,s=e.securities,l=e.spec,c=e.attachContentTypeForEmptyPayload,p=e.requestContentType;t=o({request:t,securities:s,operation:n,spec:l});var d=n.requestBody||{},h=(0,u.default)(d.content||{}),v=p&&h.indexOf(p)>-1;if(r||c){if(p&&v)t.headers["Content-Type"]=p;else if(!p){var m=h[0];m&&(t.headers["Content-Type"]=m,p=m)}}else p&&v&&(t.headers["Content-Type"]=p);return r&&(p?h.indexOf(p)>-1&&("application/x-www-form-urlencoded"===p||0===p.indexOf("multipart/")?"object"===(void 0===r?"undefined":(0,a.default)(r))?(t.form={},(0,u.default)(r).forEach(function(e){var n,o=r[e],u=void 0;"undefined"!=typeof File&&(u=o instanceof File),"undefined"!=typeof Blob&&(u=u||o instanceof Blob),void 0!==f.Buffer&&(u=u||f.Buffer.isBuffer(o)),n="object"!==(void 0===o?"undefined":(0,a.default)(o))||u?o:Array.isArray(o)?o.toString():(0,i.default)(o),t.form[e]={value:n}})):t.form=r:t.body=r):t.body=r),t},t.applySecurities=o;var s=r(n(6)),l=r(n(12)),c=r(n(13)),f=n(63)},function(e,t){e.exports=n(54)},function(e,t,n){"use strict";function r(e){return e&&e.__esModule?e:{default:e}}function o(e){var t=e.request,n=e.securities,r=void 0===n?{}:n,o=e.operation,s=void 0===o?{}:o,l=e.spec,c=(0,u.default)({},t),f=r.authorized,p=void 0===f?{}:f,d=r.specSecurity,h=void 0===d?[]:d,v=s.security||h,m=p&&!!(0,i.default)(p).length,g=l.securityDefinitions;return c.headers=c.headers||{},c.query=c.query||{},(0,i.default)(r).length&&m&&v&&(!Array.isArray(s.security)||s.security.length)?(v.forEach(function(e,t){for(var n in e){var r=p[n];if(r){var o=r.token,i=r.value||r,u=g[n],s=u.type,l=u["x-tokenName"]||"access_token",f=o&&o[l],d=o&&o.token_type;if(r)if("apiKey"===s){var h="query"===u.in?"query":"headers";c[h]=c[h]||{},c[h][u.name]=i}else"basic"===s?i.header?c.headers.authorization=i.header:(i.base64=(0,a.default)(i.username+":"+i.password),c.headers.authorization="Basic "+i.base64):"oauth2"===s&&f&&(d=d&&"bearer"!==d.toLowerCase()?d:"Bearer",c.headers.authorization=d+" "+f)}}}),c):t}Object.defineProperty(t,"__esModule",{value:!0});var i=r(n(0));t.default=function(e,t){var n=e.spec,r=e.operation,i=e.securities,a=e.requestContentType,u=e.attachContentTypeForEmptyPayload;if((t=o({request:t,securities:i,operation:r,spec:n})).body||t.form||u)a?t.headers["Content-Type"]=a:Array.isArray(r.consumes)?t.headers["Content-Type"]=r.consumes[0]:Array.isArray(n.consumes)?t.headers["Content-Type"]=n.consumes[0]:r.parameters&&r.parameters.filter(function(e){return"file"===e.type}).length?t.headers["Content-Type"]="multipart/form-data":r.parameters&&r.parameters.filter(function(e){return"formData"===e.in}).length&&(t.headers["Content-Type"]="application/x-www-form-urlencoded");else if(a){var s=r.parameters&&r.parameters.filter(function(e){return"body"===e.in}).length>0,l=r.parameters&&r.parameters.filter(function(e){return"formData"===e.in}).length>0;(s||l)&&(t.headers["Content-Type"]=a)}return t},t.applySecurities=o;var a=r(n(13)),u=r(n(6));r(n(7))}])},function(e,t,n){"use strict";var r=Object.prototype.hasOwnProperty,o=function(){for(var e=[],t=0;t<256;++t)e.push("%"+((t<16?"0":"")+t.toString(16)).toUpperCase());return e}();t.arrayToObject=function(e,t){for(var n=t&&t.plainObjects?Object.create(null):{},r=0;r=48&&i<=57||i>=65&&i<=90||i>=97&&i<=122?n+=t.charAt(r):i<128?n+=o[i]:i<2048?n+=o[192|i>>6]+o[128|63&i]:i<55296||i>=57344?n+=o[224|i>>12]+o[128|i>>6&63]+o[128|63&i]:(r+=1,i=65536+((1023&i)<<10|1023&t.charCodeAt(r)),n+=o[240|i>>18]+o[128|i>>12&63]+o[128|i>>6&63]+o[128|63&i])}return n},t.compact=function(e){for(var t=[{obj:{o:e},prop:"o"}],n=[],r=0;r=0;l--)if(f[l]!=p[l])return!1;for(l=f.length-1;l>=0;l--)if(c=f[l],!a(e[c],t[c],n))return!1;return typeof e==typeof t}(e,t,n))};function u(e){return null===e||void 0===e}function s(e){return!(!e||"object"!=typeof e||"number"!=typeof e.length)&&("function"==typeof e.copy&&"function"==typeof e.slice&&!(e.length>0&&"number"!=typeof e[0]))}},function(e,t,n){var r={strict:!0},o=n(390),i=function(e,t){return o(e,t,r)},a=n(231);t.JsonPatchError=a.PatchError,t.deepClone=a._deepClone;var u={add:function(e,t,n){return e[t]=this.value,{newDocument:n}},remove:function(e,t,n){var r=e[t];return delete e[t],{newDocument:n,removed:r}},replace:function(e,t,n){var r=e[t];return e[t]=this.value,{newDocument:n,removed:r}},move:function(e,t,n){var r=l(n,this.path);r&&(r=a._deepClone(r));var o=c(n,{op:"remove",path:this.from}).removed;return c(n,{op:"add",path:this.path,value:o}),{newDocument:n,removed:r}},copy:function(e,t,n){var r=l(n,this.from);return c(n,{op:"add",path:this.path,value:a._deepClone(r)}),{newDocument:n}},test:function(e,t,n){return{newDocument:n,test:i(e[t],this.value)}},_get:function(e,t,n){return this.value=e[t],{newDocument:n}}},s={add:function(e,t,n){return a.isInteger(t)?e.splice(t,0,this.value):e[t]=this.value,{newDocument:n,index:t}},remove:function(e,t,n){return{newDocument:n,removed:e.splice(t,1)[0]}},replace:function(e,t,n){var r=e[t];return e[t]=this.value,{newDocument:n,removed:r}},move:u.move,copy:u.copy,test:u.test,_get:u._get};function l(e,t){if(""==t)return e;var n={op:"_get",path:t};return c(e,n),n.value}function c(e,n,r,o){if(void 0===r&&(r=!1),void 0===o&&(o=!0),r&&("function"==typeof r?r(n,0,e,n.path):p(n,0)),""===n.path){var c={newDocument:e};if("add"===n.op)return c.newDocument=n.value,c;if("replace"===n.op)return c.newDocument=n.value,c.removed=e,c;if("move"===n.op||"copy"===n.op)return c.newDocument=l(e,n.from),"move"===n.op&&(c.removed=e),c;if("test"===n.op){if(c.test=i(e,n.value),!1===c.test)throw new t.JsonPatchError("Test operation failed","TEST_OPERATION_FAILED",0,n,e);return c.newDocument=e,c}if("remove"===n.op)return c.removed=e,c.newDocument=null,c;if("_get"===n.op)return n.value=e,c;if(r)throw new t.JsonPatchError("Operation `op` property is not one of operations defined in RFC-6902","OPERATION_OP_INVALID",0,n,e);return c}o||(e=a._deepClone(e));var f=(n.path||"").split("/"),d=e,h=1,v=f.length,m=void 0,g=void 0,y=void 0;for(y="function"==typeof r?r:p;;){if(g=f[h],r&&void 0===m&&(void 0===d[g]?m=f.slice(0,h).join("/"):h==v-1&&(m=n.path),void 0!==m&&y(n,0,e,m)),h++,Array.isArray(d)){if("-"===g)g=d.length;else{if(r&&!a.isInteger(g))throw new t.JsonPatchError("Expected an unsigned base-10 integer value, making the new referenced value the array element with the zero-based index","OPERATION_PATH_ILLEGAL_ARRAY_INDEX",0,n.path,n);a.isInteger(g)&&(g=~~g)}if(h>=v){if(r&&"add"===n.op&&g>d.length)throw new t.JsonPatchError("The specified index MUST NOT be greater than the number of elements in the array","OPERATION_VALUE_OUT_OF_BOUNDS",0,n.path,n);if(!1===(c=s[n.op].call(n,d,g,e)).test)throw new t.JsonPatchError("Test operation failed","TEST_OPERATION_FAILED",0,n,e);return c}}else if(g&&-1!=g.indexOf("~")&&(g=a.unescapePathComponent(g)),h>=v){if(!1===(c=u[n.op].call(n,d,g,e)).test)throw new t.JsonPatchError("Test operation failed","TEST_OPERATION_FAILED",0,n,e);return c}d=d[g]}}function f(e,n,r,o){if(void 0===o&&(o=!0),r&&!Array.isArray(n))throw new t.JsonPatchError("Patch sequence must be an array","SEQUENCE_NOT_AN_ARRAY");o||(e=a._deepClone(e));for(var i=new Array(n.length),u=0,s=n.length;u0)throw new t.JsonPatchError('Operation `path` property must start with "/"',"OPERATION_PATH_INVALID",n,e,r);if(("move"===e.op||"copy"===e.op)&&"string"!=typeof e.from)throw new t.JsonPatchError("Operation `from` property is not present (applicable in `move` and `copy` operations)","OPERATION_FROM_REQUIRED",n,e,r);if(("add"===e.op||"replace"===e.op||"test"===e.op)&&void 0===e.value)throw new t.JsonPatchError("Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)","OPERATION_VALUE_REQUIRED",n,e,r);if(("add"===e.op||"replace"===e.op||"test"===e.op)&&a.hasUndefined(e.value))throw new t.JsonPatchError("Operation `value` property is not present (applicable in `add`, `replace` and `test` operations)","OPERATION_VALUE_CANNOT_CONTAIN_UNDEFINED",n,e,r);if(r)if("add"==e.op){var i=e.path.split("/").length,s=o.split("/").length;if(i!==s+1&&i!==s)throw new t.JsonPatchError("Cannot perform an `add` operation at the desired path","OPERATION_PATH_CANNOT_ADD",n,e,r)}else if("replace"===e.op||"remove"===e.op||"_get"===e.op){if(e.path!==o)throw new t.JsonPatchError("Cannot perform the operation at a path that does not exist","OPERATION_PATH_UNRESOLVABLE",n,e,r)}else if("move"===e.op||"copy"===e.op){var l=d([{op:"_get",path:e.from,value:void 0}],r);if(l&&"OPERATION_PATH_UNRESOLVABLE"===l.name)throw new t.JsonPatchError("Cannot perform the operation from a path that does not exist","OPERATION_FROM_UNRESOLVABLE",n,e,r)}}function d(e,n,r){try{if(!Array.isArray(e))throw new t.JsonPatchError("Patch sequence must be an array","SEQUENCE_NOT_AN_ARRAY");if(n)f(a._deepClone(n),a._deepClone(e),r||!0);else{r=r||p;for(var o=0;o1&&void 0!==arguments[1]?arguments[1]:(0,a.List)();return function(e){return(e.authSelectors.definitionsToAuthorize()||(0,a.List)()).filter(function(e){return t.some(function(t){return t.get(e.keySeq().first())})})}},t.authorized=(0,i.createSelector)(s,function(e){return e.get("authorized")||(0,a.Map)()}),t.isAuthorized=function(e,t){return function(e){var n=e.authSelectors.authorized();return a.List.isList(t)?!!t.toJS().filter(function(e){return-1===(0,r.default)(e).map(function(e){return!!n.get(e)}).indexOf(!1)}).length:null}},t.getConfigs=(0,i.createSelector)(s,function(e){return e.get("configs")})},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.execute=void 0;var r,o=n(25),i=(r=o)&&r.__esModule?r:{default:r};t.execute=function(e,t){var n=t.authSelectors,r=t.specSelectors;return function(t){var o=t.path,a=t.method,u=t.operation,s=t.extras,l={authorized:n.authorized()&&n.authorized().toJS(),definitions:r.securityDefinitions()&&r.securityDefinitions().toJS(),specSecurity:r.security()&&r.security().toJS()};return e((0,i.default)({path:o,method:a,operation:u,securities:l},s))}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{fn:{shallowEqualKeys:r.shallowEqualKeys}}};var r=n(9)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=s(n(41)),o=s(n(23));t.default=function(e){var t=e.fn,n={download:function(e){return function(n){var r=n.errActions,i=n.specSelectors,a=n.specActions,s=n.getConfigs,l=t.fetch,c=s();function f(t){if(t instanceof Error||t.status>=400)return a.updateLoadingStatus("failed"),r.newThrownErr((0,o.default)(new Error((t.message||t.statusText)+" "+e),{source:"fetch"})),void(!t.status&&t instanceof Error&&function(){try{var t=void 0;if("URL"in u.default?t=new URL(e):(t=document.createElement("a")).href=e,"https:"!==t.protocol&&"https:"===u.default.location.protocol){var n=(0,o.default)(new Error("Possible mixed-content issue? The page was loaded over https:// but a "+t.protocol+"// URL was specified. Check that you are not attempting to load mixed content."),{source:"fetch"});return void r.newThrownErr(n)}if(t.origin!==u.default.location.origin){var i=(0,o.default)(new Error("Possible cross-origin (CORS) issue? The URL origin ("+t.origin+") does not match the page ("+u.default.location.origin+"). Check the server returns the correct 'Access-Control-Allow-*' headers."),{source:"fetch"});r.newThrownErr(i)}}catch(e){return}}());a.updateLoadingStatus("success"),a.updateSpec(t.text),i.url()!==e&&a.updateUrl(e)}e=e||i.url(),a.updateLoadingStatus("loading"),r.clear({source:"fetch"}),l({url:e,loadSpec:!0,requestInterceptor:c.requestInterceptor||function(e){return e},responseInterceptor:c.responseInterceptor||function(e){return e},credentials:"same-origin",headers:{Accept:"application/json,*/*"}}).then(f,f)}},updateLoadingStatus:function(e){var t=[null,"loading","failed","success","failedConfig"];return-1===t.indexOf(e)&&console.error("Error: "+e+" is not one of "+(0,r.default)(t)),{type:"spec_update_loading_status",payload:e}}},s={loadingStatus:(0,i.createSelector)(function(e){return e||(0,a.Map)()},function(e){return e.get("loadingStatus")||null})};return{statePlugins:{spec:{actions:n,reducers:{spec_update_loading_status:function(e,t){return"string"==typeof t.payload?e.set("loadingStatus",t.payload):e}},selectors:s}}}};var i=n(58),a=n(7),u=s(n(32));function s(e){return e&&e.__esModule?e:{default:e}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return{statePlugins:{spec:{actions:a,selectors:f},configs:{reducers:s.default,actions:i,selectors:u}}}};var r=c(n(934)),o=n(234),i=l(n(235)),a=l(n(401)),u=l(n(402)),s=c(n(403));function l(e){if(e&&e.__esModule)return e;var t={};if(null!=e)for(var n in e)Object.prototype.hasOwnProperty.call(e,n)&&(t[n]=e[n]);return t.default=e,t}function c(e){return e&&e.__esModule?e:{default:e}}var f={getLocalConfig:function(){return(0,o.parseYamlConfig)(r.default)}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getConfigByUrl=t.downloadConfig=void 0;var r=n(234);t.downloadConfig=function(e){return function(t){return(0,t.fn.fetch)(e)}},t.getConfigByUrl=function(e,t){return function(n){var o=n.specActions;if(e)return o.downloadConfig(e).then(i,i);function i(n){n instanceof Error||n.status>=400?(o.updateLoadingStatus("failedConfig"),o.updateLoadingStatus("failedConfig"),o.updateUrl(""),console.error(n.statusText+" "+e.url),t(null)):t((0,r.parseYamlConfig)(n.text))}}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});t.get=function(e,t){return e.getIn(Array.isArray(t)?t:[t])}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r,o,i=n(22),a=(r=i)&&r.__esModule?r:{default:r},u=n(7),s=n(235);t.default=(o={},(0,a.default)(o,s.UPDATE_CONFIGS,function(e,t){return e.merge((0,u.fromJS)(t.payload))}),(0,a.default)(o,s.TOGGLE_CONFIGS,function(e,t){var n=t.payload,r=e.get(n);return e.set(n,!r)}),o)},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(){return[r.default,{statePlugins:{configs:{wrapActions:{loaded:function(e,t){return function(){e.apply(void 0,arguments);var n=decodeURIComponent(window.location.hash);t.layoutActions.parseDeepLinkHash(n)}}}}},wrapComponents:{operation:o.default,OperationTag:i.default}}]};var r=a(n(405)),o=a(n(407)),i=a(n(408));function a(e){return e&&e.__esModule?e:{default:e}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.clearScrollTo=t.scrollToElement=t.readyToScroll=t.parseDeepLinkHash=t.scrollTo=t.show=void 0;var r,o=f(n(22)),i=f(n(17)),a=n(406),u=f(n(935)),s=n(9),l=n(7),c=f(l);function f(e){return e&&e.__esModule?e:{default:e}}var p=t.show=function(e,t){var n=t.getConfigs,r=t.layoutSelectors;return function(){for(var t=arguments.length,o=Array(t),u=0;u-1&&(console.warn("Warning: escaping deep link whitespace with `_` will be unsupported in v4.0, use `%20` instead."),n.show(h.map(function(e){return e.replace(/_/g," ")}),!0)),n.show(h,!0)}(f.indexOf("_")>-1||d.indexOf("_")>-1)&&(console.warn("Warning: escaping deep link whitespace with `_` will be unsupported in v4.0, use `%20` instead."),n.show(u.map(function(e){return e.replace(/_/g," ")}),!0)),n.show(u,!0),n.scrollTo(u)}}},v=t.readyToScroll=function(e,t){return function(n){var r=n.layoutSelectors.getScrollToKey();c.default.is(r,(0,l.fromJS)(e))&&(n.layoutActions.scrollToElement(t),n.layoutActions.clearScrollTo())}},m=t.scrollToElement=function(e,t){return function(n){try{t=t||n.fn.getScrollParent(e),u.default.createScroller(t).to(e)}catch(e){console.error(e)}}},g=t.clearScrollTo=function(){return{type:"layout_clear_scroll"}};t.default={fn:{getScrollParent:function(e,t){var n=document.documentElement,r=getComputedStyle(e),o="absolute"===r.position,i=t?/(auto|scroll|hidden)/:/(auto|scroll)/;if("fixed"===r.position)return n;for(var a=e;a=a.parentElement;)if(r=getComputedStyle(a),(!o||"static"!==r.position)&&i.test(r.overflow+r.overflowY+r.overflowX))return a;return n}},statePlugins:{layout:{actions:{scrollToElement:m,scrollTo:d,clearScrollTo:g,readyToScroll:v,parseDeepLinkHash:h},selectors:{getScrollToKey:function(e){return e.get("scrollToKey")},isShownKeyFromUrlHashArray:function(e,t){var n=(0,i.default)(t,2),r=n[0],o=n[1];return o?["operations",r,o]:r?["operations-tag",r]:[]},urlHashArrayFromIsShownKey:function(e,t){var n=(0,i.default)(t,3),r=n[0],o=n[1],a=n[2];return"operations"==r?[o,a]:"operations-tag"==r?[o]:[]}},reducers:(r={},(0,o.default)(r,"layout_scroll_to",function(e,t){return e.set("scrollToKey",c.default.fromJS(t.payload))}),(0,o.default)(r,"layout_clear_scroll",function(e){return e.delete("scrollToKey")}),r),wrapActions:{show:p}}}}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});t.setHash=function(e){return e?history.pushState(null,null,"#"+e):window.location.hash=""}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=l(n(4)),o=l(n(2)),i=l(n(3)),a=l(n(5)),u=l(n(6)),s=l(n(0));l(n(12));function l(e){return e&&e.__esModule?e:{default:e}}t.default=function(e,t){return function(n){function l(){var e,n,i,u;(0,o.default)(this,l);for(var s=arguments.length,c=Array(s),f=0;f",Gt:"≫",gt:">",gtcc:"⪧",gtcir:"⩺",gtdot:"⋗",gtlPar:"⦕",gtquest:"⩼",gtrapprox:"⪆",gtrarr:"⥸",gtrdot:"⋗",gtreqless:"⋛",gtreqqless:"⪌",gtrless:"≷",gtrsim:"≳",gvertneqq:"≩︀",gvnE:"≩︀",Hacek:"ˇ",hairsp:" ",half:"½",hamilt:"ℋ",HARDcy:"Ъ",hardcy:"ъ",hArr:"⇔",harr:"↔",harrcir:"⥈",harrw:"↭",Hat:"^",hbar:"ℏ",Hcirc:"Ĥ",hcirc:"ĥ",hearts:"♥",heartsuit:"♥",hellip:"…",hercon:"⊹",Hfr:"ℌ",hfr:"𝔥",HilbertSpace:"ℋ",hksearow:"⤥",hkswarow:"⤦",hoarr:"⇿",homtht:"∻",hookleftarrow:"↩",hookrightarrow:"↪",Hopf:"ℍ",hopf:"𝕙",horbar:"―",HorizontalLine:"─",Hscr:"ℋ",hscr:"𝒽",hslash:"ℏ",Hstrok:"Ħ",hstrok:"ħ",HumpDownHump:"≎",HumpEqual:"≏",hybull:"⁃",hyphen:"‐",Iacute:"Í",iacute:"í",ic:"⁣",Icirc:"Î",icirc:"î",Icy:"И",icy:"и",Idot:"İ",IEcy:"Е",iecy:"е",iexcl:"¡",iff:"⇔",Ifr:"ℑ",ifr:"𝔦",Igrave:"Ì",igrave:"ì",ii:"ⅈ",iiiint:"⨌",iiint:"∭",iinfin:"⧜",iiota:"℩",IJlig:"IJ",ijlig:"ij",Im:"ℑ",Imacr:"Ī",imacr:"ī",image:"ℑ",ImaginaryI:"ⅈ",imagline:"ℐ",imagpart:"ℑ",imath:"ı",imof:"⊷",imped:"Ƶ",Implies:"⇒",in:"∈",incare:"℅",infin:"∞",infintie:"⧝",inodot:"ı",Int:"∬",int:"∫",intcal:"⊺",integers:"ℤ",Integral:"∫",intercal:"⊺",Intersection:"⋂",intlarhk:"⨗",intprod:"⨼",InvisibleComma:"⁣",InvisibleTimes:"⁢",IOcy:"Ё",iocy:"ё",Iogon:"Į",iogon:"į",Iopf:"𝕀",iopf:"𝕚",Iota:"Ι",iota:"ι",iprod:"⨼",iquest:"¿",Iscr:"ℐ",iscr:"𝒾",isin:"∈",isindot:"⋵",isinE:"⋹",isins:"⋴",isinsv:"⋳",isinv:"∈",it:"⁢",Itilde:"Ĩ",itilde:"ĩ",Iukcy:"І",iukcy:"і",Iuml:"Ï",iuml:"ï",Jcirc:"Ĵ",jcirc:"ĵ",Jcy:"Й",jcy:"й",Jfr:"𝔍",jfr:"𝔧",jmath:"ȷ",Jopf:"𝕁",jopf:"𝕛",Jscr:"𝒥",jscr:"𝒿",Jsercy:"Ј",jsercy:"ј",Jukcy:"Є",jukcy:"є",Kappa:"Κ",kappa:"κ",kappav:"ϰ",Kcedil:"Ķ",kcedil:"ķ",Kcy:"К",kcy:"к",Kfr:"𝔎",kfr:"𝔨",kgreen:"ĸ",KHcy:"Х",khcy:"х",KJcy:"Ќ",kjcy:"ќ",Kopf:"𝕂",kopf:"𝕜",Kscr:"𝒦",kscr:"𝓀",lAarr:"⇚",Lacute:"Ĺ",lacute:"ĺ",laemptyv:"⦴",lagran:"ℒ",Lambda:"Λ",lambda:"λ",Lang:"⟪",lang:"⟨",langd:"⦑",langle:"⟨",lap:"⪅",Laplacetrf:"ℒ",laquo:"«",Larr:"↞",lArr:"⇐",larr:"←",larrb:"⇤",larrbfs:"⤟",larrfs:"⤝",larrhk:"↩",larrlp:"↫",larrpl:"⤹",larrsim:"⥳",larrtl:"↢",lat:"⪫",lAtail:"⤛",latail:"⤙",late:"⪭",lates:"⪭︀",lBarr:"⤎",lbarr:"⤌",lbbrk:"❲",lbrace:"{",lbrack:"[",lbrke:"⦋",lbrksld:"⦏",lbrkslu:"⦍",Lcaron:"Ľ",lcaron:"ľ",Lcedil:"Ļ",lcedil:"ļ",lceil:"⌈",lcub:"{",Lcy:"Л",lcy:"л",ldca:"⤶",ldquo:"“",ldquor:"„",ldrdhar:"⥧",ldrushar:"⥋",ldsh:"↲",lE:"≦",le:"≤",LeftAngleBracket:"⟨",LeftArrow:"←",Leftarrow:"⇐",leftarrow:"←",LeftArrowBar:"⇤",LeftArrowRightArrow:"⇆",leftarrowtail:"↢",LeftCeiling:"⌈",LeftDoubleBracket:"⟦",LeftDownTeeVector:"⥡",LeftDownVector:"⇃",LeftDownVectorBar:"⥙",LeftFloor:"⌊",leftharpoondown:"↽",leftharpoonup:"↼",leftleftarrows:"⇇",LeftRightArrow:"↔",Leftrightarrow:"⇔",leftrightarrow:"↔",leftrightarrows:"⇆",leftrightharpoons:"⇋",leftrightsquigarrow:"↭",LeftRightVector:"⥎",LeftTee:"⊣",LeftTeeArrow:"↤",LeftTeeVector:"⥚",leftthreetimes:"⋋",LeftTriangle:"⊲",LeftTriangleBar:"⧏",LeftTriangleEqual:"⊴",LeftUpDownVector:"⥑",LeftUpTeeVector:"⥠",LeftUpVector:"↿",LeftUpVectorBar:"⥘",LeftVector:"↼",LeftVectorBar:"⥒",lEg:"⪋",leg:"⋚",leq:"≤",leqq:"≦",leqslant:"⩽",les:"⩽",lescc:"⪨",lesdot:"⩿",lesdoto:"⪁",lesdotor:"⪃",lesg:"⋚︀",lesges:"⪓",lessapprox:"⪅",lessdot:"⋖",lesseqgtr:"⋚",lesseqqgtr:"⪋",LessEqualGreater:"⋚",LessFullEqual:"≦",LessGreater:"≶",lessgtr:"≶",LessLess:"⪡",lesssim:"≲",LessSlantEqual:"⩽",LessTilde:"≲",lfisht:"⥼",lfloor:"⌊",Lfr:"𝔏",lfr:"𝔩",lg:"≶",lgE:"⪑",lHar:"⥢",lhard:"↽",lharu:"↼",lharul:"⥪",lhblk:"▄",LJcy:"Љ",ljcy:"љ",Ll:"⋘",ll:"≪",llarr:"⇇",llcorner:"⌞",Lleftarrow:"⇚",llhard:"⥫",lltri:"◺",Lmidot:"Ŀ",lmidot:"ŀ",lmoust:"⎰",lmoustache:"⎰",lnap:"⪉",lnapprox:"⪉",lnE:"≨",lne:"⪇",lneq:"⪇",lneqq:"≨",lnsim:"⋦",loang:"⟬",loarr:"⇽",lobrk:"⟦",LongLeftArrow:"⟵",Longleftarrow:"⟸",longleftarrow:"⟵",LongLeftRightArrow:"⟷",Longleftrightarrow:"⟺",longleftrightarrow:"⟷",longmapsto:"⟼",LongRightArrow:"⟶",Longrightarrow:"⟹",longrightarrow:"⟶",looparrowleft:"↫",looparrowright:"↬",lopar:"⦅",Lopf:"𝕃",lopf:"𝕝",loplus:"⨭",lotimes:"⨴",lowast:"∗",lowbar:"_",LowerLeftArrow:"↙",LowerRightArrow:"↘",loz:"◊",lozenge:"◊",lozf:"⧫",lpar:"(",lparlt:"⦓",lrarr:"⇆",lrcorner:"⌟",lrhar:"⇋",lrhard:"⥭",lrm:"‎",lrtri:"⊿",lsaquo:"‹",Lscr:"ℒ",lscr:"𝓁",Lsh:"↰",lsh:"↰",lsim:"≲",lsime:"⪍",lsimg:"⪏",lsqb:"[",lsquo:"‘",lsquor:"‚",Lstrok:"Ł",lstrok:"ł",LT:"<",Lt:"≪",lt:"<",ltcc:"⪦",ltcir:"⩹",ltdot:"⋖",lthree:"⋋",ltimes:"⋉",ltlarr:"⥶",ltquest:"⩻",ltri:"◃",ltrie:"⊴",ltrif:"◂",ltrPar:"⦖",lurdshar:"⥊",luruhar:"⥦",lvertneqq:"≨︀",lvnE:"≨︀",macr:"¯",male:"♂",malt:"✠",maltese:"✠",Map:"⤅",map:"↦",mapsto:"↦",mapstodown:"↧",mapstoleft:"↤",mapstoup:"↥",marker:"▮",mcomma:"⨩",Mcy:"М",mcy:"м",mdash:"—",mDDot:"∺",measuredangle:"∡",MediumSpace:" ",Mellintrf:"ℳ",Mfr:"𝔐",mfr:"𝔪",mho:"℧",micro:"µ",mid:"∣",midast:"*",midcir:"⫰",middot:"·",minus:"−",minusb:"⊟",minusd:"∸",minusdu:"⨪",MinusPlus:"∓",mlcp:"⫛",mldr:"…",mnplus:"∓",models:"⊧",Mopf:"𝕄",mopf:"𝕞",mp:"∓",Mscr:"ℳ",mscr:"𝓂",mstpos:"∾",Mu:"Μ",mu:"μ",multimap:"⊸",mumap:"⊸",nabla:"∇",Nacute:"Ń",nacute:"ń",nang:"∠⃒",nap:"≉",napE:"⩰̸",napid:"≋̸",napos:"ʼn",napprox:"≉",natur:"♮",natural:"♮",naturals:"ℕ",nbsp:" ",nbump:"≎̸",nbumpe:"≏̸",ncap:"⩃",Ncaron:"Ň",ncaron:"ň",Ncedil:"Ņ",ncedil:"ņ",ncong:"≇",ncongdot:"⩭̸",ncup:"⩂",Ncy:"Н",ncy:"н",ndash:"–",ne:"≠",nearhk:"⤤",neArr:"⇗",nearr:"↗",nearrow:"↗",nedot:"≐̸",NegativeMediumSpace:"​",NegativeThickSpace:"​",NegativeThinSpace:"​",NegativeVeryThinSpace:"​",nequiv:"≢",nesear:"⤨",nesim:"≂̸",NestedGreaterGreater:"≫",NestedLessLess:"≪",NewLine:"\n",nexist:"∄",nexists:"∄",Nfr:"𝔑",nfr:"𝔫",ngE:"≧̸",nge:"≱",ngeq:"≱",ngeqq:"≧̸",ngeqslant:"⩾̸",nges:"⩾̸",nGg:"⋙̸",ngsim:"≵",nGt:"≫⃒",ngt:"≯",ngtr:"≯",nGtv:"≫̸",nhArr:"⇎",nharr:"↮",nhpar:"⫲",ni:"∋",nis:"⋼",nisd:"⋺",niv:"∋",NJcy:"Њ",njcy:"њ",nlArr:"⇍",nlarr:"↚",nldr:"‥",nlE:"≦̸",nle:"≰",nLeftarrow:"⇍",nleftarrow:"↚",nLeftrightarrow:"⇎",nleftrightarrow:"↮",nleq:"≰",nleqq:"≦̸",nleqslant:"⩽̸",nles:"⩽̸",nless:"≮",nLl:"⋘̸",nlsim:"≴",nLt:"≪⃒",nlt:"≮",nltri:"⋪",nltrie:"⋬",nLtv:"≪̸",nmid:"∤",NoBreak:"⁠",NonBreakingSpace:" ",Nopf:"ℕ",nopf:"𝕟",Not:"⫬",not:"¬",NotCongruent:"≢",NotCupCap:"≭",NotDoubleVerticalBar:"∦",NotElement:"∉",NotEqual:"≠",NotEqualTilde:"≂̸",NotExists:"∄",NotGreater:"≯",NotGreaterEqual:"≱",NotGreaterFullEqual:"≧̸",NotGreaterGreater:"≫̸",NotGreaterLess:"≹",NotGreaterSlantEqual:"⩾̸",NotGreaterTilde:"≵",NotHumpDownHump:"≎̸",NotHumpEqual:"≏̸",notin:"∉",notindot:"⋵̸",notinE:"⋹̸",notinva:"∉",notinvb:"⋷",notinvc:"⋶",NotLeftTriangle:"⋪",NotLeftTriangleBar:"⧏̸",NotLeftTriangleEqual:"⋬",NotLess:"≮",NotLessEqual:"≰",NotLessGreater:"≸",NotLessLess:"≪̸",NotLessSlantEqual:"⩽̸",NotLessTilde:"≴",NotNestedGreaterGreater:"⪢̸",NotNestedLessLess:"⪡̸",notni:"∌",notniva:"∌",notnivb:"⋾",notnivc:"⋽",NotPrecedes:"⊀",NotPrecedesEqual:"⪯̸",NotPrecedesSlantEqual:"⋠",NotReverseElement:"∌",NotRightTriangle:"⋫",NotRightTriangleBar:"⧐̸",NotRightTriangleEqual:"⋭",NotSquareSubset:"⊏̸",NotSquareSubsetEqual:"⋢",NotSquareSuperset:"⊐̸",NotSquareSupersetEqual:"⋣",NotSubset:"⊂⃒",NotSubsetEqual:"⊈",NotSucceeds:"⊁",NotSucceedsEqual:"⪰̸",NotSucceedsSlantEqual:"⋡",NotSucceedsTilde:"≿̸",NotSuperset:"⊃⃒",NotSupersetEqual:"⊉",NotTilde:"≁",NotTildeEqual:"≄",NotTildeFullEqual:"≇",NotTildeTilde:"≉",NotVerticalBar:"∤",npar:"∦",nparallel:"∦",nparsl:"⫽⃥",npart:"∂̸",npolint:"⨔",npr:"⊀",nprcue:"⋠",npre:"⪯̸",nprec:"⊀",npreceq:"⪯̸",nrArr:"⇏",nrarr:"↛",nrarrc:"⤳̸",nrarrw:"↝̸",nRightarrow:"⇏",nrightarrow:"↛",nrtri:"⋫",nrtrie:"⋭",nsc:"⊁",nsccue:"⋡",nsce:"⪰̸",Nscr:"𝒩",nscr:"𝓃",nshortmid:"∤",nshortparallel:"∦",nsim:"≁",nsime:"≄",nsimeq:"≄",nsmid:"∤",nspar:"∦",nsqsube:"⋢",nsqsupe:"⋣",nsub:"⊄",nsubE:"⫅̸",nsube:"⊈",nsubset:"⊂⃒",nsubseteq:"⊈",nsubseteqq:"⫅̸",nsucc:"⊁",nsucceq:"⪰̸",nsup:"⊅",nsupE:"⫆̸",nsupe:"⊉",nsupset:"⊃⃒",nsupseteq:"⊉",nsupseteqq:"⫆̸",ntgl:"≹",Ntilde:"Ñ",ntilde:"ñ",ntlg:"≸",ntriangleleft:"⋪",ntrianglelefteq:"⋬",ntriangleright:"⋫",ntrianglerighteq:"⋭",Nu:"Ν",nu:"ν",num:"#",numero:"№",numsp:" ",nvap:"≍⃒",nVDash:"⊯",nVdash:"⊮",nvDash:"⊭",nvdash:"⊬",nvge:"≥⃒",nvgt:">⃒",nvHarr:"⤄",nvinfin:"⧞",nvlArr:"⤂",nvle:"≤⃒",nvlt:"<⃒",nvltrie:"⊴⃒",nvrArr:"⤃",nvrtrie:"⊵⃒",nvsim:"∼⃒",nwarhk:"⤣",nwArr:"⇖",nwarr:"↖",nwarrow:"↖",nwnear:"⤧",Oacute:"Ó",oacute:"ó",oast:"⊛",ocir:"⊚",Ocirc:"Ô",ocirc:"ô",Ocy:"О",ocy:"о",odash:"⊝",Odblac:"Ő",odblac:"ő",odiv:"⨸",odot:"⊙",odsold:"⦼",OElig:"Œ",oelig:"œ",ofcir:"⦿",Ofr:"𝔒",ofr:"𝔬",ogon:"˛",Ograve:"Ò",ograve:"ò",ogt:"⧁",ohbar:"⦵",ohm:"Ω",oint:"∮",olarr:"↺",olcir:"⦾",olcross:"⦻",oline:"‾",olt:"⧀",Omacr:"Ō",omacr:"ō",Omega:"Ω",omega:"ω",Omicron:"Ο",omicron:"ο",omid:"⦶",ominus:"⊖",Oopf:"𝕆",oopf:"𝕠",opar:"⦷",OpenCurlyDoubleQuote:"“",OpenCurlyQuote:"‘",operp:"⦹",oplus:"⊕",Or:"⩔",or:"∨",orarr:"↻",ord:"⩝",order:"ℴ",orderof:"ℴ",ordf:"ª",ordm:"º",origof:"⊶",oror:"⩖",orslope:"⩗",orv:"⩛",oS:"Ⓢ",Oscr:"𝒪",oscr:"ℴ",Oslash:"Ø",oslash:"ø",osol:"⊘",Otilde:"Õ",otilde:"õ",Otimes:"⨷",otimes:"⊗",otimesas:"⨶",Ouml:"Ö",ouml:"ö",ovbar:"⌽",OverBar:"‾",OverBrace:"⏞",OverBracket:"⎴",OverParenthesis:"⏜",par:"∥",para:"¶",parallel:"∥",parsim:"⫳",parsl:"⫽",part:"∂",PartialD:"∂",Pcy:"П",pcy:"п",percnt:"%",period:".",permil:"‰",perp:"⊥",pertenk:"‱",Pfr:"𝔓",pfr:"𝔭",Phi:"Φ",phi:"φ",phiv:"ϕ",phmmat:"ℳ",phone:"☎",Pi:"Π",pi:"π",pitchfork:"⋔",piv:"ϖ",planck:"ℏ",planckh:"ℎ",plankv:"ℏ",plus:"+",plusacir:"⨣",plusb:"⊞",pluscir:"⨢",plusdo:"∔",plusdu:"⨥",pluse:"⩲",PlusMinus:"±",plusmn:"±",plussim:"⨦",plustwo:"⨧",pm:"±",Poincareplane:"ℌ",pointint:"⨕",Popf:"ℙ",popf:"𝕡",pound:"£",Pr:"⪻",pr:"≺",prap:"⪷",prcue:"≼",prE:"⪳",pre:"⪯",prec:"≺",precapprox:"⪷",preccurlyeq:"≼",Precedes:"≺",PrecedesEqual:"⪯",PrecedesSlantEqual:"≼",PrecedesTilde:"≾",preceq:"⪯",precnapprox:"⪹",precneqq:"⪵",precnsim:"⋨",precsim:"≾",Prime:"″",prime:"′",primes:"ℙ",prnap:"⪹",prnE:"⪵",prnsim:"⋨",prod:"∏",Product:"∏",profalar:"⌮",profline:"⌒",profsurf:"⌓",prop:"∝",Proportion:"∷",Proportional:"∝",propto:"∝",prsim:"≾",prurel:"⊰",Pscr:"𝒫",pscr:"𝓅",Psi:"Ψ",psi:"ψ",puncsp:" ",Qfr:"𝔔",qfr:"𝔮",qint:"⨌",Qopf:"ℚ",qopf:"𝕢",qprime:"⁗",Qscr:"𝒬",qscr:"𝓆",quaternions:"ℍ",quatint:"⨖",quest:"?",questeq:"≟",QUOT:'"',quot:'"',rAarr:"⇛",race:"∽̱",Racute:"Ŕ",racute:"ŕ",radic:"√",raemptyv:"⦳",Rang:"⟫",rang:"⟩",rangd:"⦒",range:"⦥",rangle:"⟩",raquo:"»",Rarr:"↠",rArr:"⇒",rarr:"→",rarrap:"⥵",rarrb:"⇥",rarrbfs:"⤠",rarrc:"⤳",rarrfs:"⤞",rarrhk:"↪",rarrlp:"↬",rarrpl:"⥅",rarrsim:"⥴",Rarrtl:"⤖",rarrtl:"↣",rarrw:"↝",rAtail:"⤜",ratail:"⤚",ratio:"∶",rationals:"ℚ",RBarr:"⤐",rBarr:"⤏",rbarr:"⤍",rbbrk:"❳",rbrace:"}",rbrack:"]",rbrke:"⦌",rbrksld:"⦎",rbrkslu:"⦐",Rcaron:"Ř",rcaron:"ř",Rcedil:"Ŗ",rcedil:"ŗ",rceil:"⌉",rcub:"}",Rcy:"Р",rcy:"р",rdca:"⤷",rdldhar:"⥩",rdquo:"”",rdquor:"”",rdsh:"↳",Re:"ℜ",real:"ℜ",realine:"ℛ",realpart:"ℜ",reals:"ℝ",rect:"▭",REG:"®",reg:"®",ReverseElement:"∋",ReverseEquilibrium:"⇋",ReverseUpEquilibrium:"⥯",rfisht:"⥽",rfloor:"⌋",Rfr:"ℜ",rfr:"𝔯",rHar:"⥤",rhard:"⇁",rharu:"⇀",rharul:"⥬",Rho:"Ρ",rho:"ρ",rhov:"ϱ",RightAngleBracket:"⟩",RightArrow:"→",Rightarrow:"⇒",rightarrow:"→",RightArrowBar:"⇥",RightArrowLeftArrow:"⇄",rightarrowtail:"↣",RightCeiling:"⌉",RightDoubleBracket:"⟧",RightDownTeeVector:"⥝",RightDownVector:"⇂",RightDownVectorBar:"⥕",RightFloor:"⌋",rightharpoondown:"⇁",rightharpoonup:"⇀",rightleftarrows:"⇄",rightleftharpoons:"⇌",rightrightarrows:"⇉",rightsquigarrow:"↝",RightTee:"⊢",RightTeeArrow:"↦",RightTeeVector:"⥛",rightthreetimes:"⋌",RightTriangle:"⊳",RightTriangleBar:"⧐",RightTriangleEqual:"⊵",RightUpDownVector:"⥏",RightUpTeeVector:"⥜",RightUpVector:"↾",RightUpVectorBar:"⥔",RightVector:"⇀",RightVectorBar:"⥓",ring:"˚",risingdotseq:"≓",rlarr:"⇄",rlhar:"⇌",rlm:"‏",rmoust:"⎱",rmoustache:"⎱",rnmid:"⫮",roang:"⟭",roarr:"⇾",robrk:"⟧",ropar:"⦆",Ropf:"ℝ",ropf:"𝕣",roplus:"⨮",rotimes:"⨵",RoundImplies:"⥰",rpar:")",rpargt:"⦔",rppolint:"⨒",rrarr:"⇉",Rrightarrow:"⇛",rsaquo:"›",Rscr:"ℛ",rscr:"𝓇",Rsh:"↱",rsh:"↱",rsqb:"]",rsquo:"’",rsquor:"’",rthree:"⋌",rtimes:"⋊",rtri:"▹",rtrie:"⊵",rtrif:"▸",rtriltri:"⧎",RuleDelayed:"⧴",ruluhar:"⥨",rx:"℞",Sacute:"Ś",sacute:"ś",sbquo:"‚",Sc:"⪼",sc:"≻",scap:"⪸",Scaron:"Š",scaron:"š",sccue:"≽",scE:"⪴",sce:"⪰",Scedil:"Ş",scedil:"ş",Scirc:"Ŝ",scirc:"ŝ",scnap:"⪺",scnE:"⪶",scnsim:"⋩",scpolint:"⨓",scsim:"≿",Scy:"С",scy:"с",sdot:"⋅",sdotb:"⊡",sdote:"⩦",searhk:"⤥",seArr:"⇘",searr:"↘",searrow:"↘",sect:"§",semi:";",seswar:"⤩",setminus:"∖",setmn:"∖",sext:"✶",Sfr:"𝔖",sfr:"𝔰",sfrown:"⌢",sharp:"♯",SHCHcy:"Щ",shchcy:"щ",SHcy:"Ш",shcy:"ш",ShortDownArrow:"↓",ShortLeftArrow:"←",shortmid:"∣",shortparallel:"∥",ShortRightArrow:"→",ShortUpArrow:"↑",shy:"­",Sigma:"Σ",sigma:"σ",sigmaf:"ς",sigmav:"ς",sim:"∼",simdot:"⩪",sime:"≃",simeq:"≃",simg:"⪞",simgE:"⪠",siml:"⪝",simlE:"⪟",simne:"≆",simplus:"⨤",simrarr:"⥲",slarr:"←",SmallCircle:"∘",smallsetminus:"∖",smashp:"⨳",smeparsl:"⧤",smid:"∣",smile:"⌣",smt:"⪪",smte:"⪬",smtes:"⪬︀",SOFTcy:"Ь",softcy:"ь",sol:"/",solb:"⧄",solbar:"⌿",Sopf:"𝕊",sopf:"𝕤",spades:"♠",spadesuit:"♠",spar:"∥",sqcap:"⊓",sqcaps:"⊓︀",sqcup:"⊔",sqcups:"⊔︀",Sqrt:"√",sqsub:"⊏",sqsube:"⊑",sqsubset:"⊏",sqsubseteq:"⊑",sqsup:"⊐",sqsupe:"⊒",sqsupset:"⊐",sqsupseteq:"⊒",squ:"□",Square:"□",square:"□",SquareIntersection:"⊓",SquareSubset:"⊏",SquareSubsetEqual:"⊑",SquareSuperset:"⊐",SquareSupersetEqual:"⊒",SquareUnion:"⊔",squarf:"▪",squf:"▪",srarr:"→",Sscr:"𝒮",sscr:"𝓈",ssetmn:"∖",ssmile:"⌣",sstarf:"⋆",Star:"⋆",star:"☆",starf:"★",straightepsilon:"ϵ",straightphi:"ϕ",strns:"¯",Sub:"⋐",sub:"⊂",subdot:"⪽",subE:"⫅",sube:"⊆",subedot:"⫃",submult:"⫁",subnE:"⫋",subne:"⊊",subplus:"⪿",subrarr:"⥹",Subset:"⋐",subset:"⊂",subseteq:"⊆",subseteqq:"⫅",SubsetEqual:"⊆",subsetneq:"⊊",subsetneqq:"⫋",subsim:"⫇",subsub:"⫕",subsup:"⫓",succ:"≻",succapprox:"⪸",succcurlyeq:"≽",Succeeds:"≻",SucceedsEqual:"⪰",SucceedsSlantEqual:"≽",SucceedsTilde:"≿",succeq:"⪰",succnapprox:"⪺",succneqq:"⪶",succnsim:"⋩",succsim:"≿",SuchThat:"∋",Sum:"∑",sum:"∑",sung:"♪",Sup:"⋑",sup:"⊃",sup1:"¹",sup2:"²",sup3:"³",supdot:"⪾",supdsub:"⫘",supE:"⫆",supe:"⊇",supedot:"⫄",Superset:"⊃",SupersetEqual:"⊇",suphsol:"⟉",suphsub:"⫗",suplarr:"⥻",supmult:"⫂",supnE:"⫌",supne:"⊋",supplus:"⫀",Supset:"⋑",supset:"⊃",supseteq:"⊇",supseteqq:"⫆",supsetneq:"⊋",supsetneqq:"⫌",supsim:"⫈",supsub:"⫔",supsup:"⫖",swarhk:"⤦",swArr:"⇙",swarr:"↙",swarrow:"↙",swnwar:"⤪",szlig:"ß",Tab:"\t",target:"⌖",Tau:"Τ",tau:"τ",tbrk:"⎴",Tcaron:"Ť",tcaron:"ť",Tcedil:"Ţ",tcedil:"ţ",Tcy:"Т",tcy:"т",tdot:"⃛",telrec:"⌕",Tfr:"𝔗",tfr:"𝔱",there4:"∴",Therefore:"∴",therefore:"∴",Theta:"Θ",theta:"θ",thetasym:"ϑ",thetav:"ϑ",thickapprox:"≈",thicksim:"∼",ThickSpace:"  ",thinsp:" ",ThinSpace:" ",thkap:"≈",thksim:"∼",THORN:"Þ",thorn:"þ",Tilde:"∼",tilde:"˜",TildeEqual:"≃",TildeFullEqual:"≅",TildeTilde:"≈",times:"×",timesb:"⊠",timesbar:"⨱",timesd:"⨰",tint:"∭",toea:"⤨",top:"⊤",topbot:"⌶",topcir:"⫱",Topf:"𝕋",topf:"𝕥",topfork:"⫚",tosa:"⤩",tprime:"‴",TRADE:"™",trade:"™",triangle:"▵",triangledown:"▿",triangleleft:"◃",trianglelefteq:"⊴",triangleq:"≜",triangleright:"▹",trianglerighteq:"⊵",tridot:"◬",trie:"≜",triminus:"⨺",TripleDot:"⃛",triplus:"⨹",trisb:"⧍",tritime:"⨻",trpezium:"⏢",Tscr:"𝒯",tscr:"𝓉",TScy:"Ц",tscy:"ц",TSHcy:"Ћ",tshcy:"ћ",Tstrok:"Ŧ",tstrok:"ŧ",twixt:"≬",twoheadleftarrow:"↞",twoheadrightarrow:"↠",Uacute:"Ú",uacute:"ú",Uarr:"↟",uArr:"⇑",uarr:"↑",Uarrocir:"⥉",Ubrcy:"Ў",ubrcy:"ў",Ubreve:"Ŭ",ubreve:"ŭ",Ucirc:"Û",ucirc:"û",Ucy:"У",ucy:"у",udarr:"⇅",Udblac:"Ű",udblac:"ű",udhar:"⥮",ufisht:"⥾",Ufr:"𝔘",ufr:"𝔲",Ugrave:"Ù",ugrave:"ù",uHar:"⥣",uharl:"↿",uharr:"↾",uhblk:"▀",ulcorn:"⌜",ulcorner:"⌜",ulcrop:"⌏",ultri:"◸",Umacr:"Ū",umacr:"ū",uml:"¨",UnderBar:"_",UnderBrace:"⏟",UnderBracket:"⎵",UnderParenthesis:"⏝",Union:"⋃",UnionPlus:"⊎",Uogon:"Ų",uogon:"ų",Uopf:"𝕌",uopf:"𝕦",UpArrow:"↑",Uparrow:"⇑",uparrow:"↑",UpArrowBar:"⤒",UpArrowDownArrow:"⇅",UpDownArrow:"↕",Updownarrow:"⇕",updownarrow:"↕",UpEquilibrium:"⥮",upharpoonleft:"↿",upharpoonright:"↾",uplus:"⊎",UpperLeftArrow:"↖",UpperRightArrow:"↗",Upsi:"ϒ",upsi:"υ",upsih:"ϒ",Upsilon:"Υ",upsilon:"υ",UpTee:"⊥",UpTeeArrow:"↥",upuparrows:"⇈",urcorn:"⌝",urcorner:"⌝",urcrop:"⌎",Uring:"Ů",uring:"ů",urtri:"◹",Uscr:"𝒰",uscr:"𝓊",utdot:"⋰",Utilde:"Ũ",utilde:"ũ",utri:"▵",utrif:"▴",uuarr:"⇈",Uuml:"Ü",uuml:"ü",uwangle:"⦧",vangrt:"⦜",varepsilon:"ϵ",varkappa:"ϰ",varnothing:"∅",varphi:"ϕ",varpi:"ϖ",varpropto:"∝",vArr:"⇕",varr:"↕",varrho:"ϱ",varsigma:"ς",varsubsetneq:"⊊︀",varsubsetneqq:"⫋︀",varsupsetneq:"⊋︀",varsupsetneqq:"⫌︀",vartheta:"ϑ",vartriangleleft:"⊲",vartriangleright:"⊳",Vbar:"⫫",vBar:"⫨",vBarv:"⫩",Vcy:"В",vcy:"в",VDash:"⊫",Vdash:"⊩",vDash:"⊨",vdash:"⊢",Vdashl:"⫦",Vee:"⋁",vee:"∨",veebar:"⊻",veeeq:"≚",vellip:"⋮",Verbar:"‖",verbar:"|",Vert:"‖",vert:"|",VerticalBar:"∣",VerticalLine:"|",VerticalSeparator:"❘",VerticalTilde:"≀",VeryThinSpace:" ",Vfr:"𝔙",vfr:"𝔳",vltri:"⊲",vnsub:"⊂⃒",vnsup:"⊃⃒",Vopf:"𝕍",vopf:"𝕧",vprop:"∝",vrtri:"⊳",Vscr:"𝒱",vscr:"𝓋",vsubnE:"⫋︀",vsubne:"⊊︀",vsupnE:"⫌︀",vsupne:"⊋︀",Vvdash:"⊪",vzigzag:"⦚",Wcirc:"Ŵ",wcirc:"ŵ",wedbar:"⩟",Wedge:"⋀",wedge:"∧",wedgeq:"≙",weierp:"℘",Wfr:"𝔚",wfr:"𝔴",Wopf:"𝕎",wopf:"𝕨",wp:"℘",wr:"≀",wreath:"≀",Wscr:"𝒲",wscr:"𝓌",xcap:"⋂",xcirc:"◯",xcup:"⋃",xdtri:"▽",Xfr:"𝔛",xfr:"𝔵",xhArr:"⟺",xharr:"⟷",Xi:"Ξ",xi:"ξ",xlArr:"⟸",xlarr:"⟵",xmap:"⟼",xnis:"⋻",xodot:"⨀",Xopf:"𝕏",xopf:"𝕩",xoplus:"⨁",xotime:"⨂",xrArr:"⟹",xrarr:"⟶",Xscr:"𝒳",xscr:"𝓍",xsqcup:"⨆",xuplus:"⨄",xutri:"△",xvee:"⋁",xwedge:"⋀",Yacute:"Ý",yacute:"ý",YAcy:"Я",yacy:"я",Ycirc:"Ŷ",ycirc:"ŷ",Ycy:"Ы",ycy:"ы",yen:"¥",Yfr:"𝔜",yfr:"𝔶",YIcy:"Ї",yicy:"ї",Yopf:"𝕐",yopf:"𝕪",Yscr:"𝒴",yscr:"𝓎",YUcy:"Ю",yucy:"ю",Yuml:"Ÿ",yuml:"ÿ",Zacute:"Ź",zacute:"ź",Zcaron:"Ž",zcaron:"ž",Zcy:"З",zcy:"з",Zdot:"Ż",zdot:"ż",zeetrf:"ℨ",ZeroWidthSpace:"​",Zeta:"Ζ",zeta:"ζ",Zfr:"ℨ",zfr:"𝔷",ZHcy:"Ж",zhcy:"ж",zigrarr:"⇝",Zopf:"ℤ",zopf:"𝕫",Zscr:"𝒵",zscr:"𝓏",zwj:"‍",zwnj:"‌"}},function(e,t,n){"use strict";var r=n(419),o=n(27).unescapeMd;e.exports=function(e,t){var n,i,a,u=t,s=e.posMax;if(60===e.src.charCodeAt(t)){for(t++;t8&&n<14);)if(92===n&&t+11)break;if(41===n&&--i<0)break;t++}return u!==t&&(a=o(e.src.slice(u,t)),!!e.parser.validateLink(a)&&(e.linkContent=a,e.pos=t,!0))}},function(e,t,n){"use strict";var r=n(27).replaceEntities;e.exports=function(e){var t=r(e);try{t=decodeURI(t)}catch(e){}return encodeURI(t)}},function(e,t,n){"use strict";var r=n(27).unescapeMd;e.exports=function(e,t){var n,o=t,i=e.posMax,a=e.src.charCodeAt(t);if(34!==a&&39!==a&&40!==a)return!1;for(t++,40===a&&(a=41);t1?r-1:0),i=1;i1?t-1:0),r=1;r0){var S=a("JsonSchemaForm"),C=a("ParameterExt"),k=w.get("properties",(0,o.OrderedMap)());return n=o.Map.isMap(n)?n:(0,o.OrderedMap)(),r.default.createElement("div",{className:"table-container"},y&&r.default.createElement(h,{source:y}),r.default.createElement("table",null,r.default.createElement("tbody",null,k.map(function(e,t){var u=g?(0,i.getCommonExtensions)(e):null,s=w.get("required",(0,o.List)()).includes(t),c=e.get("type"),p=e.get("format"),v=e.get("description"),m=n.get(t),y=e.get("default")||e.get("example")||"";""===y&&"object"===c&&(y=(0,i.getSampleSchema)(e,!1,{includeWriteOnly:!0})),"string"!=typeof y&&"object"===c&&(y=(0,i.stringify)(y));var b="string"===c&&("binary"===p||"base64"===p);return r.default.createElement("tr",{key:t,className:"parameters","data-property-name":t},r.default.createElement("td",{className:"col parameters-col_name"},r.default.createElement("div",{className:s?"parameter__name required":"parameter__name"},t,s?r.default.createElement("span",{style:{color:"red"}}," *"):null),r.default.createElement("div",{className:"parameter__type"},c,p&&r.default.createElement("span",{className:"prop-format"},"($",p,")"),g&&u.size?u.map(function(e,t){return r.default.createElement(C,{key:t+"-"+e,xKey:t,xVal:e})}):null),r.default.createElement("div",{className:"parameter__deprecated"},e.get("deprecated")?"deprecated":null)),r.default.createElement("td",{className:"col parameters-col_description"},r.default.createElement(h,{source:v}),f?r.default.createElement("div",null,r.default.createElement(S,{fn:l,dispatchInitialValue:!b,schema:e,description:t,getComponent:a,value:void 0===m?y:m,onChange:function(e){d(e,[t])}})):null))}))))}return r.default.createElement("div",null,y&&r.default.createElement(h,{source:y}),r.default.createElement(v,{getComponent:a,getConfigs:u,specSelectors:s,expandDepth:1,isExecute:f,schema:_.get("schema"),specPath:p.push("content",c),example:r.default.createElement(m,{requestBody:t,onChange:d,mediaType:c,getComponent:a,isExecute:f,specSelectors:s})}))}},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=f(n(41)),o=f(n(4)),i=f(n(2)),a=f(n(3)),u=f(n(5)),s=f(n(6)),l=n(0),c=f(l);f(n(1)),f(n(12));function f(e){return e&&e.__esModule?e:{default:e}}var p=function(e){function t(){return(0,i.default)(this,t),(0,u.default)(this,(t.__proto__||(0,o.default)(t)).apply(this,arguments))}return(0,s.default)(t,e),(0,a.default)(t,[{key:"render",value:function(){var e=this.props,t=e.link,n=e.name,o=(0,e.getComponent)("Markdown"),i=t.get("operationId")||t.get("operationRef"),a=t.get("parameters")&&t.get("parameters").toJS(),u=t.get("description");return c.default.createElement("div",{style:{marginBottom:"1.5em"}},c.default.createElement("div",{style:{marginBottom:".5em"}},c.default.createElement("b",null,c.default.createElement("code",null,n)),u?c.default.createElement(o,{source:u}):null),c.default.createElement("pre",null,"Operation `",i,"`",c.default.createElement("br",null),c.default.createElement("br",null),"Parameters ",function(e,t){if("string"!=typeof t)return"";return t.split("\n").map(function(t,n){return n>0?Array(e+1).join(" ")+t:t}).join("\n")}(0,(0,r.default)(a,null,2))||"{}",c.default.createElement("br",null)))}}]),t}(l.Component);t.default=p},function(e,t,n){"use strict";Object.defineProperty(t,"__esModule",{value:!0});var r=c(n(4)),o=c(n(2)),i=c(n(3)),a=c(n(5)),u=c(n(6)),s=c(n(0)),l=n(7);c(n(1)),c(n(12));function c(e){return e&&e.__esModule?e:{default:e}}var f=function(e){function t(){var e,n,i,u;(0,o.default)(this,t);for(var s=arguments.length,l=Array(s),c=0;c=e.length?(this._t=void 0,o(1)):o(0,"keys"==t?n:"values"==t?e[n]:[n,e[n]])},"values"),i.Arguments=i.Array,r("keys"),r("values"),r("entries")},function(e,t){e.exports=function(){}},function(e,t){e.exports=function(e,t){return{value:t,done:!!e}}},function(e,t,n){"use strict";var r=n(160),o=n(95),i=n(97),a={};n(50)(a,n(19)("iterator"),function(){return this}),e.exports=function(e,t,n){e.prototype=r(a,{next:o(1,n)}),i(e,t+" Iterator")}},function(e,t,n){var r=n(40),o=n(36),i=n(96);e.exports=n(44)?Object.defineProperties:function(e,t){o(e);for(var n,a=i(t),u=a.length,s=0;u>s;)r.f(e,n=a[s++],t[n]);return e}},function(e,t,n){var r=n(71),o=n(115),i=n(455);e.exports=function(e){return function(t,n,a){var u,s=r(t),l=o(s.length),c=i(a,l);if(e&&n!=n){for(;l>c;)if((u=s[c++])!=u)return!0}else for(;l>c;c++)if((e||c in s)&&s[c]===n)return e||c||0;return!e&&-1}}},function(e,t,n){var r=n(161),o=Math.max,i=Math.min;e.exports=function(e,t){return(e=r(e))<0?o(e+t,0):i(e,t)}},function(e,t,n){var r=n(161),o=n(156);e.exports=function(e){return function(t,n){var i,a,u=String(o(t)),s=r(n),l=u.length;return s<0||s>=l?e?"":void 0:(i=u.charCodeAt(s))<55296||i>56319||s+1===l||(a=u.charCodeAt(s+1))<56320||a>57343?e?u.charAt(s):i:e?u.slice(s,s+2):a-56320+(i-55296<<10)+65536}}},function(e,t,n){var r=n(36),o=n(165);e.exports=n(15).getIterator=function(e){var t=o(e);if("function"!=typeof t)throw TypeError(e+" is not iterable!");return r(t.call(e))}},function(e,t,n){n(459),n(245),n(470),n(474),n(485),n(486),e.exports=n(61).Promise},function(e,t,n){"use strict";var r=n(167),o={};o[n(18)("toStringTag")]="z",o+""!="[object z]"&&n(73)(Object.prototype,"toString",function(){return"[object "+r(this)+"]"},!0)},function(e,t,n){e.exports=!n(100)&&!n(101)(function(){return 7!=Object.defineProperty(n(169)("div"),"a",{get:function(){return 7}}).a})},function(e,t,n){var r=n(74);e.exports=function(e,t){if(!r(e))return e;var n,o;if(t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;if("function"==typeof(n=e.valueOf)&&!r(o=n.call(e)))return o;if(!t&&"function"==typeof(n=e.toString)&&!r(o=n.call(e)))return o;throw TypeError("Can't convert object to primitive value")}},function(e,t,n){"use strict";var r=n(463),o=n(244),i=n(171),a={};n(59)(a,n(18)("iterator"),function(){return this}),e.exports=function(e,t,n){e.prototype=r(a,{next:o(1,n)}),i(e,t+" Iterator")}},function(e,t,n){var r=n(60),o=n(464),i=n(251),a=n(170)("IE_PROTO"),u=function(){},s=function(){var e,t=n(169)("iframe"),r=i.length;for(t.style.display="none",n(252).appendChild(t),t.src="javascript:",(e=t.contentWindow.document).open(),e.write("