Clickhouse mode of study view

.circleci/config.yml

@@ -223,6 +223,135 @@ jobs:
             DOCKER_IMAGE_SESSION_SERVICE: cbioportal/session-service:0.5.0
             FRONTEND_TEST_DO_NOT_LOAD_EXTERNAL_FRONTEND: true
 
+    pull_cbioportal_test_codebase:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - run:
+            name: Checkout cbioportal/cbioportal-test
+            environment:
+              TEST_REPO_URL: https://github.com/cBioPortal/cbioportal-test
+            command: |
+              git clone ${TEST_REPO_URL}
+        - persist_to_workspace:
+            root: /tmp/repos
+            paths:
+              - cbioportal-test
+
+    pull_cbioportal_frontend_codebase:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - run:
+            name: Checkout cbioportal/cbioportal-frontend
+            environment:
+              FRONTEND_REPO_URL: https://github.com/alisman/cbioportal-frontend.git
+              FRONTEND_REPO_BRANCH: clickhouse
+            command: |
+              git clone -b ${FRONTEND_REPO_BRANCH} --single-branch ${FRONTEND_REPO_URL}
+        - persist_to_workspace:
+            root: /tmp/repos
+            paths:
+              - cbioportal-frontend
+
+    build_push_image:
+      machine:
+        image: ubuntu-2204:2024.08.1
+      resource_class: medium
+      working_directory: /tmp/repos
+      steps:
+        - attach_workspace:
+            at: /tmp/repos
+        - checkout:
+            path: /tmp/repos/cbioportal
+        - run:
+            name: Build cBioPortal docker image
+            environment:
+              DOCKER_REPO: cbioportal/cbioportal-dev
+            command: |
+              cd cbioportal-test
+              export DOCKER_TAG=$CIRCLE_SHA1
+              ./scripts/build-push-image.sh --src=/tmp/repos/cbioportal --push=true
+              EXISTS=$(docker manifest inspect $DOCKER_REPO:$DOCKER_TAG-web-shenandoah > /dev/null; echo $?)
+              if [ $EXISTS -eq 0 ]; then
+                echo "Build succeeded!"
+              else
+                echo "Build failed!"
+                exit 1
+              fi
+
+    run_api_tests:
+      machine:
+        image: ubuntu-2204:2024.08.1
+        docker_layer_caching: true
+      resource_class: large
+      working_directory: /tmp/repos
+      steps:
+        - attach_workspace:
+            at: /tmp/repos
+        - checkout:
+            path: /tmp/repos/cbioportal
+        - run:
+            name: Instantiate a cbioportal instance
+            environment:
+              DOCKER_REPO: cbioportal/cbioportal-dev
+            command: |
+              cd cbioportal-test
+              export DOCKER_IMAGE_CBIOPORTAL=$DOCKER_REPO:$CIRCLE_SHA1-web-shenandoah
+              nohup ./scripts/docker-compose.sh >> /tmp/repos/docker-compose-logs.txt 2>&1 &
+        - run:
+            name: Wait for cbioportal to be live at localhost
+            command: |
+              for i in {1..20}; do
+                if curl -s http://localhost:8080 > /dev/null; then
+                  echo "cbioportal is live at localhost:8080!"
+                  exit 0
+                fi
+                echo "Waiting for cbioportal at localhost:8080 ..."
+                sleep 5
+              done
+              echo "Failed to connect to cbioportal at localhost:8080!"
+              exit 1
+        - run:
+            name: Confirm cbioportal config matches PR
+            command: |
+              cd cbioportal
+              echo "Matching gitCommitId..."
+              INSTANCE_COMMIT_ID=$(curl -s http://localhost:8080/api/info | jq -r '.["gitCommitId"]')
+              PR_COMMIT_ID=$CIRCLE_SHA1
+              if [ "$INSTANCE_COMMIT_ID" = "$PR_COMMIT_ID" ]; then
+                echo "gitCommitId successfully matched!"
+                echo "cBioPortal is ready:"
+                curl -s http://localhost:8080/api/info | jq
+                exit 0
+              else
+                echo "gitCommitIds do not match!"
+                echo "Instance Commit ID: $INSTANCE_COMMIT_ID"
+                echo "PR Commit ID: $PR_COMMIT_ID"
+                exit 1
+              fi
+        - run:
+            name: Run API Tests
+            environment:
+              API_TEST_HOST: http://localhost:8080
+              BACKEND_ROOT: /tmp/repos/cbioportal
+            command: |
+              cd cbioportal-frontend
+              nvm install 15.2.1
+              nvm use 15.2.1
+              npm install -g [email protected]
+              yarn --ignore-engines
+              yarn run apitests
+
+        - store_artifacts:
+            path: /tmp/repos/cbioportal-test/web-metadata.json
+        - store_artifacts:
+            path: /tmp/repos/docker-compose-logs.txt
+
 
 workflows:
     version: 2
@@ -238,3 +367,26 @@ workflows:
                     - build_backend
                     - pull_frontend_codebase
                     - install_yarn
+    api_tests:
+      jobs:
+        - pull_cbioportal_test_codebase
+        - pull_cbioportal_frontend_codebase
+        - wait_for_approval:
+            type: approval
+            requires:
+              - pull_cbioportal_test_codebase
+              - pull_cbioportal_frontend_codebase
+        - build_push_image:
+            context:
+              - api-tests
+            requires:
+              - wait_for_approval
+              - pull_cbioportal_test_codebase
+        - run_api_tests:
+            context:
+              - api-tests
+            requires:
+              - wait_for_approval
+              - pull_cbioportal_test_codebase
+              - pull_cbioportal_frontend_codebase
+              - build_push_image

.github/workflows/integration-test.yml

@@ -42,9 +42,9 @@ jobs:
         run: |
           cd ./data && ./init.sh && rm -rf ./studies/* && cd ../config && \
           cat $PORTAL_SOURCE_DIR/src/main/resources/application.properties | \
-              sed 's|spring.datasource.url=.*|spring.datasource.url=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false|' | \
-              sed 's|spring.datasource.username=.*|spring.datasource.username=cbio_user|' | \
-              sed 's|spring.datasource.password=.*|spring.datasource.password=somepassword|' \
+              sed 's|spring.datasource.mysql.url=.*|spring.datasource.mysql.url=jdbc:mysql://cbioportal-database:3306/cbioportal?useSSL=false|' | \
+              sed 's|spring.datasource.mysql.username=.*|spring.datasource.mysql.username=cbio_user|' | \
+              sed 's|spring.datasource.mysql.password=.*|spring.datasource.mysql.password=somepassword|' \
               > application.properties
       - name: 'Copy cgds.sql file into Docker Compose'
         run: cp ./cbioportal/src/main/resources/db-scripts/cgds.sql ./cbioportal-docker-compose/data/.

.github/workflows/sonarcloud.yml

@@ -34,7 +34,7 @@ jobs:
           cp src/main/resources/application.properties.EXAMPLE src/main/resources/application.properties
       - name: Build and analyze
         env:
-          SONAR_TOKEN:  ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN:  de1b5cc660cd210dde840f492c371da6cc801763
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn clean install verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
       - name: 'Add host.testcontainers.internal to /etc/hosts'
@@ -45,6 +45,6 @@ jobs:
           mvn verify -Pintegration-test
       - name: Code Coverage
         env:
-          SONAR_TOKEN:  ${{ secrets.SONAR_TOKEN }}
+          SONAR_TOKEN:  de1b5cc660cd210dde840f492c371da6cc801763
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: mvn sonar:sonar -Pcoverage

pom.xml

@@ -99,6 +99,7 @@
         <selenium.version>4.17.0</selenium.version>
 		<sentry.version>7.1.0</sentry.version>
         <apache_httpclient.version>5.2.1</apache_httpclient.version>
+        <clickhouse_testcontainer.version>1.19.7</clickhouse_testcontainer.version>
 
 
 		<!-- No sure what these are for -->
@@ -354,6 +355,31 @@
             <scope>test</scope>
         </dependency>
 
+        <dependency>
+            <groupId>com.clickhouse</groupId>
+            <artifactId>clickhouse-jdbc</artifactId>
+            <version>0.6.2</version>
+            <!-- use uber jar with all dependencies included, change classifier
+            to http for smaller jar -->
+            <classifier>all</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>clickhouse</artifactId>
+            <version>${clickhouse_testcontainer.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mybatis.spring.boot</groupId>
+            <artifactId>mybatis-spring-boot-test-autoconfigure</artifactId>
+            <version>3.0.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-testcontainers</artifactId>
+            <scope>test</scope>
+        </dependency>
     </dependencies>
 
     <dependencyManagement>
@@ -418,9 +444,25 @@
 				<includes>
 					<include>cgds.sql</include>
 					<include>migration.sql</include>
+                    <include>clickhouse/clickhouse.sql</include>
+                    <include>clickhouse/materialized_views.sql</include>
 				</includes>
 			</resource>
 		</resources>
+        <testResources>
+            <testResource>
+                <directory>src/main/resources/db-scripts</directory>
+                <filtering>false</filtering>
+                <includes>
+                    <include>clickhouse/clickhouse.sql</include>
+                    <include>clickhouse/materialized_views.sql</include>
+                </includes> 
+            </testResource>
+            <testResource>
+                <directory>src/test/resources</directory>
+                <filtering>false</filtering>
+            </testResource>
+        </testResources>
 
 		<plugins>
 			<plugin>

src/main/java/org/cbioportal/WebAppConfig.java

@@ -2,6 +2,7 @@
 
 import java.util.List;
 
+import org.cbioportal.web.ExecuterTimeInterceptor;
 import org.cbioportal.web.util.InvolvedCancerStudyExtractorInterceptor;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.context.annotation.Bean;
@@ -12,6 +13,7 @@
 import org.springframework.web.servlet.config.annotation.ResourceHandlerRegistry;
 import org.springframework.web.servlet.config.annotation.ViewControllerRegistry;
 import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
+import org.springframework.web.servlet.handler.WebRequestHandlerInterceptorAdapter;
 
 // TODO Consider creating separate DispatcherServlets as in the original web.xml
 // See: https://stackoverflow.com/a/30686733/11651683
@@ -75,6 +77,9 @@ public HandlerInterceptor involvedCancerStudyExtractorInterceptor() {
 	@Override
 	public void addInterceptors(InterceptorRegistry registry) {
 		registry.addInterceptor(involvedCancerStudyExtractorInterceptor());
+        registry.addInterceptor(new WebRequestHandlerInterceptorAdapter(
+            new ExecuterTimeInterceptor()
+        )).addPathPatterns("/**");
 	}
 
 	@Override

src/main/java/org/cbioportal/model/AlterationType.java

@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
 public enum AlterationType {
-    MUTATION,
-    COPY_NUMBER_ALTERATION
+    MUTATION_EXTENDED,
+    COPY_NUMBER_ALTERATION,
+    STRUCTURAL_VARIANT,
+    GENERIC_ASSAY;
 }

src/main/java/org/cbioportal/model/ClinicalViolinPlotBoxData.java

@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class ClinicalViolinPlotBoxData {
+import java.io.Serializable;
+
+public class ClinicalViolinPlotBoxData implements Serializable {
     private double whiskerLower;
     private double whiskerUpper;
     private double median;

src/main/java/org/cbioportal/model/ClinicalViolinPlotData.java

@@ -1,8 +1,9 @@
 package org.cbioportal.model;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ClinicalViolinPlotData {
+public class ClinicalViolinPlotData implements Serializable {
     private List<ClinicalViolinPlotRowData> rows;
     private Double axisStart;
     private Double axisEnd;

src/main/java/org/cbioportal/model/ClinicalViolinPlotIndividualPoint.java

@@ -1,6 +1,8 @@
 package org.cbioportal.model;
 
-public class ClinicalViolinPlotIndividualPoint {
+import java.io.Serializable;
+
+public class ClinicalViolinPlotIndividualPoint implements Serializable {
     private String sampleId;
     private String studyId;
     private double value;

src/main/java/org/cbioportal/model/ClinicalViolinPlotRowData.java

@@ -1,8 +1,9 @@
 package org.cbioportal.model;
 
+import java.io.Serializable;
 import java.util.List;
 
-public class ClinicalViolinPlotRowData {
+public class ClinicalViolinPlotRowData implements Serializable {
     private String category;
     private Integer numSamples;
     private List<Double> curveData;

src/main/java/org/cbioportal/model/GenomicDataCount.java

@@ -10,6 +10,21 @@ public class GenomicDataCount implements Serializable {
     private Integer count;
     private Integer uniqueCount;
 
+    public GenomicDataCount() {}
+
+    public GenomicDataCount(String label, String value, Integer count) {
+        this.label = label;
+        this.value = value;
+        this.count = count;
+    }
+
+    public GenomicDataCount(String label, String value, Integer count, Integer uniqueCount) {
+        this.label = label;
+        this.value = value;
+        this.count = count;
+        this.uniqueCount = uniqueCount;
+    }
+
     public String getLabel() {
         return label;
     }

src/main/java/org/cbioportal/model/GenomicDataCountItem.java

@@ -9,6 +9,14 @@ public class GenomicDataCountItem implements Serializable {
     private String profileType;
     private List<GenomicDataCount> counts;
 
+    public GenomicDataCountItem() {}
+
+    public GenomicDataCountItem(String hugoGeneSymbol, String profileType, List<GenomicDataCount> counts) {
+        this.hugoGeneSymbol = hugoGeneSymbol;
+        this.profileType = profileType;
+        this.counts = counts;
+    }
+
     public String getHugoGeneSymbol() {
         return hugoGeneSymbol;
     }

src/main/java/org/cbioportal/model/PatientTreatment.java

@@ -0,0 +1,7 @@
+package org.cbioportal.model;
+
+import java.io.Serializable;
+
+public record PatientTreatment (String treatment, int count) implements Serializable {
+
+}

src/main/java/org/cbioportal/model/PatientTreatmentReport.java

@@ -0,0 +1,11 @@
+package org.cbioportal.model;
+
+import java.io.Serializable;
+import java.util.Collections;
+import java.util.List;
+
+public record PatientTreatmentReport (int totalPatients, int totalSamples, List<PatientTreatment> patientTreatments) implements Serializable {
+    public PatientTreatmentReport(int totalPatients, int totalSamples) {
+        this(totalPatients, totalSamples, Collections.emptyList());
+    } 
+}

src/main/java/org/cbioportal/model/SampleTreatment.java

@@ -0,0 +1,4 @@
+package org.cbioportal.model;
+
+public record SampleTreatment(String treatment, int preSampleCount, int postSampleCount) {
+}