cloudbuild.yaml

# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.


# This is a config file used by Cloud Build and the Cloud Build local runner to 
# run a CI pipeline to check if any changes done pass tests.


steps:

- name: 'docker/compose'
  id: 'Launch HAPI Source and Sink Servers'
  args: ['-f', './docker/hapi-compose.yml', '-f', './docker/sink-compose.yml', 'up', '--force-recreate', '--remove-orphan', '-d']

# Note license checking will fail on Cloud Build because of .git dependencies
# and there is not much point updating license headers at this stage anyway.
# Ditto for Spotless (fails because of NPM dep).
- name: 'maven:3.8.5-openjdk-17'
  id: 'Compile Bunsen and Pipeline'
  entrypoint: /bin/bash
  args:
  - -c
  - mvn --no-transfer-progress -e install -Dlicense.skip=true -Dspotless.apply.skip=true
  waitFor: ['-']

# - name: 'gcr.io/cloud-builders/curl'
#   id: 'Upload coverage reports to codecov.io'
#   entrypoint: bash
#   args: ['-c', 'bash <(curl -s https://codecov.io/bash)']
#   env:
#     - 'VCS_COMMIT_ID=$COMMIT_SHA'
#     - 'VCS_BRANCH_NAME=$BRANCH_NAME'
#     - 'VCS_PULL_REQUEST=$_PR_NUMBER'
#     - 'CI_BUILD_ID=$BUILD_ID'
#     - 'CODECOV_TOKEN=$_CODECOV_TOKEN' # This token is generated when you setup your repo on codecov: https://docs.codecov.com/docs

- name: 'gcr.io/cloud-builders/docker'
  id: 'Build Uploader Image'
  entrypoint: /bin/bash
  args:
  - -c
  - cd synthea-hiv/uploader; docker build -t ${_REPOSITORY}/synthea-uploader:${_TAG} .

- name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
  id: 'Run Uploader Unit Tests'
  entrypoint: /bin/bash
  args:
  - -c
  - cd /uploader; python -m unittest discover  -p '*_test.py'

- name: 'gcr.io/cloud-builders/docker'
  id: 'Build E2E Image'
  entrypoint: /bin/bash
  args:
  - -c
  - cd e2e-tests;
    docker build -t ${_REPOSITORY}/e2e-tests:${_TAG} .;
    docker build -t ${_REPOSITORY}/e2e-tests/controller-spark:${_TAG} ./controller-spark/.;

- name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
  id: 'Upload to HAPI'
  env:
    - INPUT_DIR=/workspace/synthea-hiv/sample_data
    - SINK_TYPE=HAPI
    - FHIR_ENDPOINT=http://hapi-server:8080/fhir
    - CORES=--cores 8

- name: 'gcr.io/cloud-builders/docker'
  id: 'Build Pipeline Images'
  entrypoint: /bin/bash
  args:
  - -c
  - cd pipelines/batch;
    docker build -t ${_REPOSITORY}/batch-pipeline:${_TAG} .;
    cd ../streaming-binlog;
    docker build -t ${_REPOSITORY}/streaming-pipeline:${_TAG} .;

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline in FHIR-search mode with HAPI source'
  env:
  - FHIR_SERVER_URL=http://hapi-server:8080/fhir
  - PARQUET_PATH=/workspace/e2e-tests/FHIR_SEARCH_HAPI
  - SINK_PATH=http://sink-server:8080/fhir
  - SINK_USERNAME=hapi
  - SINK_PASSWORD=hapi

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for FHIR-search mode with HAPI source'
  env:
  - PARQUET_SUBDIR=FHIR_SEARCH_HAPI
  - DOCKER_NETWORK=--use_docker_network

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline for JDBC mode with HAPI source'
  env:
    - JDBC_MODE_ENABLED=true
    - JDBC_MODE_HAPI=true
    - FHIR_SERVER_URL=http://hapi-server:8080/fhir
    - SINK_PATH=http://sink-server:8080/fhir
    - SINK_USERNAME=hapi
    - SINK_PASSWORD=hapi
    - FHIR_DATABASE_CONFIG_PATH=/workspace/utils/hapi-postgres-config.json
    - PARQUET_PATH=/workspace/e2e-tests/JDBC_HAPI
    - JDBC_FETCH_SIZE=1000

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for JDBC mode with HAPI source'
  env:
    - PARQUET_SUBDIR=JDBC_HAPI
    - DOCKER_NETWORK=--use_docker_network

- name: 'docker/compose'
  id: 'Bring up controller and Spark containers'
  env:
    - PIPELINE_CONFIG=/workspace/docker/config
    - DWH_ROOT=/workspace/e2e-tests/controller-spark/dwh
  args: [ '-f', './docker/compose-controller-spark-sql-single.yaml', 'up',
          '--force-recreate', '-d' ]

- name: '${_REPOSITORY}/e2e-tests/controller-spark:${_TAG}'
  id: 'Run E2E Test for Dockerized Controller and Spark Thriftserver'

- name: 'docker/compose'
  id: 'Bring down controller and Spark containers'
  args: [ '-f', './docker/compose-controller-spark-sql-single.yaml', 'down' ]

- name: 'docker/compose'
  id: 'Turn down HAPI Source and Sink Servers'
  args: [ '-f', './docker/hapi-compose.yml', '-f', './docker/sink-compose.yml', 'down' ]

- name: 'docker/compose'
  id: 'Launch OpenMRS Server and HAPI FHIR Sink Server'
  args: [ '-f', './docker/openmrs-compose.yaml', '-f', './docker/sink-compose.yml', 'up',
          '--force-recreate', '--remove-orphan', '-d' ]

- name: 'gcr.io/cloud-builders/docker'
  id: 'Wait for Servers Start'
  entrypoint: /bin/bash
  args:
  - -c
  - e2e-tests/wait_for_start.sh --use_docker_network

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Launch Streaming Pipeline'
  entrypoint: /bin/bash
  args:
    - -c
    -  e2e-tests/wait_for_streaming.sh --use_docker_network

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for STREAMING, using OpenMRS Source'
  env:
    - PARQUET_SUBDIR=STREAMING
    - DOCKER_NETWORK=--use_docker_network
    - STREAMING_TEST=--streaming
    - OPENMRS_TEST=--openmrs

- name: '${_REPOSITORY}/synthea-uploader:${_TAG}'
  id: 'Upload to OpenMRS'
  env:
    - CONVERT=--convert_to_openmrs
    - INPUT_DIR=/workspace/synthea-hiv/sample_data
    - SINK_TYPE=OpenMRS
    - FHIR_ENDPOINT=http://openmrs:8080/openmrs/ws/fhir2/R4

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline FHIR-search mode with OpenMRS source'
  env:
    - PARQUET_PATH=/workspace/e2e-tests/FHIR_SEARCH_OPENMRS
    - SINK_PATH=http://sink-server:8080/fhir
    - SINK_USERNAME=hapi
    - SINK_PASSWORD=hapi

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for FHIR-search mode with OpenMRS source'
  env:
    - PARQUET_SUBDIR=FHIR_SEARCH_OPENMRS
    - DOCKER_NETWORK=--use_docker_network
    - OPENMRS_TEST=--openmrs

- name: '${_REPOSITORY}/batch-pipeline:${_TAG}'
  id: 'Run Batch Pipeline for JDBC mode with OpenMRS source'
  env:
    - JDBC_MODE_ENABLED=true
    - PARQUET_PATH=/workspace/e2e-tests/JDBC_OPENMRS
    - SINK_PATH=http://sink-server:8080/fhir
    - SINK_USERNAME=hapi
    - SINK_PASSWORD=hapi
    - FHIR_DATABASE_CONFIG_PATH=/workspace/utils/dbz_event_to_fhir_config.json

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Run E2E Test for JDBC mode with OpenMRS source'
  env:
    - PARQUET_SUBDIR=JDBC_OPENMRS
    - DOCKER_NETWORK=--use_docker_network
    - OPENMRS_TEST=--openmrs

- name: '${_REPOSITORY}/e2e-tests:${_TAG}'
  id: 'Test Indicators'
  entrypoint: /bin/bash
  args:
  - -c
  - 'cd dwh; ./validate_indicators.sh'

- name: 'docker/compose'
  id: 'Turn down Webserver and HAPI Server'
  args: ['-f', './docker/openmrs-compose.yaml', '-f', './docker/sink-compose.yml', 'down']

substitutions:
  # To use substitutions in your local build, use the flag --substitutions 
  # along with the key=value pair that you want to substitute
  # More details here: https://cloud.google.com/build/docs/build-debug-locally
  _TAG: local # Cloud Build replaces this with the Commit SHA
  _REPOSITORY: fhir-analytics # Cloud Build replaces this with 
                              # us-docker.pkg.dev/${PROJECT_ID}/fhir-analytics

images: 
  # If run locally, images are available on your local machine through Docker
  # You can then re-tag images and push to your own Docker repo
  - '${_REPOSITORY}/streaming-pipeline:${_TAG}'
  - '${_REPOSITORY}/batch-pipeline:${_TAG}'
  - '${_REPOSITORY}/e2e-tests:${_TAG}'
  - '${_REPOSITORY}/synthea-uploader:${_TAG}'
logsBucket: "gs://cloud-build-gh-logs"
timeout: '2h'
options:
  machineType: 'N1_HIGHCPU_32'