ci/pipeline.yml

---
groups:
  - name: bosh
    jobs:
      - unit-2.4
      - unit-2.4-mysql
      - unit-2.4-postgres
      - unit-2.4-rds
      - blobstore-client-integration
      - integration-postgres-gocli-sha2
      - integration-mysql-gocli-sha1
      - load-tests-postgres
      - load-tests-mysql
      - legacy-load-tests-postgres
      - legacy-load-tests-mysql
      - fuzz-tests
      - candidate-release
      - finalize-bosh-release
      - bats-centos
      - bats-ubuntu
      - brats-ubuntu
      - upgrade-tests
      - blobstore-performance

  - name: mysql
    jobs:
      - unit-2.4-mysql
      - unit-2.4-rds
      - integration-mysql-gocli-sha1

  - name: postgres
    jobs:
      - unit-2.4-postgres
      - unit-2.4-rds

  - name: finalize
    jobs:
      - finalize-bosh-release

shared:
  - &deploy-director
    task: deploy-director
    tags: [vsphere-v5.1]
    file: bosh-src/ci/bats/tasks/deploy-director.yml
    params:
      BAT_INFRASTRUCTURE: vsphere
      BOSH_CLIENT:                          {{stemcell-test-director-username}}
      BOSH_CLIENT_SECRET:                   {{stemcell-test-director-password}}
      BOSH_VSPHERE_VCENTER:                 {{vcenter-ip}}
      BOSH_VSPHERE_VCENTER_USER:            {{vcenter-user}}
      BOSH_VSPHERE_VCENTER_PASSWORD:        {{vcenter-password}}
      BOSH_VSPHERE_VERSION:                 {{vsphere-version}}
      BOSH_VSPHERE_VCENTER_DC:              {{vcenter-dc}}
      BOSH_VSPHERE_VCENTER_CLUSTER:         {{vcenter-cluster}}
      BOSH_VSPHERE_VCENTER_DATASTORE:       {{vcenter-datastore}}
      BOSH_VSPHERE_VCENTER_VLAN:            {{vcenter-vlan}}
      BOSH_VSPHERE_VCENTER_VM_FOLDER:       {{vcenter-vm-folder}}
      BOSH_VSPHERE_VCENTER_TEMPLATE_FOLDER: {{vcenter-template-folder}}
      BOSH_VSPHERE_VCENTER_DISK_PATH:       {{vcenter-disk-path}}

  - &prepare-bats-config
    task: prepare-bats
    tags: [vsphere-v5.1]
    file: bosh-src/ci/bats/iaas/vsphere/prepare-bats-config.yml

  - &run-bats
    task: run-bats
    tags: [vsphere-v5.1]
    file: bats/ci/tasks/run-bats.yml

  - &teardown
    task: teardown
    tags: [vsphere-v5.1]
    file: bosh-src/ci/bats/tasks/destroy-director.yml

  - &slack-alert
    put: slack-alert
    params:
      channel: {{slack_channel_name}}
      icon_url: http://cl.ly/image/3e1h0H3H2s0P/concourse-logo.png
      text: {{slack_failure_message}}

jobs:
  - name: unit-2.4
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - { get: bosh-src, trigger: true }

      - task: test
        file: bosh-src/ci/tasks/test-unit.yml
        params:
          RUBY_VERSION: 2.4.2
          DB: sqlite
        on_failure:
          <<: *slack-alert

  - name: unit-2.4-mysql
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - { get: bosh-src, trigger: true }
      - aggregate:
        - task: test-mysql-5.5
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-mysql-5.5
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: mysql
          on_failure:
            <<: *slack-alert

        - task: test-mysql-5.6
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-mysql-5.6
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: mysql
          on_failure:
            <<: *slack-alert

        - task: test-mysql-5.7
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-mysql-5.7
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: mysql
          on_failure:
            <<: *slack-alert

  - name: unit-2.4-postgres
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - { get: bosh-src, trigger: true }
      - aggregate:
        - task: test-postgres-9.3
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-postgres-9.3
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: postgresql
              DB_VERSION: 9.3
          on_failure:
            <<: *slack-alert

        - task: test-postgres-9.4
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-postgres-9.4
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: postgresql
              DB_VERSION: 9.4
          on_failure:
            <<: *slack-alert

        - task: test-postgres-9.5
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-postgres-9.5
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: postgresql
              DB_VERSION: 9.5
          on_failure:
            <<: *slack-alert

        - task: test-postgres-9.6
          privileged: true
          config:
            platform: linux
            image_resource:
              type: docker-image
              source:
                repository: bosh/main-postgres-9.6
            inputs:
              - name: bosh-src
            run:
              path: bosh-src/ci/tasks/test-unit.sh
            params:
              RUBY_VERSION: 2.4.2
              DB: postgresql
              DB_VERSION: 9.6
          on_failure:
            <<: *slack-alert

  - name: unit-2.4-rds
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - { get: bosh-src, trigger: true }
      - aggregate:
        - task: test-mysql-rds
          privileged: true
          file: bosh-src/ci/tasks/test-unit-remote-db.yml
          params:
            RUBY_VERSION: 2.4.2
            DB: mysql
            DB_HOST: {{mysql-rds-host}}
            DB_USER: {{mysql-rds-user}}
            DB_PASSWORD: {{mysql-rds-password}}
            AWS_ACCESS_KEY_ID: {{mysql-rds-aws-access-key-id}}
            AWS_SECRET_ACCESS_KEY: {{mysql-rds-aws-secret-access-key}}
            AWS_REGION: {{mysql-rds-aws-region}}
            RDS_MYSQL_DB_IDENTIFIER: {{mysql-rds-db-identifier}}
        - task: test-postgresql-rds
          privileged: true
          file: bosh-src/ci/tasks/test-unit-remote-db.yml
          params:
            RUBY_VERSION: 2.4.2
            DB: postgresql
            DB_HOST: {{postgresql-rds-host}}
            DB_USER: {{postgresql-rds-user}}
            DB_PASSWORD: {{postgresql-rds-password}}
          on_failure:
            <<: *slack-alert

  - name: blobstore-client-integration
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - { get: bosh-src, trigger: true }
      - task: test-s3
        file: bosh-src/ci/tasks/test-s3-blobstore-client-integration.yml
        params:
          access_key_id: {{blobstore_client_aws_access_key_id}}
          secret_access_key: {{blobstore_client_aws_secret_access_key}}
          s3_region: {{blobstore_client_aws_s3_region}}
          s3_host: {{blobstore_client_aws_s3_host}}
          run_aws_tests: "Not null"
      - task: test-local
        file: bosh-src/ci/tasks/test-local-blobstore-client-integration.yml
      - task: test-dav
        file: bosh-src/ci/tasks/test-dav-blobstore-client-integration.yml
      - task: test-gcs
        file: bosh-src/ci/tasks/test-gcs-blobstore-client-integration.yml
        params:
          google_project: {{blobstore_client_google_project}}
          google_json_key_data: {{blobstore_client_google_json_key_data}}

  - name: blobstore-performance
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - { get: bosh-src, trigger: true }
      - { get: davcli, trigger: true }
      - task: test
        privileged: true
        file: bosh-src/ci/tasks/test-blobstore-load.yml
        on_failure:
          <<: *slack-alert

  - name: integration-postgres-gocli-sha2
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - aggregate:
        - get: bosh-src
          trigger: true
        - get: bosh-cli
          trigger: true
        - get: bosh-agent
          trigger: true
      - aggregate:
        - task: test-group-1
          privileged: true
          file: bosh-src/ci/tasks/test-integration-gocli.yml
          tags: ["bosh-integration-7"]
          params:
            DB:           postgresql
            RUBY_VERSION: 2.4.2
            NUM_GROUPS:   24
            GROUP:        1,4,7,10,13,16,19,22
            SHA2_MODE:    true

        - task: test-group-2
          privileged: true
          file: bosh-src/ci/tasks/test-integration-gocli.yml
          tags: ["bosh-integration-8"]
          params:
            DB:           postgresql
            RUBY_VERSION: 2.4.2
            NUM_GROUPS:   24
            GROUP:        2,5,8,11,14,17,20,23
            SHA2_MODE:    true

        - task: test-group-3
          privileged: true
          file: bosh-src/ci/tasks/test-integration-gocli.yml
          tags: ["bosh-integration-9"]
          params:
            DB:           postgresql
            RUBY_VERSION: 2.4.2
            NUM_GROUPS:   24
            GROUP:        3,6,9,12,15,18,21,24
            SHA2_MODE:    true

        on_failure:
          <<: *slack-alert

  - name: integration-mysql-gocli-sha1
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - aggregate:
        - get: bosh-src
          trigger: true
        - get: bosh-cli
          trigger: true
        - get: bosh-agent
          trigger: true
      - aggregate:
        - task: test-group-1
          privileged: true
          file: bosh-src/ci/tasks/test-integration-gocli.yml
          tags: ["bosh-integration-4"]
          params:
            DB:           mysql
            RUBY_VERSION: 2.4.2
            NUM_GROUPS: 24
            GROUP:      1,4,7,10,13,16,19,22

        - task: test-group-2
          privileged: true
          file: bosh-src/ci/tasks/test-integration-gocli.yml
          tags: ["bosh-integration-5"]
          params:
            DB:           mysql
            RUBY_VERSION: 2.4.2
            NUM_GROUPS: 24
            GROUP:      2,5,8,11,14,17,20,23

        - task: test-group-3
          privileged: true
          file: bosh-src/ci/tasks/test-integration-gocli.yml
          tags: ["bosh-integration-6"]
          params:
            DB:           mysql
            RUBY_VERSION: 2.4.2
            NUM_GROUPS: 24
            GROUP:      3,6,9,12,15,18,21,24

        on_failure:
          <<: *slack-alert

  - name: upgrade-tests
    public: true
    serial: true
    build_logs_to_retain: 250
    plan:
      - aggregate:
        - get: bosh-src
          trigger: true
        - get: bosh-cli
          trigger: true
        - get: bosh-agent
          trigger: true
      - aggregate:
        - task: upgrade-with-postgres
          privileged: true
          file: bosh-src/ci/tasks/test-upgrade.yml
          tags: ["bosh-integration"]
          params:
            DB: postgresql
            RUBY_VERSION: 2.4.2

        - task: upgrade-with-mysql
          privileged: true
          file: bosh-src/ci/tasks/test-upgrade.yml
          tags: ["bosh-integration"]
          params:
            DB: mysql
            RUBY_VERSION: 2.4.2

  - name: load-tests-postgres
    public: true
    serial: true
    serial_groups: ["load_tests"] # heavy footprint! run one at a time.
    build_logs_to_retain: 250
    plan:
    - { get: load-tests-interval, trigger: true }
    - { get: bosh-load-tests-workspace }
    - { get: warden-ubuntu-trusty }
    - { get: bosh-candidate-release-tarballs }
    - { get: bosh-cli }

    - task: test
      privileged: true
      file: bosh-load-tests-workspace/ci/tasks/test.yml
      tags: ["bosh-load-tests"]
      input_mapping:
        bosh-candidate-stemcell: warden-ubuntu-trusty
        bosh-candidate-release: bosh-candidate-release-tarballs
      params:
        DB: postgresql
        LEGACY: false
      on_failure:
        <<: *slack-alert

  - name: legacy-load-tests-postgres
    public: true
    serial: true
    serial_groups: ["load_tests"] # heavy footprint! run one at a time.
    build_logs_to_retain: 250
    plan:
    - { get: load-tests-interval, trigger: true }
    - { get: bosh-load-tests-workspace }
    - { get: bosh-cli }
    - { get: bosh-candidate-release-tarballs }
    - { get: warden-ubuntu-trusty }

    - task: test
      privileged: true
      file: bosh-load-tests-workspace/ci/tasks/test.yml
      tags: ["bosh-load-tests"]
      input_mapping:
        bosh-candidate-stemcell: warden-ubuntu-trusty
        bosh-candidate-release: bosh-candidate-release-tarballs
      params:
        DB: postgresql
        LEGACY: true
      on_failure:
        <<: *slack-alert

  - name: load-tests-mysql
    public: true
    serial: true
    serial_groups: ["load_tests"] # heavy footprint! run one at a time.
    build_logs_to_retain: 250
    plan:
    - { get: load-tests-interval, trigger: true }
    - { get: bosh-load-tests-workspace }
    - { get: bosh-cli }
    - { get: bosh-candidate-release-tarballs }
    - { get: warden-ubuntu-trusty }

    - task: test
      privileged: true
      file: bosh-load-tests-workspace/ci/tasks/test.yml
      tags: ["bosh-load-tests"]
      input_mapping:
        bosh-candidate-stemcell: warden-ubuntu-trusty
        bosh-candidate-release: bosh-candidate-release-tarballs
      params:
        DB: mysql
        LEGACY: false
      on_failure:
        <<: *slack-alert

  - name: legacy-load-tests-mysql
    public: true
    serial: true
    serial_groups: ["load_tests"] # heavy footprint! run one at a time.
    build_logs_to_retain: 250
    plan:
    - { get: load-tests-interval, trigger: true }
    - { get: bosh-load-tests-workspace }
    - { get: bosh-cli }
    - { get: bosh-candidate-release-tarballs }
    - { get: warden-ubuntu-trusty }

    - task: test
      privileged: true
      file: bosh-load-tests-workspace/ci/tasks/test.yml
      tags: ["bosh-load-tests"]
      input_mapping:
        bosh-candidate-stemcell: warden-ubuntu-trusty
        bosh-candidate-release: bosh-candidate-release-tarballs
      params:
        DB: mysql
        LEGACY: true
      on_failure:
        <<: *slack-alert

  - name: fuzz-tests
    public: true
    serial: true
    build_logs_to_retain: 2500
    plan:
    - { get: fuzz-interval-trigger, trigger: true }
    - { get: bosh-src, trigger: true }
    - { get: bosh-agent }
    - { get: bosh-fuzz-tests }
    - { get: bosh-cli }

    - task: test
      privileged: true
      file: bosh-fuzz-tests/ci/tasks/test.yml
      tags: ["bosh-integration"]
      params:
        BOSH_SRC_PATH: bosh-src/src
        RUBY_VERSION: 2.4.2
      on_failure:
        <<: *slack-alert

  - name: candidate-release
    plan:
    - get: bosh-src
      trigger: true
      passed:
      - unit-2.4
      - unit-2.4-mysql
      - unit-2.4-postgres
      - integration-mysql-gocli-sha1
      - integration-postgres-gocli-sha2
      - blobstore-client-integration
      - fuzz-tests
      - upgrade-tests
      - blobstore-performance
    - get: bosh-cli
    - get: candidate-version
      params:
        bump: major
    - task: make
      file: bosh-src/ci/tasks/make-candidate.yml
    - put: bosh-candidate-release-tarballs
      params:
        file: "release/bosh-dev-release.tgz"

  - name: bats-centos
    serial: true
    plan:
    - do:
      - aggregate:
        - get: bosh-release
          resource: bosh-candidate-release-tarballs
          trigger: true
          passed:
          - candidate-release
        - get: cpi-release
        - get: stemcell
          resource: vsphere-esxi-centos-7
        - get: bosh-cli
        - get: bats
        - get: bosh-deployment
        - get: bosh-src
          passed:
          - candidate-release

      - put: environment
        params:
          acquire: true

      - do:
        - <<: *deploy-director

        - <<: *prepare-bats-config
          params:
            STEMCELL_NAME: bosh-vsphere-esxi-centos-7-go_agent

        - <<: *run-bats
        ensure:
          do:
            - <<: *teardown
      ensure:
        do:
        - {put: environment, params: {release: environment}}

  - name: bats-ubuntu
    serial: true
    plan:
    - do:
      - aggregate:
        - get: bosh-release
          resource: bosh-candidate-release-tarballs
          trigger: true
          passed:
            - candidate-release
        - get: cpi-release
        - get: stemcell
          resource: vsphere-esxi-ubuntu-trusty
        - get: bosh-cli
        - get: bats
        - get: bosh-deployment
        - get: bosh-src
          passed:
            - candidate-release

      - put: environment
        params:
          acquire: true

      - do:
        - <<: *deploy-director

        - <<: *prepare-bats-config
          params:
            STEMCELL_NAME: bosh-vsphere-esxi-ubuntu-trusty-go_agent

        - <<: *run-bats
        ensure:
          do:
            - <<: *teardown
      ensure:
        do:
        - {put: environment, params: {release: environment}}

  - name: brats-ubuntu
    serial: true
    plan:
    - do:
      - aggregate:
        - get: bosh-src
          passed:
          - candidate-release
        - get: bosh-dns-release
        - get: candidate-warden-ubuntu-stemcell
        - get: bosh-release
          resource: bosh-candidate-release-tarballs
          trigger: true
          passed:
            - candidate-release
      - task: test-brats
        file: bosh-src/ci/tasks/test-brats.yml
        tags: ["worker-brats"]
        privileged: true
        input_mapping:
          bosh-dev-release: bosh-release

  - name: finalize-bosh-release
    serial: true
    plan:
      - aggregate:
        - get: bosh-src-master
          resource: bosh-src
          passed:
            - bats-centos
            - bats-ubuntu
        - get: bosh-dev-release
          resource: bosh-candidate-release-tarballs
          passed:
            - bats-centos
            - bats-ubuntu
        - get: bosh-cli
        - get: candidate-version
          params:
            bump: major
      - task: promote-release
        file: bosh-src-master/ci/tasks/finalize-bosh-release.yml
        params:
          BLOBSTORE_ACCESS_KEY_ID: {{bosh_release_access_key_id}}
          BLOBSTORE_SECRET_ACCESS_KEY: {{bosh_release_secret_access_key}}
      - put: candidate-version
        params:
          file: bumped-candidate-version/number
      - put: bosh-master
        params:
          repository: bosh-src-with-final
          tag: /tmp/build/put/bosh-src-with-final-tag/tag-name
          annotate: /tmp/build/put/bosh-src-with-final-tag/annotate-msg

resources:
  - name: bosh-src
    type: git
    source:
      uri: {{bosh_src_url}}
      branch: master
      private_key: {{github_deployment_key}}

  - name: candidate-version
    type: semver
    source:
      bucket: {{candidate_release_bucket}}
      key: version
      access_key_id: {{candidate_release_access_key_id}}
      secret_access_key: {{candidate_release_secret_access_key}}

  - name: bosh-candidate-release-tarballs
    type: s3
    source:
      bucket: {{candidate_release_bucket}}
      access_key_id: {{candidate_release_access_key_id}}
      secret_access_key: {{candidate_release_secret_access_key}}
      versioned_file: "bosh-dev-release.tgz"

  - name: davcli
    type: s3
    source:
      regexp: davcli-(.*)-linux-amd64
      bucket: davcli
      region_name: us-east-1

  - name: bosh-fuzz-tests
    type: git
    source:
      uri: https://github.com/cloudfoundry-incubator/bosh-fuzz-tests.git
      branch: master

  - name: bosh-load-tests-workspace
    type: git
    source:
      uri: https://github.com/cloudfoundry-incubator/bosh-load-tests-workspace
      branch: master

  - name: bosh-master
    type: git
    source:
      uri: {{bosh_src_url}}
      branch: master
      private_key: {{github_deployment_key}}

  - name: fuzz-interval-trigger
    type: time
    source:
      interval: 15m

  - name: load-tests-interval
    type: time
    source:
      interval: 1h

  - name: slack-alert
    type: slack-notification
    source:
      url: {{slack_hook_url}}

  - name: bosh-cli
    type: s3
    source:
      regexp: alpha-bosh-cli-(.*)-linux-amd64
      bucket: {{bosh_cli_aws_s3_alpha_release_bucket}}
      region_name: {{bosh_cli_aws_s3_release_bucket_region}}

  - name: bosh-dns-release
    type: bosh-io-release
    source:
      repository: cloudfoundry/dns-release

  #
  # BATS
  #

  - name: bats
    type: git
    source:
      uri: https://github.com/cloudfoundry/bosh-acceptance-tests.git
      branch: gocli-bats

  - name: bosh-deployment
    type: git
    source:
      uri: https://github.com/cloudfoundry/bosh-deployment
      branch: master

  - name: environment
    type: pool
    source:
      pool: vsphere
      uri: git@github.com:pivotal-cf-experimental/bats-concourse-pool.git
      branch: master
      private_key: {{github_deployment_key__bosh-cpi-environments}}

  - name: vsphere-esxi-ubuntu-trusty
    type: bosh-io-stemcell
    source:
      name: bosh-vsphere-esxi-ubuntu-trusty-go_agent

  - name: candidate-warden-ubuntu-stemcell
    type: s3
    source:
      bucket: bosh-core-stemcells-candidate
      regexp: warden/bosh-stemcell-(.+)-warden-boshlite-ubuntu-trusty-go_agent.tgz

  - name: warden-ubuntu-trusty
    type: bosh-io-stemcell
    source:
      name: bosh-warden-boshlite-ubuntu-trusty-go_agent

  - name: vsphere-esxi-centos-7
    type: bosh-io-stemcell
    source:
      name: bosh-vsphere-esxi-centos-7-go_agent

  - name: cpi-release
    type: bosh-io-release
    source:
      repository: cloudfoundry-incubator/bosh-vsphere-cpi-release

  - name: bosh-agent
    type: git
    source:
      uri: https://github.com/cloudfoundry/bosh-agent
      branch: master