Merge pull request #55 from arenadata/5.24.0-sync

ADBDEV-498 ADB 5.24.0 sync
arenadata · Dec 20, 2019 · 203ab61 · 203ab61
2 parents fb9fe2f + 58214d7
commit 203ab61
Show file tree

Hide file tree

Showing 112 changed files with 3,507 additions and 1,282 deletions.
diff --git a/add_libs/gporca b/add_libs/gporca
diff --git a/concourse/pipelines/5X_STABLE-generated.yml b/concourse/pipelines/5X_STABLE-generated.yml
@@ -12,7 +12,7 @@
 ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline
 ## using appropriate tool (example: gen_pipeline.py -t prod).
 ## ----------------------------------------------------------------------
-## Generated by gen_pipeline.py at: 2019-10-22 11:52:17.174533
+## Generated by gen_pipeline.py at: 2019-12-06 08:32:51.370793
 ## Template file: gpdb-tpl.yml
 ## OS Types: ['centos6', 'centos7', 'sles', 'aix7', 'win', 'ubuntu16']
 ## Test Sections: ['ICW', 'CS', 'MPP', 'MM', 'DPM', 'UD', 'FileRep', 'AA']
@@ -95,6 +95,7 @@ groups:
   - MM_reindexdb
   - MM_gpperfmon
   - MM_gpinitsystem
+  - MM_gpstart
   - MM_gppkg
   - MM_gpcheck
   - MM_gpexpand
@@ -118,7 +119,6 @@ groups:
   ## --------------------------------------------------------------------
   - gate_ud_start
   - regression_tests_gphdfs_hadoop_centos
-  - regression_tests_gphdfs_mapr_centos
   - gate_ud_end
   ## --------------------------------------------------------------------
   - gate_advanced_analytics_start
@@ -234,6 +234,7 @@ groups:
   - MM_reindexdb
   - MM_gpperfmon
   - MM_gpinitsystem
+  - MM_gpstart
   - MM_gppkg
   - MM_gpcheck
   - MM_gpexpand
@@ -281,7 +282,6 @@ groups:
   jobs:
   - gate_ud_start
   - regression_tests_gphdfs_hadoop_centos
-  - regression_tests_gphdfs_mapr_centos
   - gate_ud_end
 
 ## ======================================================================
@@ -2977,6 +2977,15 @@ jobs:
     params:
       BEHAVE_TAGS: gpinitsystem
 
+- name: MM_gpstart
+  plan:
+  - aggregate: *gets_for_behave
+  - task: gpstart
+    file: gpdb_src/concourse/tasks/behave_gpdb.yml
+    image: centos-gpdb-dev-6
+    params:
+      BEHAVE_TAGS: gpstart
+
 - name: MM_pt-rebuild
   plan:
   - aggregate: *gets_for_behave
@@ -3059,6 +3068,7 @@ jobs:
       - MM_reindexdb
       - MM_gpperfmon
       - MM_gpinitsystem
+      - MM_gpstart
       - MM_gppkg
       - MM_gpcheck
       - MM_gpexpand
@@ -3887,47 +3897,12 @@ jobs:
       TARGET_OS: centos
       TARGET_OS_VERSION: 6
 
-- name: regression_tests_gphdfs_mapr_centos
-  ensure:
-    <<: *set_failed
-  on_success:
-    <<: *ccp_destroy
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [gate_ud_start]
-    - get: bin_gpdb
-      passed: [gate_ud_start]
-      trigger: true
-      resource: bin_gpdb_centos6
-    - get: ccp_src
-    - get: centos-gpdb-dev-6
-  - put: terraform
-    params:
-      <<: *ccp_default_params
-      vars:
-        <<: *ccp_default_vars
-        number_of_nodes: 1
-        PLATFORM: centos7
-        instance_type: n1-standard-4
-  - task: gen_and_initialize_mapr
-    file: gpdb_src/concourse/tasks/gen_mapr.yml
-    params:
-      <<: *ccp_gen_cluster_default_params
-  - task: regression_tests_gphdfs_mapr
-    file: gpdb_src/concourse/tasks/regression_tests_gphdfs_mapr.yml
-    image: centos-gpdb-dev-6
-    params:
-      TARGET_OS: centos
-      TARGET_OS_VERSION: 6
-
 - name: gate_ud_end
   plan:
   - aggregate:
     - get: gpdb_src
       passed: &gate_ud_end
       - regression_tests_gphdfs_hadoop_centos
-      - regression_tests_gphdfs_mapr_centos
       trigger: true
     - get: bin_gpdb_centos6
       passed: *gate_ud_end
@@ -4527,7 +4502,6 @@ jobs:
       - cs_filerep_end_to_end_full_primary
       - cs_filerep_end_to_end_incr_primary
       - regression_tests_gphdfs_hadoop_centos
-      - regression_tests_gphdfs_mapr_centos
       - DPM_backup-restore_ddboost_part1
       - DPM_backup-restore_ddboost_part2
       - DPM_backup-restore_ddboost_part3
@@ -4543,6 +4517,7 @@ jobs:
       - MM_reindexdb
       - MM_gpperfmon
       - MM_gpinitsystem
+      - MM_gpstart
       - gptransfer-43x-to-5x
       - gptransfer-5x-to-5x
       - mpp_interconnect

diff --git a/concourse/pipelines/pr_pipeline.yml b/concourse/pipelines/pr_pipeline.yml
@@ -20,6 +20,19 @@ resources:
         - gpdb-doc/*
         - README*
 
+  - name: gpdb_5x_upstream
+    type: git
+    source:
+      uri: https://github.com/greenplum-db/gpdb
+      branch: "5X_STABLE"
+
+  - name: gpaddon_src
+    type: git
+    source:
+      branch: {{gpaddon-git-branch}}
+      private_key: {{gpaddon-git-key}}
+      uri: {{gpaddon-git-remote}}
+
   - name: centos-gpdb-dev-6
     type: docker-image
     source:
@@ -35,49 +48,86 @@ resources:
       secret_access_key: {{bucket-secret-access-key}}
       versioned_file: {{binary_swap_gpdb_centos_versioned_file}}
 
-  - name: bin_gpdb_centos6
-    type: s3
-    source:
-      access_key_id: {{bucket-access-key-id}}
-      bucket: {{bucket-name}}
-      region_name: {{aws-region}}
-      secret_access_key: {{bucket-secret-access-key}}
-      versioned_file: {{bin_gpdb_centos_versioned_file}}
-
 jobs:
   - name: compile-and-test-pull-request
     plan:
-      - aggregate:
+      - in_parallel:
           - get: gpdb_pr
             version: every
             trigger: true
-          - get: bin_gpdb
-            resource: bin_gpdb_centos6
-            trigger: true
           - get: binary_swap_gpdb
             resource: binary_swap_gpdb_centos6
-            trigger: true
           - get: centos-gpdb-dev-6
-            trigger: true
+          - get: gpaddon_src
+          - get: gpdb_5x_upstream
+      # We use the 5X_STABLE source for running make sync_tools instead for security.
+      # For example, this would prevent a user to submit a PR that maliciously
+      # modifies the sync_tools task to expose the ivy credentials.
+      - task: sync_tools
+        input_mapping:
+          gpdb_src: gpdb_5x_upstream
+        file: gpdb_5x_upstream/concourse/tasks/sync_tools.yml
+        image: centos-gpdb-dev-6
+        params:
+          IVYREPO_HOST: {{ivyrepo_host}}
+          IVYREPO_REALM: {{ivyrepo_realm}}
+          IVYREPO_USER: {{ivyrepo_user}}
+          IVYREPO_PASSWD: {{ivyrepo_passwd}}
+          TARGET_OS: centos
+          TARGET_OS_VERSION: 6
+          TASK_OS: centos
+          TASK_OS_VERSION: 6
 
-      - aggregate:
-        - put: gpdb_pr
+      - put: gpdb_pr
+        params:
+          path: gpdb_pr
+          status: pending
+      - # "do" the remaining steps with these hooks:
+        on_failure:
+          put: gpdb_pr
           params:
             path: gpdb_pr
-            status: pending
+            status: failure
+        on_success:
+          put: report_pr_success
+          resource: gpdb_pr
+          params:
+            path: gpdb_pr
+            status: success
+        do:
+        # Fetch tags and submodules, because the PR resource doesn't.
+        # gpdb_src is to be used by compile_gpdb and ic_gpdb.
+        - task: init gpdb_src
+          image: centos-gpdb-dev-6
+          config:
+            platform: linux
+            run:
+              path: bash
+              args:
+                - -c
+                - |
+                  git clone gpdb_pr gpdb_src &&
+                  cd gpdb_src &&
+                  git fetch https://github.com/greenplum-db/gpdb.git --tags &&
+                  git submodule update --init --recursive
+            inputs: [{ name: gpdb_pr }]
+            outputs: [{ name: gpdb_src }]
+        - task: compile_gpdb
+          file: gpdb_src/concourse/tasks/compile_gpdb.yml
+          image: centos-gpdb-dev-6
+          params:
+            CONFIGURE_FLAGS: {{configure_flags}}
+            TARGET_OS: centos
+            TARGET_OS_VERSION: 6
+            BLD_TARGETS: "clients loaders"
+
         - task: ic_gpdb
-          file: gpdb_pr/concourse/tasks/ic_gpdb_binary_swap.yml
+          file: gpdb_src/concourse/tasks/ic_gpdb_binary_swap.yml
           image: centos-gpdb-dev-6
           input_mapping:
-            gpdb_src: gpdb_pr
+            bin_gpdb: gpdb_artifacts
           params:
             MAKE_TEST_COMMAND: PGOPTIONS='-c optimizer=off' installcheck-world
             BLDWRAP_POSTGRES_CONF_ADDONS: "fsync=off"
             TEST_OS: centos
             CONFIGURE_FLAGS: {{configure_flags}}
-
-      - put: report_pr_success
-        resource: gpdb_pr
-        params:
-          path: gpdb_pr
-          status: success
diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml
@@ -23,7 +23,8 @@
 {% set MM_2_suites = ['analyzedb',
                       'reindexdb',
                       'gpperfmon',
-                      'gpinitsystem'] %}
+                      'gpinitsystem',
+                      'gpstart'] %}
 {% set aa_Test_suites = ['Planner_centos6',
                          'Orca_centos6',
                          'Planner_centos7',
@@ -131,7 +132,6 @@ groups:
   ## --------------------------------------------------------------------
   - gate_ud_start
   - regression_tests_gphdfs_hadoop_centos
-  - regression_tests_gphdfs_mapr_centos
   - gate_ud_end
 {% endif %}
 {% if "AA" in test_sections %}
@@ -299,7 +299,6 @@ groups:
   jobs:
   - gate_ud_start
   - regression_tests_gphdfs_hadoop_centos
-  - regression_tests_gphdfs_mapr_centos
   - gate_ud_end
 
 {% endif %}
@@ -3522,47 +3521,12 @@ jobs:
       TARGET_OS: centos
       TARGET_OS_VERSION: 6
 
-- name: regression_tests_gphdfs_mapr_centos
-  ensure:
-    <<: *set_failed
-  on_success:
-    <<: *ccp_destroy
-  plan:
-  - aggregate:
-    - get: gpdb_src
-      passed: [gate_ud_start]
-    - get: bin_gpdb
-      passed: [gate_ud_start]
-      trigger: [[ test_trigger ]]
-      resource: bin_gpdb_centos6
-    - get: ccp_src
-    - get: centos-gpdb-dev-6
-  - put: terraform
-    params:
-      <<: *ccp_default_params
-      vars:
-        <<: *ccp_default_vars
-        number_of_nodes: 1
-        PLATFORM: centos7
-        instance_type: n1-standard-4
-  - task: gen_and_initialize_mapr
-    file: gpdb_src/concourse/tasks/gen_mapr.yml
-    params:
-      <<: *ccp_gen_cluster_default_params
-  - task: regression_tests_gphdfs_mapr
-    file: gpdb_src/concourse/tasks/regression_tests_gphdfs_mapr.yml
-    image: centos-gpdb-dev-6
-    params:
-      TARGET_OS: centos
-      TARGET_OS_VERSION: 6
-
 - name: gate_ud_end
   plan:
   - aggregate:
     - get: gpdb_src
       passed: &gate_ud_end
       - regression_tests_gphdfs_hadoop_centos
-      - regression_tests_gphdfs_mapr_centos
       trigger: true
     - get: bin_gpdb_centos6
       passed: *gate_ud_end
@@ -3873,7 +3837,6 @@ jobs:
       - cs_[[ test_name ]]
 {% endfor %}
       - regression_tests_gphdfs_hadoop_centos
-      - regression_tests_gphdfs_mapr_centos
       - DPM_backup-restore_ddboost_part1
       - DPM_backup-restore_ddboost_part2
       - DPM_backup-restore_ddboost_part3