Split GCB triggers for Kubernetes applications (GoogleCloudPlatform#535)

.gitignore

@@ -1 +1,2 @@
 .build/
+*.pyc

.travis.yml

@@ -1,10 +1,11 @@
 matrix:
   include:
   - language: python
+    python: 2.7
     install: pip install jinja2
-    script: ./cloudbuild-k8s-generator.py --verify_only
+    env: TARGET=python-test
   - language: ruby
     install: gem install foodcritic
-    script:
-    - foodcritic --version
-    - foodcritic --cookbook-path=vm/chef/cookbooks --rule-file=vm/chef/.foodcritic --epic-fai=any
+    env: TARGET=vm-lint
+
+script: make "${TARGET}"

Makefile

@@ -0,0 +1,23 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# TODO(wgrzelak): Invoke targets into the container.
+
+python-test:
+	@python --version
+	@python -m unittest discover -s scripts -p "*_test.py"
+
+vm-lint:
+	@foodcritic --version
+	@foodcritic --cookbook-path=vm/chef/cookbooks --rule-file=vm/chef/.foodcritic --epic-fail=any

README.md

@@ -1,44 +1,41 @@
 # About
 
-Source for Google Click to Deploy solutions listed on
-Google Cloud Marketplace.
+Source for Google Click to Deploy solutions listed on Google Cloud Marketplace.
 
 # Disclaimer
 
 This is not an officially supported Google product.
 
 # Cloud Build CI
 
-This repository uses Cloud Build for continuous integration. The Cloud Build configuration file is located at [`cloudbuild.yaml`](cloudbuild.yaml).
+This repository uses Cloud Build for continuous integration. The Cloud Build
+configuration file is located at [`cloudbuild-k8s.yaml`](cloudbuild-k8s.yaml)
+for Kubernetes applications and [`cloudbuild-vm.yaml`](cloudbuild-vm.yaml) for
+[VM applications](vm/README.md#cloud-build-ci).
 
-## Manually run the build
+## Manually run the build for K8s applications
 
-Cloud Build can be triggered manually by running the following command
-from the root directory of this repository:
+Cloud Build can be triggered manually. Run the following command from the root
+directory of this repository:
 
 ```shell
 export GCP_PROJECT_TO_RUN_CLOUD_BUILD=<>
 export GKE_CLUSTER_NAME=<>
 export GKE_CLUSTER_LOCATION=<e.g. us-central1>
+export SOLUTION_NAME=<e.g. wordpress>
 
 gcloud builds submit . \
-  --config cloudbuild.yaml \
-  --substitutions _CLUSTER_NAME=$GKE_CLUSTER_NAME,_CLUSTER_LOCATION=$GKE_CLUSTER_LOCATION \
+  --config cloudbuild-k8s.yaml \
+  --substitutions _CLUSTER_NAME=$GKE_CLUSTER_NAME,_CLUSTER_LOCATION=$GKE_CLUSTER_LOCATION,_SOLUTION_NAME=$SOLUTION_NAME \
   --project $GCP_PROJECT_TO_RUN_CLOUD_BUILD \
   --verbosity info
 ```
 
-## Cloud Build configuration generator
+### Build steps
 
-To make the `cloudbuild.yaml` configuration easier to maintain, a generator for
-its contents was created.
-
-1.  The generator uses Jinja2 templates, install it using `pip install jinja2`
-    command.
-1.  To regenerate the file, run the following command:
-
-    ```shell
-    ./cloudbuild-k8s-generator.py
-    ```
-
-1.  As a result, new content will be saved in the `cloudbuild.yaml` file.
+1.  Build `click-to-deploy` Docker image. The image contains the necessary tools
+    required by the CI pipeline.
+1.  Generate Cloud Build configuration for an application defined in
+    `$_SOLUTION_NAME` variable. The configuration contains all required steps to
+    test the application.
+1.  Run additional Cloud Build instance using the generated configuration.

cloudbuild-k8s.yaml

@@ -0,0 +1,47 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 1800s # 30m
+steps:
+
+- id: Build C2D Docker Image
+  name: gcr.io/cloud-builders/docker
+  args:
+  - build
+  - --tag
+  - click-to-deploy
+  - --file
+  - k8s.Dockerfile
+  - .
+
+- id: Run Cloud Build K8s Generator
+  name: click-to-deploy
+  entrypoint: python
+  args:
+  - scripts/cloudbuild_k8s_generator.py
+  - --solution
+  - $_SOLUTION_NAME
+  - --output
+  - k8s/$_SOLUTION_NAME/cloudbuild.yaml
+
+- id: Test Solution
+  name: gcr.io/cloud-builders/gcloud
+  args:
+  - builds
+  - submit
+  - --config
+  - k8s/$_SOLUTION_NAME/cloudbuild.yaml
+  - --substitutions
+  - _CLUSTER_NAME=$_CLUSTER_NAME,_CLUSTER_LOCATION=$_CLUSTER_LOCATION
+  - .

k8s.Dockerfile

@@ -0,0 +1,20 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/google-appengine/debian9:latest
+
+RUN set -eux \
+    && apt-get update \
+    && apt-get install python-pip -y \
+    && pip install jinja2

cloudbuild-k8s-generator.py → scripts/cloudbuild_k8s_generator.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,10 +18,8 @@
 import argparse
 from jinja2 import Template
 
-CLOUDBUILD_CONFIG = 'cloudbuild.yaml'
-
 CLOUDBUILD_TEMPLATE = """
-# Copyright 2018 Google LLC
+# Copyright 2019 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -36,7 +34,7 @@
 # limitations under the License.
 
 ##################################################################################
-## This file is generated by cloudbuild-k8s-generator.py. Do not manually edit. ##
+## This file is generated by cloudbuild_k8s_generator.py. Do not manually edit. ##
 ##################################################################################
 
 timeout: 1800s # 30m
@@ -60,6 +58,8 @@
 
 - id: Get Kubernetes Credentials
   name: gcr.io/cloud-builders/gcloud
+  waitFor:
+  - '-'
   args:
   - container
   - clusters
@@ -92,8 +92,6 @@
     mkdir -p /workspace/.config/gcloud/
     cp -r $$HOME/.config/gcloud/ /workspace/.config/
 
-{%- for solution in solutions %}
-
 - id: Build {{ solution }}
   name: gcr.io/cloud-marketplace-tools/k8s/dev:local
   env:
@@ -108,10 +106,6 @@
   - -j4
   - app/build
 
-{%- endfor %}
-
-{%- for solution in solutions %}
-
 - id: Verify {{ solution }}
   name: gcr.io/cloud-marketplace-tools/k8s/dev:local
   waitFor:
@@ -131,8 +125,8 @@
   - -j4
   - app/verify
 
-{%- for extra_config in extra_configs[solution] %}
-{%- if not extra_config.ignore %}
+{%- for extra_config in extra_configs %}
+
 - id: Verify {{ solution }} ({{ extra_config['name'] }})
   name: gcr.io/cloud-marketplace-tools/k8s/dev:local
   waitFor:
@@ -156,82 +150,91 @@
   - -j4
   - app/verify
 
-{%- endif %}
-{%- endfor %}
-
 {%- endfor %}
 """.strip()
 
 
-def verify_cloudbuild(cloudbuild_contents):
-  if not os.path.isfile(CLOUDBUILD_CONFIG):
-    is_up_to_date = False
-  else:
-    with open(CLOUDBUILD_CONFIG, 'r') as cloudbuild_file:
-      is_up_to_date = cloudbuild_file.read() == cloudbuild_contents
+class CloudBuildConfig():
+
+  def __init__(self, solution):
+    self._solution = solution
+
+    self.extra_configs = []
+    self.path = None
+    self.template = CLOUDBUILD_TEMPLATE
 
-  return is_up_to_date
+  def exists(self):
+    return os.path.isfile(self.path)
+
+  def generate(self):
+    return Template(self.template).render(
+        solution=self._solution, extra_configs=self.extra_configs)
+
+  def verify(self):
+    if not self.exists():
+      return False
+
+    with open(self.path, 'r') as cloudbuild_file:
+      return cloudbuild_file.read() == self.generate()
+
+  def save(self):
+    with open(self.path, 'w') as cloudbuild_file:
+      cloudbuild_file.write(self.generate())
+
+  def remove(self):
+    return os.remove(self.path)
 
 
 def main():
   parser = argparse.ArgumentParser()
+  parser.add_argument(
+      '--solution',
+      type=str,
+      required=True,
+      help='solution name')
+  parser.add_argument(
+      '--output',
+      type=str,
+      required=True,
+      help='path to save configuration')
   parser.add_argument(
       '--verify_only',
       action='store_true',
       default=False,
-      help='verify %s file' % CLOUDBUILD_CONFIG)
+      help='verify configuration')
   args = parser.parse_args()
 
-  skiplist = []
-
   # Use extra_configs to run additional deployments
   # with non-default configurations.
   extra_configs = {
-    'wordpress': [
-      {
-        'name': 'Public service and ingress',
-        'env_vars': [
-          'PUBLIC_SERVICE_AND_INGRESS_ENABLED=true'
-        ],
-        # TODO(ISSUE/532): [WordPress] Fix flaky test
-        'ignore': True
-      },
-      {
-        'name': 'Prometheus metrics',
-        'env_vars': [
-          'METRICS_EXPORTER_ENABLED=true'
-        ]
-      },
-    ]
+      'wordpress': [
+          {
+              'name': 'Public service and ingress',
+              'env_vars': ['PUBLIC_SERVICE_AND_INGRESS_ENABLED=true']
+          },
+          {
+              'name': 'Prometheus metrics',
+              'env_vars': ['METRICS_EXPORTER_ENABLED=true']
+          },
+      ]
   }
 
-  listdir = [f for f in os.listdir('k8s')
-             if os.path.isdir(os.path.join('k8s', f))]
-  listdir.sort()
-
-  solutions_to_build = []
-
-  for solution in listdir:
-    if solution in skiplist:
-      print('Skipping solution: ' + solution)
-    else:
-      print('Adding config for solution: ' + solution)
-      solutions_to_build.append(solution)
+  solution = args.solution
+  path = args.output
 
-  cloudbuild_contents = Template(CLOUDBUILD_TEMPLATE).render(
-      solutions=solutions_to_build, extra_configs=extra_configs) + '\n'
+  cloudbuild = CloudBuildConfig(solution=solution)
+  cloudbuild.extra_configs = extra_configs.get(solution, [])
+  cloudbuild.path = path
 
   if args.verify_only:
-    if verify_cloudbuild(cloudbuild_contents):
-      print('The %s file is up-to-date' % CLOUDBUILD_CONFIG)
+    if cloudbuild.verify():
+      print('The %s file is up-to-date' % path)
       os.sys.exit(0)
     else:
-      print('The %s file is not up-to-date. Please re-generate it' %
-            CLOUDBUILD_CONFIG)
+      print('The %s file is not up-to-date. Please re-generate it' % path)
       os.sys.exit(1)
   else:
-    with open(CLOUDBUILD_CONFIG, 'w') as cloudbuild_file:
-      cloudbuild_file.write(cloudbuild_contents)
+    cloudbuild.save()
 
 
 if __name__ == '__main__':