S-111472 Introduce new slim images (#228)

* S-111472 Introduce new slim images - Add new mount points for drivers - Modify wrapper files to include drivers in classpath - Introduce new slim tag which will not have any thirdparty db drivers except postgresql * Build and push slim images * Update commit method to push slim files * S-111472 support slim for Deploy images and remove skip_vulnerable_libs * S-111472 fixing DB and MQ configuration for the Deploy --------- Co-authored-by: Vedran Pugar <[email protected]>
xebialabs · Jul 26, 2024 · b71c7ae · b71c7ae
1 parent 7f8df75
commit b71c7ae
Show file tree

Hide file tree

Showing 13 changed files with 226 additions and 66 deletions.
diff --git a/applejack.py b/applejack.py
@@ -43,7 +43,6 @@ def applejack():
 @applejack.command(help="Render the templates")
 @shared_opts
 @click.option('--commit', '-c', is_flag=True, help="Commit and tag the generated Dockerfiles.")
-@click.option('--skip_vulnerable_libs', '-s', is_flag=True, help="Remove from the image vulnerable libraries.")
 def render(**kwargs):
     renderer = Renderer(kwargs)
     for product in (kwargs['product'] or all_product_configs()):
@@ -75,9 +74,11 @@ def build(**kwargs):
         builder = ImageBuilder(kwargs, product_conf)
         for target_os in (kwargs['target_os'] or target_systems(product_conf)):
             print("Building Docker image for %s %s" % (product_conf['name'], target_os))
-            image_id = builder.build_docker_image(target_os)
+            image_id = builder.build_docker_image(target_os, is_slim=False)
+            slim_image_id = builder.build_docker_image(target_os, is_slim=True)
             if kwargs['push']:
-                builder.push_image(image_id, target_os)
+                builder.push_image(image_id, target_os, is_slim=False)
+                builder.push_image(slim_image_id, target_os, is_slim=True)
 
 
 if __name__ == '__main__':

diff --git a/applejack/builder.py b/applejack/builder.py
@@ -34,13 +34,18 @@ def convert_build_logs(generator):
                 elif "error" in j:
                     raise Exception(j["error"])
 
-    def build_docker_image(self, target_os):
+    def build_docker_image(self, target_os, is_slim):
         client = docker.from_env()
+        if is_slim:
+            docker_file = str(Path(target_os) / "Dockerfile.slim").replace('\\', '/')
+        else:
+            docker_file = str(Path(target_os) / "Dockerfile").replace('\\', '/')
+
         generator = client.api.build(
             nocache=not self.use_cache,
             pull=not self.use_cache,
             path=str(target_path(self.product_conf['name'], self.image_version)),
-            dockerfile=str(Path(target_os) / "Dockerfile").replace('\\', '/'),
+            dockerfile=docker_file,
             rm=True,
         )
         for line in self.convert_build_logs(generator):
@@ -56,6 +61,8 @@ def build_docker_image(self, target_os):
         image = client.images.get(image_id)
         repo = "%s/%s" % (self.registry, self.repository)
         for tag, _ in all_tags(target_os, self.image_version, self.product_conf['dockerfiles']['default']):
+            if is_slim:
+                tag += "-slim"
             print("Tag image with %s:%s" % (repo, tag))
             image.tag(repo, tag)
         image.reload()
@@ -76,13 +83,15 @@ def convert_push_logs(generator):
                 if 'error' in j:
                     raise Exception(j['error'])
 
-    def push_image(self, image_id, target_os):
+    def push_image(self, image_id, target_os, is_slim):
         print("Pushing image with id %s to %s" % (image_id, self.registry))
         client = docker.from_env()
         image = client.images.get(image_id)
         print("image = %s" % image)
         for tag, _ in all_tags(target_os, self.image_version, self.product_conf['dockerfiles']['default']):
             repo = "%s/%s" % (self.registry, self.repository)
+            if is_slim:
+                tag += "-slim"
             for line in self.convert_push_logs(client.images.push(repo, tag=tag, stream=True)):
                 print(line)
             print("Image %s with tag %s has been pushed to %s" % (image_id, tag, repo))
diff --git a/applejack/conf/products/central-configuration.yml b/applejack/conf/products/central-configuration.yml
@@ -27,7 +27,6 @@ context:
   product_description: Enterprise-scale Application Release Automation for any environment
   boot_conf: deployit.conf
   wrapper_conf: xlc-wrapper.conf.common
-  skip_vulnerable_libs: false
   central_config_files:
     - deploy-cluster.yaml
     - deploy-metrics.yaml

diff --git a/applejack/conf/products/deploy-task-engine.yml b/applejack/conf/products/deploy-task-engine.yml
@@ -27,10 +27,11 @@ context:
   product_description: Enterprise-scale Application Release Automation for any environment
   license_file: deployit-license.lic
   wrapper_conf: xld-wrapper.conf.common
-  skip_vulnerable_libs: false
   port: 8180
   volumes:
     - "${APP_HOME}/conf"
+    - "${APP_HOME}/driver/jdbc"
+    - "${APP_HOME}/driver/mq"
     - "${APP_HOME}/ext"
     - "${APP_HOME}/hotfix/lib"
     - "${APP_HOME}/hotfix/plugins"

diff --git a/applejack/conf/products/xl-deploy.yml b/applejack/conf/products/xl-deploy.yml
@@ -28,7 +28,6 @@ context:
   license_file: deployit-license.lic
   boot_conf: deployit.conf
   wrapper_conf: xld-wrapper.conf.common
-  skip_vulnerable_libs: false
   conf_files:
     - xl-deploy.conf
   central_config_files:
@@ -46,6 +45,8 @@ context:
   volumes:
     - "${APP_HOME}/centralConfiguration"
     - "${APP_HOME}/conf"
+    - "${APP_HOME}/driver/jdbc"
+    - "${APP_HOME}/driver/mq"
     - "${APP_HOME}/export"
     - "${APP_HOME}/ext"
     - "${APP_HOME}/hotfix/lib"

diff --git a/applejack/conf/products/xl-release.yml b/applejack/conf/products/xl-release.yml
@@ -26,14 +26,14 @@ context:
   license_file: xl-release-license.lic
   boot_conf: xl-release-server.conf
   wrapper_conf: xlr-wrapper-linux.conf
-  skip_vulnerable_libs: false
   conf_files:
   - xl-release.conf
   port: 5516
   volumes:
     - "${APP_ROOT}/bootstrap"
     - "${APP_HOME}/archive"
     - "${APP_HOME}/conf"
+    - "${APP_HOME}/driver/jdbc"
     - "${APP_HOME}/hotfix"
     - "${APP_HOME}/ext"
     - "${APP_HOME}/plugins"

diff --git a/applejack/renderer.py b/applejack/renderer.py
@@ -11,7 +11,6 @@ def __init__(self, commandline_args):
         self.commit = commandline_args['commit']
         self.registry = commandline_args['registry']
         self.version = commandline_args['xl_version']
-        self.skip_vulnerable_libs = commandline_args['skip_vulnerable_libs']
         self.image_version = image_version(commandline_args['xl_version'], commandline_args['suffix'])
 
     def __render_jinja_template(self, templates_path, template_file, target_file, context):
@@ -29,19 +28,20 @@ def render(self, target_os, product_conf):
 
     def __generate_dockerfile(self, target_os, product_conf):
         target_path = self.__get_target_path(target_os, product_conf['name'])
-        context = self.__build_render_context(product_conf, target_os)
+        context = self.__build_render_context(product_conf, target_os, is_slim=False)
+        slim_context = self.__build_render_context(product_conf, target_os, is_slim=True)
         self.__render_jinja_template(Path('templates') / 'dockerfiles', product_conf['dockerfiles']['os'][target_os], target_path / 'Dockerfile', context)
+        self.__render_jinja_template(Path('templates') / 'dockerfiles', product_conf['dockerfiles']['os'][target_os], target_path / 'Dockerfile.slim', slim_context)
         print("Dockerfile template for '%s' rendered" % target_os)
 
-    def __build_render_context(self, product_conf, target_os):
+    def __build_render_context(self, product_conf, target_os, is_slim):
         context = dict(product_conf['context'])
         context['image_version'] = self.image_version
         context['xl_version'] = self.version
         context['registry'] = self.registry
-        if self.skip_vulnerable_libs:
-            context['skip_vulnerable_libs'] = self.skip_vulnerable_libs
         context['target_os'] = target_os
         context['today'] = datetime.now().strftime('%Y-%m-%d')
+        context['is_slim'] = is_slim
         return context
 
     def __copy_render_resources(self, source_dir, product_conf, target_os):
@@ -60,8 +60,13 @@ def __copy_render_resources(self, source_dir, product_conf, target_os):
             elif p.is_file() and '.j2' in p.suffixes:
                 # Render J2 template
                 render_dest = dest_path / relative.parent / relative.stem
-                context = self.__build_render_context(product_conf, target_os)
+                context = self.__build_render_context(product_conf, target_os, is_slim=False)
                 self.__render_jinja_template(template_path, Path(source_dir) / relative, render_dest, context)
+
+                if relative.parent.name == 'bin':
+                    render_slim_dest = dest_path / relative.parent / (relative.stem + '.slim')
+                    slim_context = self.__build_render_context(product_conf, target_os, is_slim=True)
+                    self.__render_jinja_template(template_path, Path(source_dir) / relative, render_slim_dest, slim_context)
             elif p.is_file():
                 p.copy(dest_path / relative)
 
@@ -88,12 +93,16 @@ def __git_commit_dockerfiles(self):
         print(diff)
         for product_conf in all_product_configs():
             for target_os in target_systems(product_conf):
-                df = str(self.__get_target_path(target_os, product_conf['name']) / "Dockerfile")
-                print("Checking diff for %s" % df)
-                if df in diff:
-                    print("Adding modified %s" % df)
-                    changed = True
-                    repo.index.add([df])
+                dockerfile_paths = [
+                    str(self.__get_target_path(target_os, product_conf['name']) / "Dockerfile"),
+                    str(self.__get_target_path(target_os, product_conf['name']) / "Dockerfile.slim")
+                ]
+                for df in dockerfile_paths:
+                    print("Checking diff for %s" % df)
+                    if df in diff:
+                        print("Adding modified %s" % df)
+                        changed = True
+                        repo.index.add([df])
 
         # If nothing changed, no commit/tag operation is needed.
         if not changed:

diff --git a/templates/dockerfiles/deploy-task-engine/install.j2 b/templates/dockerfiles/deploy-task-engine/install.j2
@@ -7,21 +7,35 @@ COPY resources/{{ product }}-{{ xl_version }}.zip /tmp
 RUN mkdir -p ${APP_ROOT} && \
     unzip /tmp/{{ product }}-{{ xl_version }}.zip -d ${APP_ROOT} && \
     mv ${APP_ROOT}/{{ product }}-{{ xl_version }} ${APP_HOME} && \
-{% if skip_vulnerable_libs %}
-    rm ${APP_HOME}/lib/derby*.jar && \
-{% endif %}
     true
 
+# Create directories for external drivers
+RUN mkdir -p ${APP_HOME}/driver/jdbc && \
+    mkdir -p ${APP_HOME}/driver/mq
+
+{%- if is_slim %}
+# Remove bundled drivers if slim
+RUN rm ${APP_HOME}/lib/derby*.jar
+{%- endif %}
+
 # Add bin/run-in-container.sh
-COPY resources/bin/run-in-container.sh ${APP_HOME}/bin/
+{%- if is_slim %}
+COPY resources/bin/run-in-container.sh.slim ${APP_HOME}/bin/run-in-container.sh
+{%- else %}
+COPY resources/bin/run-in-container.sh ${APP_HOME}/bin/run-in-container.sh
+{%- endif %}
 
 # Add jmx-exporter for prometheus
 COPY resources/jmx-exporter/jmx_prometheus_javaagent.jar ${APP_HOME}/lib/
 
 # Add (and run) Database driver download script
-COPY resources/bin/db-drivers.sh /tmp
-RUN chmod ugo+x /tmp/db-drivers.sh && \
-    /bin/sh /tmp/db-drivers.sh
+{%- if is_slim %}
+COPY resources/bin/drivers.sh.slim /tmp/drivers.sh
+{%- else %}
+COPY resources/bin/drivers.sh /tmp/drivers.sh
+{%- endif %}
+RUN chmod ugo+x /tmp/drivers.sh && \
+    /bin/sh /tmp/drivers.sh
 
 # Modify bin/run.sh so that java becomes a child process of dumb-init
 RUN sed -i 's/^\($JAVACMD\)/exec \1/' ${APP_HOME}/bin/run.sh

diff --git a/templates/dockerfiles/install.j2 b/templates/dockerfiles/install.j2
@@ -9,22 +9,41 @@ RUN mkdir -p ${APP_ROOT} && \
     mv ${APP_ROOT}/{{ product }}-{{ xl_version }}-server ${APP_HOME} && \
     rm -fr ${APP_HOME}/serviceWrapper/ && \
     rm ${APP_HOME}/bin/.wrapper-env* ${APP_HOME}/bin/install-service* ${APP_HOME}/bin/uninstall-service* && \
-{%- if skip_vulnerable_libs %}
-    rm ${APP_HOME}/lib/derby*.jar && \
+    true
+
+# Create directories for external drivers
+RUN mkdir -p ${APP_HOME}/driver/jdbc && \
+    mkdir -p ${APP_HOME}/driver/mq
+
+{%- if is_slim %}
+# Remove bundled drivers if slim
+RUN rm ${APP_HOME}/lib/derby*.jar && \
     rm -fr ${APP_HOME}/derbyns/ && \
-{%- endif %}
+    {% if 'xl-release' in product -%}
+    rm ${APP_HOME}/lib/h2*.jar && \
+    {% endif -%}
     true
+{%- endif %}
 
 # Add bin/run-in-container.sh
-COPY resources/bin/run-in-container.sh ${APP_HOME}/bin/
+{%- if is_slim %}
+COPY resources/bin/run-in-container.sh.slim ${APP_HOME}/bin/run-in-container.sh
+{%- else %}
+COPY resources/bin/run-in-container.sh ${APP_HOME}/bin/run-in-container.sh
+{%- endif %}
 
 # Add jmx-exporter for prometheus
 COPY resources/jmx-exporter/jmx_prometheus_javaagent.jar ${APP_HOME}/lib/
 
 # Add (and run) Database driver download script
-COPY resources/bin/db-drivers.sh /tmp
-RUN chmod ugo+x /tmp/db-drivers.sh && \
-    /bin/sh /tmp/db-drivers.sh && \
+{%- if is_slim %}
+COPY resources/bin/drivers.sh.slim /tmp/drivers.sh
+{%- else %}
+COPY resources/bin/drivers.sh /tmp/drivers.sh
+{%- endif %}
+
+RUN chmod ugo+x /tmp/drivers.sh && \
+    /bin/sh /tmp/drivers.sh && \
     # Modify bin/run.sh so that java becomes a child process of dumb-init
     sed -i 's/^\($JAVACMD\)/exec \1/' ${APP_HOME}/bin/run.sh
 

diff --git a/templates/resources/common/bin/db-drivers.sh → templates/resources/common/bin/drivers.sh.j2 b/templates/resources/common/bin/db-drivers.sh → templates/resources/common/bin/drivers.sh.j2
@@ -1,26 +1,36 @@
 #!/bin/bash
 set -e
 
+POSTGRESQL_VERSION="42.6.1"
+{%- if not is_slim %}
 MYSQL_VERSION="8.1.0"
 H2_VERSION="2.2.224"
-POSTGRESQL_VERSION="42.6.1"
 MSSQL_VERSION="11.2.3.jre17"
+{%- endif %}
 
 echo "Downloading DB drivers to ${APP_ROOT}/db-libs"
 mkdir ${APP_ROOT}/db-libs
 
+curl https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_VERSION}.jar -o ${APP_ROOT}/db-libs/postgresql-${POSTGRESQL_VERSION}.jar -f
+{%- if not is_slim %}
 curl https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/${MYSQL_VERSION}/mysql-connector-j-${MYSQL_VERSION}.jar -o ${APP_ROOT}/db-libs/mysql-connector-j-${MYSQL_VERSION}.jar -f
 curl https://repo1.maven.org/maven2/com/h2database/h2/${H2_VERSION}/h2-${H2_VERSION}.jar -o ${APP_ROOT}/db-libs/h2-${H2_VERSION}.jar -f
-curl https://jdbc.postgresql.org/download/postgresql-${POSTGRESQL_VERSION}.jar -o ${APP_ROOT}/db-libs/postgresql-${POSTGRESQL_VERSION}.jar -f
 curl https://repo1.maven.org/maven2/com/microsoft/sqlserver/mssql-jdbc/${MSSQL_VERSION}/mssql-jdbc-${MSSQL_VERSION}.jar -o ${APP_ROOT}/db-libs/mssql-jdbc-${MSSQL_VERSION}.jar -f
+{%- endif %}
 
+{%- if 'xl-release' not in product %}
 echo "Downloading MQ drivers to ${APP_ROOT}/mq-libs"
 mkdir ${APP_ROOT}/mq-libs
 
 RABBIT_MQ_AMQP_VERSION="5.21.0"
 RABBIT_MQ_JMS_VERSION="3.2.0"
+{%- if not is_slim %}
 ACTIVE_MQ_VERSION="5.18.4"
+{%- endif %}
 
 curl https://repo1.maven.org/maven2/com/rabbitmq/amqp-client/${RABBIT_MQ_AMQP_VERSION}/amqp-client-${RABBIT_MQ_AMQP_VERSION}.jar -o ${APP_ROOT}/mq-libs/amqp-client-${RABBIT_MQ_AMQP_VERSION}.jar -f
 curl https://repo1.maven.org/maven2/com/rabbitmq/jms/rabbitmq-jms/${RABBIT_MQ_JMS_VERSION}/rabbitmq-jms-${RABBIT_MQ_JMS_VERSION}.jar -o ${APP_ROOT}/mq-libs/rabbitmq-jms-${RABBIT_MQ_JMS_VERSION}.jar
+{%- if not is_slim %}
 curl https://repo1.maven.org/maven2/org/apache/activemq/activemq-client-jakarta/${ACTIVE_MQ_VERSION}/activemq-client-jakarta-${ACTIVE_MQ_VERSION}.jar -o ${APP_ROOT}/mq-libs/activemq-client-jakarta-${ACTIVE_MQ_VERSION}.jar
+{%- endif %}
+{%- endif %}