feat!: adapted cmak2zk

.editorconfig

@@ -0,0 +1,16 @@
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+insert_final_newline = true
+trim_trailing_whitespace = true
+indent_style = space
+indent_size = 2
+
+[*.py]
+indent_size = 4
+
+[*.md]
+max_line_length = 120
+

.github/workflows/build.yaml

@@ -4,10 +4,10 @@ on:
     paths-ignore:
       - "examples/*.*"
       - "*.md"
-    branches: 
+    branches:
       - "master"
   pull_request:
-    branches: 
+    branches:
       - "master"
 jobs:
   build_job:
@@ -26,6 +26,5 @@ jobs:
         with:
           version: v1.38.0
       - uses: extractions/setup-just@v1
-      - run: | 
-          just build
+      - run: skaffold build -d localhost:5000
 

.github/workflows/release.yaml

@@ -12,6 +12,7 @@ jobs:
         with:
           version: v1.38.0
       - uses: extractions/setup-just@v1
+      - uses: imjasonh/[email protected]
       - uses: docker/login-action@v1
         with:
           registry: ghcr.io

.releaserc.yaml

@@ -19,4 +19,4 @@ plugins:
       failComment: false
   -
     - "@semantic-release/exec"
-    - publishCmd: "just publish ${nextRelease.version}"
+    - publishCmd: "skaffold build"

README.md

@@ -1,4 +1,4 @@
-# cmak-docker (CMAK)
+# CMAK (prev. Kafka Manager) and cmak2zk docker images
 
 ## Usage
 
@@ -31,7 +31,7 @@ services:
 To quickly launch the file above, execute
 
 ```bash
-$ curl -sL https://raw.githubusercontent.com/hleb-albau/kafka-manager-docker/master/examples/docker-compose-sample.yaml | \
+$ curl -sL https://raw.githubusercontent.com/eshepelyuk/cmak-docker/master/examples/docker-compose-sample.yaml | \
     docker-compose -f - up
 ```
 
@@ -68,18 +68,46 @@ services:
 To quickly launch the file above, execute
 
 ```bash
-$ curl -sL https://raw.githubusercontent.com/hleb-albau/kafka-manager-docker/master/examples/docker-compose-override.yaml | \
+$ curl -sL https://raw.githubusercontent.com/eshepelyuk/cmak-docker/master/examples/docker-compose-override.yaml | \
     docker-compose -f - up
 ```
 
-### Kafka clusters configuration
+### Kafka clusters configuration with cmak2zk
 
 CMAK doesn't provide tools to preconfigure managed Kafka clusters from files.
 It could be done either via HTTP API or via CMAK UI in browser.
 This could be inconvenient for declarative configuration or GitOps flow.
 
-To overcome the issue there exist standalone `cmak2zk` tool.
-The information and usage examples could be found at [cmak2zk homepage](https://github.com/eshepelyuk/cmak-operator#standalone-cmak2zk-tool).
+Its purpose is to take Kafka cluster configuration for CMAK in YAML format
+and populate CMAK compatible config in Zookeeper.
+This allows to avoid manual configuration of CMAK and provides better possibilities
+to use CMAK in declarative configuration.
+
+`cmak2zk` is distributed as [docker image](https://github.com/users/eshepelyuk/packages/container/package/dckr%2Fcmak2zk).
+
+To check out available options, run the image without parameters.
+
+```sh
+docker run ghcr.io/eshepelyuk/dckr/cmak2zk:latest
+```
+
+Example `docker-compose` and Kafka cluster configuration are located at
+[cmak2zk/examples](https://github.com/eshepelyuk/cmak-docker/tree/master/examples) directory.
+One could run them using commands below.
+
+```sh
+curl -sLo clusters.yaml \
+  https://raw.githubusercontent.com/eshepelyuk/cmak-docker/master/examples/clusters.yaml
+
+curl -sLo docker-compose-cmak2zk.yaml \
+  https://raw.githubusercontent.com/eshepelyuk/cmak-docker/master/examples/docker-compose-cmak2zk.yaml
+
+docker-compose -f docker-compose-cmak2zk.yaml up
+```
+
+Wait for some time until components are stabilizing, it may take up to 5 mins.
+Then, open your browser at http://localhost:9000.
+There should be two pre-configured clusters, pointing to the same Kafka instance, running in Docker.
 
 ## Usage in Kubernetes
 

cmak2zk/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8-alpine3.12
+
+COPY *.py /app/bin/
+
+RUN set -x && pip install click~=7.0 kazoo~=2.8 ruamel.yaml~=0.16 jsonmerge~=1.8
+
+ENTRYPOINT ["python3", "/app/bin/cmak2zk.py"]
+CMD ["--help"]
+
+WORKDIR /app/

cmak2zk/cmak2zk.py

@@ -0,0 +1,63 @@
+import click
+import logging
+from kazoo.client import KazooClient
+from pathlib import Path
+from hashlib import md5
+from ruamel.yaml import YAML
+from jsonmerge import merge
+import json
+
+FORMAT = '%(asctime)-15s %(message)s'
+logging.basicConfig(format=FORMAT, level=logging.INFO)
+
+ZK_ROOT="/kafka-manager/configs"
+
+yaml = YAML(typ='safe')
+
+
+@click.command()
+@click.option('--overwrite-zk/--no-overwrite-zk', "over_zk", default=True, help="should existing ZK data be rewritten on md5 difference")
+@click.argument("zk_url", type=str)
+@click.argument('yaml_cfg', type=click.File())
+def cmak2zk(over_zk, zk_url, yaml_cfg):
+    """
+    Populates Zookeeper at ZK_URL with Kafka cluster configuration
+    in CMAK compatible format from YAML_CFG configuration file.
+
+    ZK_URL - myzk1:2181,myzk2:2181, etc.
+
+    \b
+    YAML_CFG - Format is equal to CMAK operator Helm chart values.
+    The only section used is
+    https://artifacthub.io/packages/helm/cmak-operator/cmak-operator?modal=values-schema&path=cmak.
+
+    """
+    cmak_cfg = yaml.load(yaml_cfg)['cmak']
+    common_cfg = cmak_cfg['clustersCommon']
+
+    zk = KazooClient(hosts=zk_url)
+    zk.start()
+
+    for cl in cmak_cfg['clusters']:
+        cl = merge(common_cfg, cl)
+        dst = f"{ZK_ROOT}/{cl['name']}"
+        json_b = json.dumps(cl, separators=(',', ':')).encode()
+
+        if zk.exists(dst):
+            file_md5 = md5(json_b).hexdigest()
+
+            zk_b, stat = zk.get(dst)
+            zk_md5 = md5(zk_b).hexdigest()
+            logging.info(f"md5 of {dst}: {zk_md5}, md5 of {cl['name']}: {file_md5}")
+
+            if zk_md5 != file_md5 and over_zk is True:
+                zk.set(dst, json_b)
+                logging.info(f"Overwritten {dst} from {yaml_cfg.name}")
+        else:
+            zk.create(dst, json_b, makepath=True)
+            logging.info(f"Created {dst} from {yaml_cfg.name}")
+
+    zk.stop()
+
+if __name__ == "__main__":
+    cmak2zk()

examples/clusters.yaml

@@ -0,0 +1,48 @@
+cmak:
+  clustersCommon:
+    curatorConfig:
+      zkMaxRetry: 100
+      baseSleepTimeMs: 100
+      maxSleepTimeMs: 1000
+    enabled: true
+    kafkaVersion: "2.4.0"
+    jmxEnabled: false
+    jmxUser: null
+    jmxPass: null
+    jmxSsl: false
+    pollConsumers: true
+    filterConsumers: false
+    logkafkaEnabled: false
+    activeOffsetCacheEnabled: true
+    displaySizeEnabled: false
+    tuning:
+      brokerViewUpdatePeriodSeconds: 30
+      clusterManagerThreadPoolSize: 10
+      clusterManagerThreadPoolQueueSize: 100
+      kafkaCommandThreadPoolSize: 10
+      kafkaCommandThreadPoolQueueSize: 100
+      logkafkaCommandThreadPoolSize: 10
+      logkafkaCommandThreadPoolQueueSize: 100
+      logkafkaUpdatePeriodSeconds: 30
+      partitionOffsetCacheTimeoutSecs: 5
+      brokerViewThreadPoolSize: 10
+      brokerViewThreadPoolQueueSize: 1000
+      offsetCacheThreadPoolSize: 10
+      offsetCacheThreadPoolQueueSize: 1000
+      kafkaAdminClientThreadPoolSize: 10
+      kafkaAdminClientThreadPoolQueueSize: 1000
+      kafkaManagedOffsetMetadataCheckMillis: 30000
+      kafkaManagedOffsetGroupCacheSize: 1000000
+      kafkaManagedOffsetGroupExpireDays: 7
+    securityProtocol: PLAINTEXT
+    saslMechanism: null
+    jaasConfig: null
+
+  clusters:
+    - name: test
+      curatorConfig:
+        zkConnect: zk:2181
+    - name: test-disabled
+      enabled: false
+      curatorConfig:
+        zkConnect: zk:2181

examples/docker-compose-cmak2zk.yaml

@@ -0,0 +1,42 @@
+version: '3.6'
+services:
+  zk:
+    image: zookeeper:latest
+    restart: always
+    environment:
+      ZOO_SERVERS: server.1=0.0.0.0:2888:3888;2181
+
+  cmak:
+    image: ghcr.io/eshepelyuk/dckr/cmak-3.0.0.5:latest
+    restart: always
+    ports:
+      - "9000:9000"
+    depends_on:
+      - "zk"
+    environment:
+      ZK_HOSTS: "zk:2181"
+
+  cmak2zk:
+    image: ghcr.io/eshepelyuk/dckr/cmak2zk:latest
+    restart: on-failure
+    command:
+      - 'zk:2181'
+      - '/app/etc/clusters.yaml'
+    depends_on:
+      - "zk"
+    volumes:
+      - "${PWD}/clusters.yaml:/app/etc/clusters.yaml:ro"
+
+###############################################################
+#     This Kafka setup is only for demonstration purpose.     #
+###############################################################
+  kafka:
+    image: wurstmeister/kafka:2.12-2.4.1
+    restart: always
+    depends_on:
+      - "zk"
+    environment:
+      KAFKA_ADVERTISED_HOST_NAME: "kafka"
+      KAFKA_CREATE_TOPICS: "test:1:1"
+      KAFKA_ZOOKEEPER_CONNECT: "zk:2181"
+###############################################################

justfile

@@ -1,20 +1,9 @@
-skaffoldTags := "tags.json"
-
-# list available receipes
+# list available recipes
 default:
-    just --list
-
-build:
-    skaffold build -d localhost:5000
-
-publish version:
-    #!/usr/bin/env bash
-    set -euxo pipefail
-
-    skaffold build -t {{version}} --file-output={{skaffoldTags}}
-
-    LATEST="$(jq -r .builds[0].imageName {{skaffoldTags}}):latest"
-    CURRENT="$(jq -r .builds[0].tag {{skaffoldTags}})"
+  just --list
 
-    docker tag $CURRENT $LATEST
-    docker push $LATEST
+_latest:
+  #!/bin/sh
+  if [ -n "$(echo ${SKAFFOLD_IMAGE}|grep '^ghcr')" ]; then
+    crane tag ${SKAFFOLD_IMAGE} latest
+  fi

skaffold.yaml

@@ -4,8 +4,16 @@ metadata:
   name: cmak-docker
 build:
   artifacts:
+    - image: ghcr.io/eshepelyuk/dckr/cmak2zk
+      context: ./cmak2zk
+      hooks:
+        after:
+          - command: ["sh", "-c", "just _latest"]
     - image: ghcr.io/eshepelyuk/dckr/cmak-3.0.0.5
-      context: .
+      context: ./cmak
+      hooks:
+        after:
+          - command: ["sh", "-c", "just _latest"]
       docker:
         buildArgs:
           CMAK_VERSION: "3.0.0.5"