Skip to content

Commit

Permalink
Combine tools.deps and Leiningen images but still provide them under …
Browse files Browse the repository at this point in the history 
…separate tags
  • Loading branch information
@alexander-yakushev
alexander-yakushev committed Apr 24, 2024
1 parent 3ee0739 commit de891a4
Show file tree
Hide file tree
Showing 9 changed files with 232 additions and 327 deletions.
26 changes: 13 additions & 13 deletions src/docker_clojure/config.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,15 @@
(s/def ::distro qualified-keyword?)
(s/def ::distros (s/coll-of ::distro :distinct true :into #{}))

(s/def ::build-tool (s/or ::specific-tool ::non-blank-string
::all-tools #(= ::core/all %)))
(s/def ::build-tool keyword?)
(s/def ::build-tool-version
(s/nilable (s/and ::non-blank-string #(re-matches #"[\d\.]+" %))))
(s/def ::build-tools (s/map-of ::build-tool ::build-tool-version))
(s/def ::installer-hash ::non-blank-string)
(s/def ::version ::build-tool-version)
(s/def ::build-tools (s/map-of ::build-tool
(s/keys :req-un [::version ::installer-hash])))



(s/def ::maintainers
(s/coll-of ::non-blank-string :distinct true :into #{}))
Expand Down Expand Up @@ -71,17 +75,13 @@
17 :ubuntu/jammy
:default :debian/bookworm})

(def build-tools
{"lein" "2.11.2"
"tools-deps" "1.11.1.1435"})

(def default-build-tool "tools-deps")
(def ^:dynamic *build-tools*
{:lein {:version "2.11.2"
:installer-hash "28a1a62668c5f427b413a8677e376affaa995f023b1fcd06e2d4c98ac1df5f3e"}
:tools-deps {:version "1.11.1.1435"
:installer-hash "7edee5b12197a2dbe6338e672b109b18164cde84bea1f049ceceed41fc4dd10a"}})

(def installer-hashes
{"lein" {"2.11.1" "03b3fbf7e6fac262f88f843a87b712a2b37f39cffc4f4f384436a30d8b01d6e4"
"2.11.2" "28a1a62668c5f427b413a8677e376affaa995f023b1fcd06e2d4c98ac1df5f3e"}
"tools-deps" {"1.11.1.1429" "bf08cfeb007118b7277aa7423734f5d507604b868f7fc44c0f9929ca9cd94ed4"
"1.11.1.1435" "7edee5b12197a2dbe6338e672b109b18164cde84bea1f049ceceed41fc4dd10a"}})
(def default-build-tool :tools-deps)

(def exclusions ; don't build these for whatever reason(s)
#{; no more focal builds for JDK 20+
Expand Down
73 changes: 30 additions & 43 deletions src/docker_clojure/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@
m))

(defn variant-map [[base-image jdk-version distro
[build-tool build-tool-version]]]
[build-tool build-tool-info]]]
(let [variant-arch (get cfg/distro-architectures
(-> distro namespace keyword))
base {:jdk-version jdk-version
Expand All @@ -65,34 +65,31 @@
jdk-version distro)
:distro distro
:build-tool build-tool
:build-tool-version build-tool-version
:build-tool-version (:version build-tool-info)
:maintainer (str/join " & " cfg/maintainers)}]
(-> base
(assoc :docker-tag (default-docker-tag base))
(assoc-if #(nil? (:build-tool-version base)) :build-tool-versions
cfg/build-tools)
(assoc-if #(seq variant-arch) :architectures variant-arch))))
(cond-> base
true (assoc :docker-tag (default-docker-tag base))
variant-arch (assoc :architectures variant-arch))))

(defn pull-image [image]
(sh "docker" "pull" image))

(defn generate-dockerfile! [installer-hashes variant]
(defn generate-dockerfile! [variant]
(let [build-dir (df/build-dir variant)
filename "Dockerfile"]
(log "Generating" (str build-dir "/" filename))
(df/write-file build-dir filename installer-hashes variant)
(df/write-file build-dir filename variant)
(assoc variant
:build-dir build-dir
:dockerfile filename)))
:build-dir build-dir
:dockerfile filename)))

(defn build-image
[installer-hashes {:keys [docker-tag base-image architectures] :as variant}]
(defn build-image [{:keys [docker-tag base-image architectures] :as variant}]
(let [image-tag (str "clojure:" docker-tag)
_ (log "Pulling base image" base-image)
_ (pull-image base-image)

{:keys [dockerfile build-dir]}
(generate-dockerfile! installer-hashes variant)
(generate-dockerfile! variant)

host-arch (let [jvm-arch (System/getProperty "os.arch")]
(if (= "aarch64" jvm-arch)
Expand Down Expand Up @@ -126,33 +123,24 @@

(defn image-variant-combinations
[base-images jdk-versions distros build-tools]
(reduce
(fn [variants jdk-version]
(concat
variants
(let [jdk-base-images (get-or-default base-images jdk-version)]
(loop [[bi & r] jdk-base-images
acc #{}]
(let [vs (combo/cartesian-product #{bi}
#{jdk-version}
(get-or-default distros bi)
build-tools)
acc' (concat acc vs)]
(if (seq r)
(recur r acc')
acc'))))))
#{} jdk-versions))
(mapcat
(fn [jdk-version]
(let [jdk-base-images (get-or-default base-images jdk-version)]
(mapcat #(combo/cartesian-product #{%}
#{jdk-version}
(get-or-default distros %)
build-tools)
jdk-base-images)))
jdk-versions))

(defn image-variants
[base-images jdk-versions distros build-tools]
(into #{}
(into []
(comp
(map variant-map)
(remove #(= ::s/invalid (s/conform ::variant %))))
(conj
(image-variant-combinations base-images jdk-versions distros
build-tools)
latest-variant)))
(image-variant-combinations base-images jdk-versions distros
build-tools)))

(defn rand-delay
"Runs argument f w/ any supplied args after a random delay of 100-1000 ms"
Expand All @@ -162,27 +150,26 @@
(apply f args)))

(defn build-images
[parallelization installer-hashes variants]
[parallelization variants]
(log "Building images" parallelization "at a time")
(let [variants-ch (to-chan! variants)
builds-ch (chan parallelization)]
;; Kick off builds with a random delay so we don't have Docker race
;; conditions (e.g. build container name collisions)
(async/thread (pipeline-blocking parallelization builds-ch
(map (partial rand-delay build-image
installer-hashes))
(map (partial rand-delay build-image))
variants-ch))
(while (<!! builds-ch))))

(defn generate-dockerfiles! [installer-hashes variants]
(defn generate-dockerfiles! [variants]
(log "Generated" (count variants) "variants")
(doseq [variant variants]
(generate-dockerfile! installer-hashes variant)))
(generate-dockerfile! variant)))

(defn valid-variants []
(remove (partial exclude? cfg/exclusions)
(image-variants cfg/base-images cfg/jdk-versions cfg/distros
cfg/build-tools)))
cfg/*build-tools*)))

(defn generate-manifest! [variants args]
(let [git-head (->> ["git" "rev-parse" "HEAD"] (apply sh) :out)
Expand Down Expand Up @@ -238,9 +225,9 @@
(let [variants (generate-variants args)]
(case cmd
:clean (df/clean-all)
:dockerfiles (generate-dockerfiles! cfg/installer-hashes variants)
:dockerfiles (generate-dockerfiles! variants)
:manifest (-> variants sort-variants (generate-manifest! args))
:build-images (build-images parallelization cfg/installer-hashes variants)))
:build-images (build-images parallelization variants)))
(logger/stop))

(defn -main
Expand Down
88 changes: 27 additions & 61 deletions src/docker_clojure/dockerfile.clj
View file
Original file line number Diff line number Diff line change
@@ -1,38 +1,17 @@
(ns docker-clojure.dockerfile
(:require
[clojure.java.io :as io]
[clojure.java.shell :refer [sh]]
[clojure.string :as str]
[docker-clojure.dockerfile.lein :as lein]
[docker-clojure.dockerfile.tools-deps :as tools-deps]
[docker-clojure.dockerfile.combined :as combined]
[docker-clojure.dockerfile.shared :refer [copy-resource-file! entrypoint]]))

(defn build-dir [{:keys [base-image-tag jdk-version build-tool]}]
(str/join "/" ["target"
(str (str/replace base-image-tag ":" "-")
(when-not (str/includes? base-image-tag (str jdk-version))
(str "-" jdk-version)))
(if (= :docker-clojure.core/all build-tool)
"latest"
build-tool)]))

(defn all-prereqs [dir variant]
(tools-deps/prereqs dir variant))

(defn all-contents [installer-hashes variant]
(concat
["" "### INSTALL LEIN ###"]
(lein/install
installer-hashes
(assoc variant :build-tool-version
(get-in variant [:build-tool-versions "lein"])))
["" "### INSTALL TOOLS-DEPS ###"]
(tools-deps/install
installer-hashes
(assoc variant :build-tool-version
(get-in variant [:build-tool-versions "tools-deps"])))
[""]
(entrypoint variant)
["" "CMD [\"-M\", \"--repl\"]"]))
(io/file "target"
(str (str/replace base-image-tag ":" "-")
(when-not (str/includes? base-image-tag (str jdk-version))
(str "-" jdk-version)))
(name build-tool)))

(defn copy-java-from-temurin-contents
[{:keys [jdk-version] :as _variant}]
Expand All @@ -41,43 +20,30 @@
"ENV PATH=\"${JAVA_HOME}/bin:${PATH}\""
""])

(defn contents [installer-hashes {:keys [build-tool distro] :as variant}]
(str/join "\n"
(concat [(format "FROM %s" (:base-image-tag variant))
""]
(case (-> distro namespace keyword)
(:debian :debian-slim) (copy-java-from-temurin-contents variant)
[])
(case build-tool
:docker-clojure.core/all (all-contents installer-hashes variant)
"lein" (lein/contents installer-hashes variant)
"tools-deps" (tools-deps/contents installer-hashes variant)))))
(defn contents [{:keys [distro base-image-tag] :as variant}]
(->> (concat [(format "FROM %s" base-image-tag)
""]
(case (-> distro namespace keyword)
(:debian :debian-slim) (copy-java-from-temurin-contents variant)
[])
(combined/contents variant))
(str/join "\n")))

(defn shared-prereqs [dir {:keys [build-tool]}]
(defn do-prereqs [dir {:keys [build-tool] :as variant}]
(let [entrypoint (case build-tool
"tools-deps" "clj"
:docker-clojure.core/all "clj"
build-tool)]
:tools-deps "clj"
:lein "lein")]
(copy-resource-file! dir "entrypoint"
#(str/replace % "@@entrypoint@@" entrypoint)
#(.setExecutable % true false))))

(defn do-prereqs [dir {:keys [build-tool] :as variant}]
(shared-prereqs dir variant)
(case build-tool
:docker-clojure.core/all (all-prereqs dir variant)
"lein" (lein/prereqs dir variant)
"tools-deps" (tools-deps/prereqs dir variant)))

(defn write-file [dir file installer-hashes variant]
(let [{:keys [exit err]} (sh "mkdir" "-p" dir)]
(if (zero? exit)
(do
(do-prereqs dir variant)
(spit (str/join "/" [dir file])
(str (contents installer-hashes variant) "\n")))
(throw (ex-info (str "Error creating directory " dir)
{:error err})))))
#(.setExecutable % true false)))
(copy-resource-file! dir "rlwrap.retry" identity
#(.setExecutable % true false)))

(defn write-file [^java.io.File dir file variant]
(.mkdirs dir)
(do-prereqs dir variant)
(spit (io/file dir file)
(str (contents variant) "\n")))

(defn clean-all []
(sh "sh" "-c" "rm -rf target/*"))
43 changes: 43 additions & 0 deletions src/docker_clojure/dockerfile/combined.clj
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
(ns docker-clojure.dockerfile.combined
(:require
[docker-clojure.dockerfile.lein :as lein]
[docker-clojure.dockerfile.shared :as shared]
[docker-clojure.dockerfile.tools-deps :as tools-deps]))

(def distro-deps
{:debian-slim {:build #{"curl" "gnupg"}
:runtime #{"rlwrap" "make" "git"}}
:debian {:build #{"curl" "gnupg"}
:runtime #{"rlwrap" "make" "git"}}
:ubuntu {:build #{"gnupg"}
;; install curl as a runtime dep b/c we need it at build time
;; but upstream includes it so we don't want to uninstall it
:runtime #{"rlwrap" "make" "git" "curl"}}
:alpine {:build #{"curl" "tar" "gnupg" "openssl" "ca-certificates"}
:runtime #{"bash" "make" "git"}}})

(defn install [variant]
(concat
(lein/env-preamble)
(tools-deps/env-preamble)
[""
"WORKDIR /tmp"
""]
(shared/multiline-RUN
(concat (shared/install-distro-deps distro-deps variant)
(lein/installation-commands)
(tools-deps/installation-commands)
(shared/uninstall-distro-build-deps distro-deps variant)))
[""]
(lein/installation-postamble)
(tools-deps/installation-postamble)))

(defn contents [{:keys [build-tool] :as variant}]
(concat (install variant)
[""]
(shared/entrypoint variant)
(case build-tool
:lein (lein/command variant)
:tools-deps (tools-deps/command variant))))

#_(contents {:build-tool :lein, :build-tool-version "2.11.2", :jdk-version 17})
Loading

0 comments on commit de891a4

Please sign in to comment.