Build and Release Docker Lite #153

Workflow file for this run

.github/workflows/build-and-release-docker-lite.yml at b46dfd5

	name: Build and Release Docker Lite

	on:
	workflow_call:
	inputs:
	enabled:
	required: true
	type: boolean
	release_tag:
	required: true
	type: string
	unskript_branch:
	required: false
	default: "master"
	type: string
	awesome_branch:
	required: false
	default: "master"
	type: string
	devops_branch:
	required: false
	default: "master"
	type: string
	build_number:
	required: true
	type: string
	latest:
	required: false
	default: true
	type: boolean

	workflow_dispatch:
	inputs:
	enabled:
	description: 'Workflow Enable Flag'
	required: true
	default: false
	type: boolean
	unskript_branch:
	description: 'unSkript Branch name'
	required: true
	default: master
	type: string
	awesome_branch:
	description: 'unSkript submodule awesome Branch name'
	required: true
	default: master
	type: string
	gotty_branch:
	description: 'gotty Branch name'
	required: true
	default: master
	type: string
	devops_branch:
	description: 'Devops Branch name'
	required: false
	default: master
	type: string
	build_number:
	description: 'Docker build number'
	required: true
	type: string
	build_target:
	required: true
	default: 'build-amd64'
	options:
	- build-amd64
	- build-both
	- build-arm64
	type: choice
	latest:
	description: 'Docker Latest tag Branch name'
	default: false
	type: boolean

	concurrency:
	group: ${{ github.ref }}
	cancel-in-progress: true

	env:
	DOCKER_REGISTRY: docker.io
	DOCKER_IMAGE: unskript/awesome-runbooks
	DOCKER_USERNAME: ${{ secrets.DOCKER_USER }}
	DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
	USERNAME: ${{ secrets.BUILD_USER }}
	DOCKER_TARGET: linux/amd64, linux/arm64

	permissions:
	contents: read

	jobs:
	build-unskript:
	runs-on: ubuntu-latest
	if: ${{ inputs.enabled }}
	strategy:
	fail-fast: false

	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
	with:
	egress-policy: audit

	- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0

	- name: Get current date
	id: date
	run: echo "::set-output name=date::$(date +'%Y-%m-%d-%s')"

	- name: Configure APT and Install Dependencies
	run: \|
	# Remove existing lists and clean apt cache
	sudo rm -rf /var/lib/apt/lists/*
	sudo apt-get clean

	# Configure main repository
	echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs) main restricted universe multiverse" \| sudo tee /etc/apt/sources.list
	echo "deb http://archive.ubuntu.com/ubuntu $(lsb_release -cs)-updates main restricted universe multiverse" \| sudo tee -a /etc/apt/sources.list

	# Update package lists
	sudo apt-get update -y

	# Install ODBC packages
	sudo apt-get install -y --no-install-recommends \
	unixodbc-dev \
	unixodbc \
	unixodbc-common \
	libodbcinst2

	# Clean up
	sudo apt-get clean
	sudo rm -rf /var/lib/apt/lists/*

	- name: Set up Python 3.x
	uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
	with:
	python-version: '3.9'

	- name: Install Python dependencies
	run: \|
	pip install shyaml
	sudo apt-get update -y
	sudo apt-get install -y --no-install-recommends unixodbc-dev unixodbc unixodbc-common libodbcinst2
	sudo apt-get clean
	sudo rm -rf /var/lib/apt/lists/*

	# - name: Set up Python 3.x
	# uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
	# with:
	# # Restricting python version to 3.9
	# python-version: '3.9'

	# - name: Install system dependencies
	# run: \|
	# pip install shyaml
	# #sudo apt-get install -y --allow-downgrades unixodbc-dev=2.3.9 unixodbc=2.3.9 odbcinst1debian2=2.3.9 odbcinst=2.3.9
	# sudo apt-get install -y --allow-downgrades unixodbc-common libodbcinst2

	- name: Checkout Code
	run: \|
	wget -O /tmp/pandas-2.0.1.tar.gz https://files.pythonhosted.org/packages/6c/e0/73987b6ecc7246e02ab557240843f93fd5adf45d1355abb458aa1f2a0932/pandas-2.0.1.tar.gz
	sudo pip install /tmp/pandas-2.0.1.tar.gz
	cd $HOME
	git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/unskript.git unskript
	cd unskript
	git checkout ${{ inputs.unskript_branch }}

	# We use the --upgrade-strage only-if-needed and --use-deprecated=legacy-resolver to avoid
	# PIP dependency loop. Since we are asking the git runner to be of ubuntu-latest, it becomes
	# a moving target for us hence we want to restrict the packages that are needed for unskript
	# to compile to be fixed.
	/usr/bin/env python -m pip install -r ./requirements.txt --upgrade --upgrade-strategy only-if-needed --use-deprecated=legacy-resolver
	/usr/bin/env python -m pip install --upgrade protobuf
	# We need to restrict URLLIB3 to 1.26.6 because Version 2.x.y onwards the DEFAULT_CIPHERS
	# variable is deprecrated. Unfortunately our boto3 and botocore that is needed for our
	# unskript package does not work with the latest version of URLLIB3. Hence we need to
	# restrict it to a fixed version of 1.26.6
	/usr/bin/env python -m pip install --upgrade urllib3==1.26.6
	/usr/bin/env python -m pip install --upgrade types-urllib3==1.26.13
	/usr/bin/env python -m pip install google-api-python-client==2.77.0
	/usr/bin/env python -m pip install --upgrade numpy==1.23.4
	make awesome-submodule
	cd awesome
	git checkout ${{ inputs.awesome_branch }}
	cd ..
	make legoschema
	[ -f "setup-full.py" ] && cp "setup-full.py" ./setup.py
	/usr/bin/env python ./setup.py bdist_wheel
	mv dist/code*tar /tmp
	mv dist/unskript-0.1.0-py2.py3-none-any.whl /tmp

	- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
	with:
	name: schema-${{ github.run_id }}
	path: /tmp/code_snippet_schemas.tar

	- uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
	with:
	name: unskript-${{ github.run_id }}
	path: /tmp/unskript-0.1.0-py2.py3-none-any.whl

	build-gotty:
	runs-on: "ubuntu-latest"
	if: ${{ inputs.enabled }}
	strategy:
	fail-fast: false
	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
	with:
	egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

	- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0

	- name: Set up Go
	uses: actions/setup-go@v3
	with:
	go-version: 1.21

	- name: Checkout Code
	run: \|
	cd $HOME
	pwd
	git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/gotty.git gotty

	- name: "Build & test"
	run: \|
	cd $HOME/gotty
	git checkout ${{ inputs.gotty_branch }}
	make tools test release-artifacts
	ls -l $HOME/gotty/builds/pkg
	mkdir -p /tmp/linux_amd64
	mkdir -p /tmp/linux_arm64
	mv $HOME/gotty/builds/pkg/linux_amd64/gotty /tmp/linux_amd64
	mv $HOME/gotty/builds/pkg/linux_arm64/gotty /tmp/linux_arm64
	ls -l /tmp

	- name: Upload build artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
	with:
	name: gotty-linux-amd64-${{ github.run_id }}
	path: /tmp/linux_amd64/gotty
	- name: Upload build artifacts
	uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
	with:
	name: gotty-linux-arm64-${{ github.run_id }}
	path: /tmp/linux_arm64/gotty

	build-docker:
	runs-on: ubuntu-latest
	if: ${{ inputs.enabled }}
	needs: [build-unskript, build-gotty]
	strategy:
	fail-fast: false

	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
	with:
	egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

	- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0

	# - name: Update Spy2 to workaround for repo timeout issue
	# run: \|
	# sudo gem install apt-spy2
	# sudo apt-spy2 fix --commit --launchpad --country=US
	# sudo apt-get update


	- name: Set up Python 3.x
	uses: actions/setup-python@75f3110429a8c05be0e1bf360334e4cced2b63fa # v2.3.3
	with:
	python-version: '3.9'

	- name: Install system dependencies
	run: \|
	sudo apt update --fix-missing
	pip install shyaml

	- name: Setup Docker Buildx
	uses: crazy-max/ghaction-docker-buildx@126d331dc69f4a1aa02452e374835e6a5d565613 # v3.3.1
	with:
	version: latest
	config: .github/buildkitd.toml

	- name: Prepare Docker Buildx
	id: prepare
	run: \|
	echo ::set-output name=docker_platform::${DOCKER_TARGET}
	echo ::set-output name=docker_image::${DOCKER_REGISTRY}/${DOCKER_IMAGE}
	echo ::set-output name=version::${GITHUB_RUN_NUMBER}

	- name: Docker Login
	run: \|
	echo "${DOCKER_PASSWORD}" \| docker login --username "${DOCKER_USERNAME}" --password-stdin


	- name: Checkout Code
	run: \|
	cd $HOME
	pwd
	git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/devops.git devops


	- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
	with:
	name: schema-${{ github.run_id }}
	path: /tmp/unskript

	- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
	with:
	name: unskript-${{ github.run_id }}
	path: /tmp/unskript
	- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
	with:
	name: gotty-linux-amd64-${{ github.run_id }}
	path: /tmp/gotty/linux_amd64
	- uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
	with:
	name: gotty-linux-arm64-${{ github.run_id }}
	path: /tmp/gotty/linux_arm64

	- name: Prepare to Build
	run: \|
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite

	ls -l /tmp/unskript/
	tar xf /tmp/unskript/code_snippet_schemas.tar
	cd downloads
	mv /tmp/unskript/*.whl .
	mkdir -p gotty/linux_amd64
	mkdir -p gotty/linux_arm64
	ls -l /tmp/gotty
	ls -l /tmp/gotty/linux_amd64
	mv /tmp/gotty/linux_amd64/gotty gotty/linux_amd64
	mv /tmp/gotty/linux_arm64/gotty gotty/linux_arm64
	if [[ ${{ inputs.latest }} == "true" ]]; then
	bt="${{ env.DOCKER_IMAGE }}:minimal-${{ inputs.build_number }} -t ${{ env.DOCKER_IMAGE }}:minimal-latest"
	else
	bt="${{ env.DOCKER_IMAGE }}:minimal-${{ inputs.build_number }}"
	fi
	echo "BUILD_TAGS=$bt" >> $GITHUB_ENV

	- uses: geekyeggo/delete-artifact@54ab544f12cdb7b71613a16a2b5a37a9ade990af # v2.0.0
	with:
	name: \|
	unskript-${{ github.run_id }}
	schema-${{ github.run_id }}
	gotty-linux-amd64-${{ github.run_id }}
	gotty-linux-arm64-${{ github.run_id }}
	- name: Copy unskript-ctl files
	run: \|
	# docker buildx create --name mybuilder
	# docker buildx use mybuilder
	# docker buildx inspect --bootstrap
	# docker buildx ls
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite
	git checkout ${{ inputs.devops_branch }}
	export BUILD_NUMBER=${{ inputs.build_number }}
	make copy
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	git clone https://${{ env.USERNAME }}:${{ secrets.BUILDER_PAT }}@github.com/unskript/unskript.git unskript
	cd unskript
	git checkout ${{ inputs.unskript_branch }}
	make awesome-submodule
	cd awesome
	git checkout ${{ inputs.awesome_branch }}
	cd ..
	make syncrunbooks
	#sed -i "s/BUILD_NUMBER = .*/BUILD_NUMBER = \"${{ inputs.build_number }}\"/g" awesome/unskript-ctl/unskript_ctl_version.py
	# Lets ADD BUILD_NUMBER TO THE version file
	echo "BUILD_NUMBER = \"${{ inputs.build_number }}\"" >> awesome/unskript-ctl/unskript_ctl_version.py
	cp -Rf awesome/unskript-ctl/* $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	cp -Rf awesome/bin/* $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	# docker buildx build --platform linux/amd64,linux/arm64 --push -t ${{ env.BUILD_TAGS }} .

	- name: Docker Build & Push
	run: \|
	if [ "${{ inputs.build_target }}" = "build-amd64" ]; then
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	docker buildx build --cache-from type=gha --cache-to type=gha,mode=max --platform linux/amd64 --push -t ${{ env.BUILD_TAGS }} .
	elif [ "${{ inputs.build_target }}" = "build-arm64" ]; then
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	docker buildx build --cache-from type=gha --cache-to type=gha,mode=max --platform linux/arm64 --push -t ${{ env.BUILD_TAGS }} .
	elif [ "${{ inputs.build_target }}" = "build-both" ]; then
	cd $HOME/devops/dockers/jupyterlab/oss_docker_lite/
	docker buildx build --cache-from type=gha --cache-to type=gha,mode=max --platform linux/amd64,linux/arm64 --push -t ${{ env.BUILD_TAGS }} .
	fi
	- name: Docker Scout
	id: docker-scout
	uses: docker/scout-action@v1
	with:
	command: cves
	image: ${{ env.BUILD_TAGS }}
	only-severities: critical,high
	exit-code: true

	- name: Validate result of Docker Scout
	if: failure()
	run: \|
	sudo apt install -y jq curl
	TOKEN=$(curl -s -H "Content-Type: application/json" -X POST -d '{"username": "${{ env.DOCKER_USERNAME }}", "password": "${{ env.DOCKER_PASSWORD }}"}' https://hub.docker.com/v2/users/login/ \| jq -r .token)
	REPO_LIST=$(curl -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/unskript/?page_size=200 \| jq -r '.results \| if . then .[] \| .name else empty end')
	for i in ${REPO_LIST}
	do
	curl -X DELETE -s -H "Authorization: JWT ${TOKEN}" https://hub.docker.com/v2/repositories/unskript/${i}/tags/${{ inputs.build_number }}/
	echo "DELETED IMAGE THAT FAILED DOCKER SCOUT TEST"
	done

	cleanup:
	runs-on: ubuntu-latest
	if: ${{ inputs.enabled }}
	needs: [build-docker]
	strategy:
	fail-fast: false

	steps:
	- name: Harden Runner
	uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1
	with:
	egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs

	- uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.0
	- name: Install system dependencies
	run: \|
	pip install shyaml


	- uses: geekyeggo/delete-artifact@54ab544f12cdb7b71613a16a2b5a37a9ade990af # v2.0.0
	with:
	name: \|
	unskript-${{ github.run_id }}
	schema-${{ github.run_id }}
	gotty-linux-amd64-${{ github.run_id }}
	gotty-linux-arm64-${{ github.run_id }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Build and Release Docker Lite #153

Workflow file

Build and Release Docker Lite #153

Jobs

Run details

Workflow file for this run