diff --git a/.github/workflows/function-lib-checker.yml b/.github/workflows/function-lib-checker.yml new file mode 100644 index 000000000..010499f6e --- /dev/null +++ b/.github/workflows/function-lib-checker.yml @@ -0,0 +1,45 @@ +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + +--- +name: Build Function Library Checker + +on: + pull_request: + branches: [ master ] + paths: + - 'lib/**' + +env: + TEST_SCRIPT: "lib/tests/functionLibraryTests.sh" + +# Cancel existing runs if user makes another push. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +jobs: + libtests: + permissions: + contents: read + runs-on: ubuntu-latest + name: Run Build Function Library Tests + if: ${{ (github.repository == 'adoptium/temurin-build') || (github.event_name == 'workflow_dispatch') }} + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: "Run Tests" + run: bash "${PWD}/${TEST_SCRIPT}" \ No newline at end of file diff --git a/lib/README.md b/lib/README.md new file mode 100644 index 000000000..6f41c0d26 --- /dev/null +++ b/lib/README.md @@ -0,0 +1,12 @@ +## Build Library + +This folder contains the function library for the build repository. + +This includes a functionLibrary.sh that can be included in your scripts, +giving people the ability to download files, compare shas, etc, without +wasting the time needed to write code tocover all the edge cases +(can the file be downloaded, does it match the sha, etc). + +The tests folder contains testing for the function library, and will be +run against the function library script whenever any file in lib is changed +(see the github action \"function-lib-checker.yml\" for details) \ No newline at end of file diff --git a/lib/functionLibrary.sh b/lib/functionLibrary.sh new file mode 100644 index 000000000..4b8ee5829 --- /dev/null +++ b/lib/functionLibrary.sh @@ -0,0 +1,261 @@ +#!/usr/bin/bash +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + +# This script contains a library of useful functions +# Functions list: +# downloadFile - Downloads a file and optionally checks it against a sha. +# info - echoes any string sent to it, but only if it is enabled first. +# checkFileSha - Checks whether a named file matches a supplied sha. +# doesThisURLExist - Checks if a given URL exists. + +# This simple function takes a string, and echoes it if info-level logging is enabled. +# Info-level logging can be enabled/disabled simply passing "enable/disable" "logging" to it as 2 arguments. +# This is handy for debug messages, as it can simply not be enabled for quiet runs. +enableInfoLoggingBool=1 + +function info() { + if [ $# == 0 ]; then + echo "Warning: The library function \"info\" was called without an argument." + return 1 + elif [[ $# == 1 ]]; then + if [[ $enableInfoLoggingBool == 0 ]]; then + echo $1 + return 0 + fi + else + if [[ "${2}" == "logging" ]]; then + if [[ "${1}" == "enable" ]]; then + enableInfoLoggingBool=0 + elif [[ "${1}" == "disable" ]]; then + enableInfoLoggingBool=1 + fi + return 0 + fi + fi +} + +# This function takes the supplied sha and compares it against the supplied file. +# Example: checkFileSha 123456 /usr/bin/etc/exampleFile +# Caution: I strongly advise against using relative paths. +function checkFileSha() { + info "Checking if a file matches the sha256 checksum. Fails if there is no checksum." + if [ $# != 2 ]; then + echo "Error: checkFileSha() function was not supplied with exactly 2 arguments." + return 1 + fi + + if [[ -z $1 ]]; then + info "No sha256 checksum found." + info "Check declared failed." + return 1 + fi + + if [[ ! -r "${2}" ]]; then + info "The file we're trying to check does not exist: ${2}" + return 1 + fi + + info "Checking if a file matches the checksum." + shaReturnCode=1 + if command -v sha256sum &> /dev/null; then + echo "$1 $2" | sha256sum -c --quiet + shaReturnCode=$? + elif command -v shasum &> /dev/null; then + echo "$1 $2" | shasum -a 256 -c &> /dev/null - + shaReturnCode=$? + else + echo "Error: Neither sha256sum nor shasum is available on this machine." + return 1 + fi + + if [ $shaReturnCode != 0 ]; then + echo "Warning: File ${2} does not match the supplied checksum." + return 1 + else + info "File matches the checksum." + return 0 + fi +} + +# This function checks if a given URL (string argument) exists. +function doesThisURLExist() { + info "Checking if a given URL exists." + if [ $# == 0 ]; then + echo "Error: doesThisURLExist() function was not supplied with a URL." + return 1 + fi + + spiderOutput=1 + if command -v wget &> /dev/null; then + info "Using wget to verify URL exists." + wget --spider -q ${1} 2> /dev/null + spiderOutput=$? + elif command -v curl &> /dev/null; then + info "Using curl to verify URL exists." + curl -I ${1} -s | grep "200 OK" -q + spiderOutput=$? + else + echo "Error: Neither wget nor curl could be found when downloading this file: ${source}" + return 1 + fi + + return $spiderOutput +} + +# This function downloads files +# The accepted arguments are: +# -source (mandatory: a web URL where the file is located) +# -destination (mandatory: an existent folder where the file will be put) +# -filename (optional: the new name of the file post-download) +# -sha (optional: the anticipated sha of the downloaded file) +# -secure (optional: true/false - should this download be automatically failed?) +function downloadFile() { + source="" + destination="" + filename="" + sha="" + secure="false" + + arrayOfArgs=( "$@" ) + x=0 + while [[ ${#arrayOfArgs[@]} -gt $((x+1)) ]]; do + arg="${arrayOfArgs[x]}" + x="$((x+1))" + value="${arrayOfArgs[$x]}" + case "$arg" in + --source | -s ) + source="${value}" + ;; + + --destination | -d ) + destination="${value}" + ;; + + --filename | -f ) + filename="${value}" + ;; + + --sha | -sha ) + sha="${value}" + ;; + + --secure | -secure ) + [[ "${value}" == "true" ]] && secure="true" + ;; + + *) echo >&2 "Invalid downloadFile argument: ${arg} ${value}"; return 1;; + esac + x="$((x+1))" + done + + info "File download requested." + + if [[ "${source}" == "" || "${destination}" == "" ]]; then + echo "Error: function downloadFile requires both a source and a destination." + echo "Source detected: ${source}" + echo "Destination detected: ${destination}" + return 1 + fi + + info "File details: " + info "- source: ${source}" + info "- destination: ${destination}" + info "- file name: ${filename}" + info "- sha256 checksum: ${sha}" + info "- secure: ${secure}" + + if [ -z ${filename} ]; then + filename="${source##*/}" + fi + + [[ ${secure} == "true" ]] && echo "The attempted download of file ${filename} was blocked because secure mode is active." && return 1 + + info "Checking if source exists." + doesThisURLExist "${source}" + [[ $? != 0 ]] && echo "Error: File could not be found at source." && return 1 + info "Source exists." + + info "Checking if destination folder exists." + [ ! -r "${destination}" ] && echo "Error: Destination folder could not be found." && return 1 + + info "Destination folder exists. Checking if file is already present." + if [ -r "${destination}/${filename}" ]; then + info "Warning: File already exists." + checkFileSha "${sha}" "${destination}/${filename}" + if [[ $? != 0 ]]; then + info "Warning: A file was found with the same name, and it does not match the supplied checksum. Removing file." + rm "${destination}/${filename}" + if [ $? != 0 ]; then + echo "Error: Could not remove file." + return 1 + fi + else + info "A file was found with the same name, and it matches the supplied checksum. Skipping download." + return 0 + fi + fi + if [ -r "${destination}/${source##*/}" ]; then + info "Warning: File already exists with the default file name: ${source##*/}" + checkFileSha "${sha}" "${destination}/${source##*/}" + if [[ $? != 0 ]]; then + info "Warning: A file was found with the same name, and it does not match the supplied checksum. Removing file." + rm "${destination}/${source##*/}" + if [ $? != 0 ]; then + echo "Error: Could not remove file." + return 0 + fi + else + info "A file was found with the same name, and it matches the supplied checksum. Skipping download." + mv "${destination}/${source##*/}" "${destination}/${filename}" + return 0 + fi + fi + + info "File not already downloaded. Attempting file download." + if command -v wget &> /dev/null; then + info "Found wget. Using wget to download the file." + wget -q -P "${destination}" "${source}" + elif command -v curl &> /dev/null; then + info "Found curl. Using curl to download the file." + curl -s "${source}" -o "${destination}" + fi + if [ ! -r "${destination}/${filename}" ]; then + mv "${destination}/${source##*/}" "${destination}/${filename}" + fi + + info "File download is complete." + + if [[ -n $sha ]]; then + checkFileSha "${destination}/${filename}" + if [[ $? != 0 ]]; then + echo "Error: Checksum does not match the downloaded file. Removing file." + rm "${destination}/${filename}" + return 1 + fi + fi + + info "File has been downloaded successfully." + + info "Setting file permissions to 770." + chmod 770 "${destination}/${filename}" + if [ $? != 0 ]; then + echo "Error: Chmod has failed. Attempting to remove file." + rm "${destination}/${filename}" + return 1 + fi + info "File permissions set successfully." + info "File download script complete" + + return 0 +} diff --git a/lib/tests/functionLibraryTests.sh b/lib/tests/functionLibraryTests.sh new file mode 100644 index 000000000..7c13e44b9 --- /dev/null +++ b/lib/tests/functionLibraryTests.sh @@ -0,0 +1,156 @@ +#!/usr/bin/bash +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + +# A set of tests for the functionLibrary script + +scriptLocation=$0 +scriptDir="${scriptLocation%/*}" +[[ ! -r "${scriptDir}" || ! "${scriptDir}" =~ .*tests$ ]] && scriptDir="." +[[ ! -r "${scriptDir}/../functionLibrary.sh" ]] && echo "Error: Please launch this script with a full path, or from within the test directory." && exit 1 + +source "${scriptDir}/../functionLibrary.sh" + +sampleFileURL="https://raw.githubusercontent.com/adamfarley/temurin-build/refs/heads/build_scripts_secure_mode/lib/tests" +sampleFileName="sampleFileForTesting.txt" +sampleFileSha="041bef0ff1e6d44a0464a06131d20ea21e47da9359f485f3f59c9bdb92255379" + +successTotal=0 +failureTotal=0 + +# takes the name of the test and a boolean indicating whether it passed. +function testResults() { + if [[ $2 == 0 ]]; then + echo "Success: $1 has passed." + successTotal=$((successTotal+1)) + else + echo "Failure: $1 has failed." + failureTotal=$((failureTotal+1)) + fi +} + +# info +function infoTests(){ + # Does it work when it shouldn't? + [[ "$(info Test)" == "" ]] + testResults "infoTest 1" "$?" + + # Does it work when it should? + info "enable" "logging" + [[ "$(info 123)" == "123" ]] + testResults "infoTest 2" "$?" + + # Clean up + info "disable" "logging" +} + +# checkFileSha +function checkFileShaTests(){ + # Does it work when it should? + checkFileSha "${sampleFileSha}" "${scriptDir}/${sampleFileName}" + testResults "checkFileShaTest 1" "$?" + + # Does it fail when we have the wrong sha? + checkFileSha "12345" "${scriptDir}/${sampleFileName}" &> /dev/null + [[ "$?" != "0" ]] + testResults "checkFileShaTest 2" "$?" +} + +# doesThisURLExist +function doesThisURLExistTests(){ + # Does it pass when it should? + doesThisURLExist "https://adoptium.net/index.html" + testResults "doesThisURLExistTest 1" "$?" + + # Does it fail when it should? + doesThisURLExist "https://thisurlshouldneverexist123456gibberish.com" &> /dev/null + [[ "$?" != "0" ]] + testResults "doesThisURLExistTest 2" "$?" + + # And does it fail when it's not even a URL? + doesThisURLExist "thisnonurlshouldneverexist123456gibberish" &> /dev/null + [[ "$?" != "0" ]] + testResults "doesThisURLExistTest 3" "$?" +} + +# downloadFile +function downloadFileTests() { + workdir="${scriptDir}/tmp_test_work_dir" + # Setup + [[ -r "${workdir}" ]] && echo "Error: Temporary test work directory exists and shouldn't: ${workdir}" && exit 1 + mkdir "${workdir}" + [[ ! -r "${workdir}" ]] && echo "Error: Temporary test work directory could not be created: ${workdir}" && exit 1 + + # Does it pass when it should (no sha)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" + [[ $? == 0 && -r "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 1" "$?" + rm -rf "${workdir}/*" + + # Does it pass when it should (sha)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "${sampleFileSha}" + [[ $? == 0 && -r "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 2" "$?" + rm -rf "${workdir}/*" + + # Does it correctly rename the downloaded file? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "${sampleFileSha}" -f "newfilename" + [[ $? == 0 && -r "${workdir}/newfilename" ]] + testResults "downloadFileTest 3" "$?" + rm -rf "${workdir}/*" + + # Does it fail when it should (no sha, source does not exist)? + downloadFile -s "${sampleFileURL}/thisFileDoesNotExist" -d "${workdir}" &> /dev/null + [[ $? != 0 && ! -r "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 4" "$?" + + # Does it fail when it should (with sha, source does not exist)? + downloadFile -s "${sampleFileURL}/thisFileDoesNotExist" -d "${workdir}" -sha "${sampleFileSha}" &> /dev/null + [[ $? != 0 && ! -r "${workdir}/${sampleFileName}" ]] + testResults "downloadFileTest 5" "$?" + + # Does it fail when it should (with invalid sha, source exists)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -sha "thisisaninvalidsha12345" -f "newfilename" &> /dev/null + [[ $? != 0 && ! -r "${workdir}/newfilename" ]] + testResults "downloadFileTest 6" "$?" + + # Does it fail when it should (secure mode)? + downloadFile -s "${sampleFileURL}/${sampleFileName}" -d "${workdir}" -secure "true" &> /dev/null + [[ $? != 0 && ! -r "${workdir}/newfilename" ]] + testResults "downloadFileTest 7" "$?" + + # Clean up + rm -rf "${workdir}" + [[ $? != 0 ]] && echo "Error: Temporary test work directory could not be deleted." && exit 1 +} + +echo "Test script start." +echo "" + +# Test execution +infoTests +checkFileShaTests +doesThisURLExistTests +downloadFileTests + +echo "" +echo "${successTotal} tests have passed." +echo "${failureTotal} tests have failed." +echo "" +if [[ $failureTotal -eq 0 ]]; then + echo "This test script has passed." + exit 0 +else + echo "This test script has failed." + exit 1 +fi diff --git a/lib/tests/sampleFileForTesting.txt b/lib/tests/sampleFileForTesting.txt new file mode 100644 index 000000000..61e58c609 --- /dev/null +++ b/lib/tests/sampleFileForTesting.txt @@ -0,0 +1,16 @@ +# ******************************************************************************** +# Copyright (c) 2024 Contributors to the Eclipse Foundation +# +# See the NOTICE file(s) with this work for additional +# information regarding copyright ownership. +# +# This program and the accompanying materials are made +# available under the terms of the Apache Software License 2.0 +# which is available at https://www.apache.org/licenses/LICENSE-2.0. +# +# SPDX-License-Identifier: Apache-2.0 +# ******************************************************************************** + +# A sample file which does not change. Used for download and sha256 testing. + +Sample file contents. \ No newline at end of file diff --git a/sbin/common/config_init.sh b/sbin/common/config_init.sh index 1a649a805..b4b9ab194 100755 --- a/sbin/common/config_init.sh +++ b/sbin/common/config_init.sh @@ -63,6 +63,7 @@ DISABLE_ADOPT_BRANCH_SAFETY DOCKER_FILE_PATH DOCKER_SOURCE_VOLUME_NAME ENABLE_SBOM_STRACE +ENABLE_SECURE_MODE FREETYPE FREETYPE_DIRECTORY FREETYPE_FONT_BUILD_TYPE_PARAM @@ -299,6 +300,9 @@ function parseConfigurationArguments() { "--enable-sbom-strace" ) BUILD_CONFIG[ENABLE_SBOM_STRACE]=true;; + "--enable-secure-mode" ) + BUILD_CONFIG[ENABLE_SECURE_MODE]=true;; + "--freetype-dir" | "-f" ) BUILD_CONFIG[FREETYPE_DIRECTORY]="$1"; shift;; @@ -558,6 +562,9 @@ function configDefaults() { BUILD_CONFIG[ENABLE_SBOM_STRACE]="false" + # Set default value to "false", for maximum user convenience. "false" enables potentially-insecure functionality, like the dynamic download of boot JDKs. + BUILD_CONFIG[ENABLE_SECURE_MODE]="false" + # The default behavior of whether we want to create a separate source archive BUILD_CONFIG[CREATE_SOURCE_ARCHIVE]="false"