diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile new file mode 100644 index 00000000000..44b1d455df1 --- /dev/null +++ b/dev-support/docker/Dockerfile @@ -0,0 +1,155 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Dockerfile for installing the necessary dependencies for building Drill. +# See BUILDING.txt. + +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +WORKDIR /root + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +##### +# Disable suggests/recommends +##### +RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras +RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras + +ENV DEBIAN_FRONTEND noninteractive +ENV DEBCONF_TERSE true + + +# hadolint ignore=DL3008 +RUN apt -q update \ + && apt install -y software-properties-common apt-utils apt-transport-https \ + # Repo for different Python versions + && add-apt-repository -y ppa:deadsnakes/ppa \ + && apt-get -q install -y --no-install-recommends \ + ant \ + bats \ + bash-completion \ + build-essential \ + bzip2 \ + ca-certificates \ + clang \ + cmake \ + curl \ + docker.io \ + doxygen \ + findbugs \ + fuse \ + g++ \ + gcc \ + git \ + gnupg-agent \ + libaio1 \ + libbcprov-java \ + libbz2-dev \ + libcurl4-openssl-dev \ + libfuse-dev \ + libnuma-dev \ + libncurses5 \ + libprotobuf-dev \ + libprotoc-dev \ + libsasl2-dev \ + libsnappy-dev \ + libssl-dev \ + libtool \ + libzstd-dev \ + locales \ + make \ + maven \ +# openjdk-11-jdk \ + openjdk-8-jdk \ + pinentry-curses \ + pkg-config \ + python \ + python2.7 \ +# python-pip \ + python-pkg-resources \ + python-setuptools \ +# python-wheel \ + python3-setuptools \ + python3-pip \ + python3.5 \ + python3.6 \ + python3.7 \ + python2.7 \ + virtualenv \ + tox \ + rsync \ + shellcheck \ + software-properties-common \ + sudo \ + valgrind \ + vim \ + wget \ + zlib1g-dev \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +### +# Install grpcio-tools mypy-protobuf for `python3 sdks/python/setup.py sdist` to work +### +RUN pip3 install grpcio-tools mypy-protobuf + +### +# Install Go +### +RUN mkdir -p /goroot \ + && curl https://dl.google.com/go/go1.15.2.linux-amd64.tar.gz | tar xvzf - -C /goroot --strip-components=1 \ + && chmod a+rwX -R /goroot + +# Set environment variables for Go +ENV GOROOT /goroot +ENV GOPATH /gopath +ENV PATH $GOROOT/bin:$GOPATH/bin:$PATH +CMD go get github.com/linkedin/goavro + +### +# Miscelaneous fixes... +### +# Turns out some build tools use 'time' and in this docker image this is no longer +# an executable but ONLY an internal command of bash +COPY time.sh /usr/bin/time +RUN chmod 755 /usr/bin/time + +# Force the complete use of Java 8 +RUN apt remove -y openjdk-11-jre openjdk-11-jre-headless +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/ + + +### +# Avoid out of memory errors in builds +### +ENV MAVEN_OPTS -Xmx4g -XX:MaxPermSize=512m + +### +# Add a welcome message and environment checks. +### +RUN mkdir /scripts +COPY drill_env_checks.sh /scripts/drill_env_checks.sh +COPY bashcolors.sh /scripts/bashcolors.sh +RUN chmod 755 /scripts /scripts/drill_env_checks.sh /scripts/bashcolors.sh + +# hadolint ignore=SC2016 +RUN echo '. /etc/bash_completion' >> /root/.bash_aliases +RUN echo '. /scripts/drill_env_checks.sh' >> /root/.bash_aliases diff --git a/dev-support/docker/bashcolors.sh b/dev-support/docker/bashcolors.sh new file mode 100755 index 00000000000..0f0e05cab19 --- /dev/null +++ b/dev-support/docker/bashcolors.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Based upon info on https://wiki.archlinux.org/index.php/Color_Bash_Prompt + +# Reset +export Color_Off='\e[0m' # Text Reset + +# Regular Colors +export Black='\e[0;30m' # Black +export Red='\e[0;31m' # Red +export Green='\e[0;32m' # Green +export Yellow='\e[0;33m' # Yellow +export Blue='\e[0;34m' # Blue +export Purple='\e[0;35m' # Purple +export Cyan='\e[0;36m' # Cyan +export White='\e[0;37m' # White + +# Bold +export BBlack='\e[1;30m' # Black +export BRed='\e[1;31m' # Red +export BGreen='\e[1;32m' # Green +export BYellow='\e[1;33m' # Yellow +export BBlue='\e[1;34m' # Blue +export BPurple='\e[1;35m' # Purple +export BCyan='\e[1;36m' # Cyan +export BWhite='\e[1;37m' # White + +# Underline +export UBlack='\e[4;30m' # Black +export URed='\e[4;31m' # Red +export UGreen='\e[4;32m' # Green +export UYellow='\e[4;33m' # Yellow +export UBlue='\e[4;34m' # Blue +export UPurple='\e[4;35m' # Purple +export UCyan='\e[4;36m' # Cyan +export UWhite='\e[4;37m' # White + +# Background +export On_Black='\e[40m' # Black +export On_Red='\e[41m' # Red +export On_Green='\e[42m' # Green +export On_Yellow='\e[43m' # Yellow +export On_Blue='\e[44m' # Blue +export On_Purple='\e[45m' # Purple +export On_Cyan='\e[46m' # Cyan +export On_White='\e[47m' # White + +# High Intensity +export IBlack='\e[0;90m' # Black +export IRed='\e[0;91m' # Red +export IGreen='\e[0;92m' # Green +export IYellow='\e[0;93m' # Yellow +export IBlue='\e[0;94m' # Blue +export IPurple='\e[0;95m' # Purple +export ICyan='\e[0;96m' # Cyan +export IWhite='\e[0;97m' # White + +# Bold High Intensity +export BIBlack='\e[1;90m' # Black +export BIRed='\e[1;91m' # Red +export BIGreen='\e[1;92m' # Green +export BIYellow='\e[1;93m' # Yellow +export BIBlue='\e[1;94m' # Blue +export BIPurple='\e[1;95m' # Purple +export BICyan='\e[1;96m' # Cyan +export BIWhite='\e[1;97m' # White + +# High Intensity backgrounds +export On_IBlack='\e[0;100m' # Black +export On_IRed='\e[0;101m' # Red +export On_IGreen='\e[0;102m' # Green +export On_IYellow='\e[0;103m' # Yellow +export On_IBlue='\e[0;104m' # Blue +export On_IPurple='\e[0;105m' # Purple +export On_ICyan='\e[0;106m' # Cyan +export On_IWhite='\e[0;107m' # White diff --git a/dev-support/docker/drill_env_checks.sh b/dev-support/docker/drill_env_checks.sh new file mode 100755 index 00000000000..b92182f262f --- /dev/null +++ b/dev-support/docker/drill_env_checks.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# SHELLDOC-IGNORE + +# ------------------------------------------------------- +function showWelcome { +# http://patorjk.com/software/taag/#p=display&f=Doom&t=Drill%20Build%20Env. +cat << "Welcome-message" + +______ _ _ _ ______ _ _ _ _____ +| _ \ (_) | | | ___ \ (_) | | | | ___| +| | | |_ __ _| | | | |_/ /_ _ _| | __| | | |__ _ ____ __ +| | | | '__| | | | | ___ \ | | | | |/ _` | | __| '_ \ \ / / +| |/ /| | | | | | | |_/ / |_| | | | (_| | | |__| | | \ V / +|___/ |_| |_|_|_| \____/ \__,_|_|_|\__,_| \____/_| |_|\_(_) + +This is the standard Drill Developer build environment. +This has all the right tools installed required to build +Apache Drill from source. + +Welcome-message +} + +# ------------------------------------------------------- + +function showAbort { + cat << "Abort-message" + + ___ _ _ _ + / _ \| | | | (_) +/ /_\ \ |__ ___ _ __| |_ _ _ __ __ _ +| _ | '_ \ / _ \| '__| __| | '_ \ / _\` | +| | | | |_) | (_) | | | |_| | | | | (_| | +\_| |_/_.__/ \___/|_| \__|_|_| |_|\__, | + __/ | + |___/ + +Abort-message +} + +# ------------------------------------------------------- + +function failIfUserIsRoot { + if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong. + then + cat </dev/null && __git_ps1 " \['${BIPurple}'\]{\['${BIGreen}'\]%s\['${BIPurple}'\]}")\['${BIBlue}'\] ]\['${Color_Off}'\]\n$ ' diff --git a/dev-support/docker/time.sh b/dev-support/docker/time.sh new file mode 100644 index 00000000000..94af17234bc --- /dev/null +++ b/dev-support/docker/time.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +time "$@" diff --git a/docs/dev/Environment.md b/docs/dev/Environment.md index 47d37c138ac..46600826ac8 100644 --- a/docs/dev/Environment.md +++ b/docs/dev/Environment.md @@ -6,6 +6,14 @@ Currently, the Apache Drill build process is known to work on Linux, Windows and * Java 8 * Maven 3.6.3 or greater +## Docker based build environment + +The `start-build-env.sh` script in the root of the project source builds and starts a preconfigured environment +that contains all the tools needed to build Apache Drill from source. + +This is known to work on Ubuntu 20.04 with Docker installed. +On other systems your success may vary. On Redhat/CentOS based systems no longer have Docker. + ## Confirm settings # java -version java version "1.8.0_161" diff --git a/start-build-env.sh b/start-build-env.sh new file mode 100755 index 00000000000..0740149109c --- /dev/null +++ b/start-build-env.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -e # exit on error + +cd "$(dirname "$0")" # connect to root + +DOCKER_DIR=dev-support/docker +DOCKER_FILE="${DOCKER_DIR}/Dockerfile" + +CONTAINER_NAME=drill-dev-${USER}-$$ + +#CPU_ARCH=$(echo "$MACHTYPE" | cut -d- -f1) +#if [ "$CPU_ARCH" = "aarch64" ]; then +# DOCKER_FILE="${DOCKER_DIR}/Dockerfile_aarch64" +#fi + +docker build -t drill-build -f $DOCKER_FILE $DOCKER_DIR + +USER_NAME=${SUDO_USER:=$USER} +USER_ID=$(id -u "${USER_NAME}") + +if [ "$(uname -s)" = "Darwin" ]; then + GROUP_ID=100 +fi + +if [ "$(uname -s)" = "Linux" ]; then + GROUP_ID=$(id -g "${USER_NAME}") + # man docker-run + # When using SELinux, mounted directories may not be accessible + # to the container. To work around this, with Docker prior to 1.7 + # one needs to run the "chcon -Rt svirt_sandbox_file_t" command on + # the directories. With Docker 1.7 and later the z mount option + # does this automatically. + if command -v selinuxenabled >/dev/null && selinuxenabled; then + DCKR_VER=$(docker -v| + awk '$1 == "Docker" && $2 == "version" {split($3,ver,".");print ver[1]"."ver[2]}') + DCKR_MAJ=${DCKR_VER%.*} + DCKR_MIN=${DCKR_VER#*.} + if [ "${DCKR_MAJ}" -eq 1 ] && [ "${DCKR_MIN}" -ge 7 ] || + [ "${DCKR_MAJ}" -gt 1 ]; then + V_OPTS=:z + else + for d in "${PWD}" "${HOME}/.m2"; do + ctx=$(stat --printf='%C' "$d"|cut -d':' -f3) + if [ "$ctx" != svirt_sandbox_file_t ] && [ "$ctx" != container_file_t ]; then + printf 'INFO: SELinux is enabled.\n' + printf '\tMounted %s may not be accessible to the container.\n' "$d" + printf 'INFO: If so, on the host, run the following command:\n' + printf '\t# chcon -Rt svirt_sandbox_file_t %s\n' "$d" + fi + done + fi + fi +fi + +# Set the home directory in the Docker container. +DOCKER_HOME_DIR=${DOCKER_HOME_DIR:-/home/${USER_NAME}} + +DOCKER_GROUP_ID=$(getent group docker | cut -d':' -f3) + +docker build -t "drill-build-${USER_ID}" - < "/etc/sudoers.d/drill-build-${USER_ID}" +ENV HOME "${DOCKER_HOME_DIR}" + +UserSpecificDocker + +echo "" +echo "Docker image build completed." +echo "==============================================================================================" +echo "" + +#If this env varible is empty, docker will be started +# in non interactive mode +DOCKER_INTERACTIVE_RUN=${DOCKER_INTERACTIVE_RUN-"-i -t"} + +DOCKER_SOCKET_MOUNT="" +if [ -S /var/run/docker.sock ]; +then + DOCKER_SOCKET_MOUNT="-v /var/run/docker.sock:/var/run/docker.sock${V_OPTS:-}" + echo "Enabling Docker support with the docker build environment." +else + echo "There is NO Docker support with the docker build environment." +fi + +COMMAND=( "$@" ) +if [ $# -eq 0 ]; +then + COMMAND=( "bash" ) +fi + +[ -d "${HOME}/.gradle_drill_build_env" ] || mkdir -p "${HOME}/.gradle_drill_build_env" + +# By mapping the .m2 directory you can do an mvn install from +# within the container and use the result on your normal +# system. And this also is a significant speedup in subsequent +# builds because the dependencies are downloaded only once. +docker run --rm=true ${DOCKER_INTERACTIVE_RUN} \ + --name "${CONTAINER_NAME}" \ + -v "${HOME}/.m2:${DOCKER_HOME_DIR}/.m2${V_OPTS:-}" \ + -v "${HOME}/.gnupg:${DOCKER_HOME_DIR}/.gnupg${V_OPTS:-}" \ + -v "${HOME}/.gradle_drill_build_env:${DOCKER_HOME_DIR}/.gradle${V_OPTS:-}" \ + -v "${PWD}:${DOCKER_HOME_DIR}/drill${V_OPTS:-}" \ + -w "${DOCKER_HOME_DIR}/drill" \ + ${DOCKER_SOCKET_MOUNT} \ + -u "${USER_ID}" \ + "drill-build-${USER_ID}" "${COMMAND[@]}"