diff --git a/ansible/callback_plugins/log_beautifier.py b/ansible/callback_plugins/log_beautifier.py index e8a88c777..214c54c55 100644 --- a/ansible/callback_plugins/log_beautifier.py +++ b/ansible/callback_plugins/log_beautifier.py @@ -5,6 +5,7 @@ from __future__ import absolute_import import json import re +import sys from threading import Timer from ansible.plugins.callback import CallbackBase @@ -15,7 +16,10 @@ def obfuscate_credentials(input_value): def fixed_dump_results(self, result, indent=None, sort_keys=True, keep_invocation=False): json_message = self._original_dump_results(result, indent, sort_keys, keep_invocation) - message_dictionary = json.loads(json_message, encoding="utf-8") + if sys.version_info.major>=3 and sys.version_info.minor>=9: + message_dictionary = json.loads(json_message) + else: + message_dictionary = json.loads(json_message, encoding="utf-8") result = "" for key, value in message_dictionary.items(): if key not in ["stderr", "stdout_lines"]: diff --git a/ansible/data/requirements.txt b/ansible/data/focal/requirements.txt similarity index 100% rename from ansible/data/requirements.txt rename to ansible/data/focal/requirements.txt diff --git a/ansible/data/jammy/requirements.txt b/ansible/data/jammy/requirements.txt new file mode 100644 index 000000000..9ef6e08e8 --- /dev/null +++ b/ansible/data/jammy/requirements.txt @@ -0,0 +1,20 @@ +ansible +ansible-core +asn1crypto +bcrypt +cffi +cryptography +ipaddress +Jinja2 +MarkupSafe +packaging +paramiko +pbr +pyasn1 +pycparser +pycrypto +PyNaCl +pyopenssl +resolvelib +six +testresources diff --git a/ansible/deploy.sh b/ansible/deploy.sh index f867577b1..6f092e812 100755 --- a/ansible/deploy.sh +++ b/ansible/deploy.sh @@ -203,7 +203,8 @@ echo " | Setting up requirements |" echo " ----------------------------------" echo "" -sudo pip3 install -r "${SR_BUILD_TOOLS_HOME}/ansible/data/requirements.txt" +export ubuntu_version="$(lsb_release -cs)" +sudo pip3 install -r "${SR_BUILD_TOOLS_HOME}/ansible/data/$ubuntu_version/requirements.txt" sudo pip3 install --upgrade setuptools==51.1.1 echo "" diff --git a/ansible/docker_site.yml b/ansible/docker_site.yml index ffa9753eb..f8dea2370 100644 --- a/ansible/docker_site.yml +++ b/ansible/docker_site.yml @@ -35,13 +35,13 @@ "xml_code_style_check", "catkin_code_style_check", "code_style_check", "create_workspace", "benchmarking", "check_install"]} - - {role: build_deps/ur-client-library, tags: ["build", "unit_tests", "software_tests", "all_tests", - "code_coverage", "codecov_tool", "python_code_coverage", - "cpp_code_coverage", "check_deb_make", "complete_deb_check", - "check_build", "python_code_style_check", - "cpp_code_style_check", "xml_code_style_check", - "catkin_code_style_check", "code_style_check", - "create_workspace", "benchmarking", "check_install"]} + - {role: build_deps/ur-client-library, when: (ros_release == "noetic"), tags: ["build", "unit_tests", "software_tests", "all_tests", + "code_coverage", "codecov_tool", "python_code_coverage", + "cpp_code_coverage", "check_deb_make", "complete_deb_check", + "check_build", "python_code_style_check", + "cpp_code_style_check", "xml_code_style_check", + "catkin_code_style_check", "code_style_check", + "create_workspace", "benchmarking", "check_install"]} - {role: ci/update_dependencies, when: run_build, tags: ["build", "unit_tests", "software_tests", "all_tests", "code_coverage", "codecov_tool", "python_code_coverage", "cpp_code_coverage", "check_deb_make", diff --git a/ansible/roles/ci/init/create_workspace/tasks/main.yml b/ansible/roles/ci/init/create_workspace/tasks/main.yml index a6beb1174..c0b34264d 100644 --- a/ansible/roles/ci/init/create_workspace/tasks/main.yml +++ b/ansible/roles/ci/init/create_workspace/tasks/main.yml @@ -61,6 +61,4 @@ # chdir: "/home/user/workspace/" # become: yes # become_method: sudo - when: - - ros_release == 'rolling' - - ros_release == 'humble' + when: (ros_release == 'rolling') or (ros_release == 'humble') diff --git a/ansible/roles/ci/init/install/tasks/main.yml b/ansible/roles/ci/init/install/tasks/main.yml index 65cc26b3d..463c2196f 100644 --- a/ansible/roles/ci/init/install/tasks/main.yml +++ b/ansible/roles/ci/init/install/tasks/main.yml @@ -18,17 +18,18 @@ - ros_release != 'rolling' - ros_release != 'humble' +# Using a specific version for empy as that's the one that is installed by default in Ubuntu Jammy from debian repos +# so pip will skip it. Otherwise pip throws an error when failing to uninstall the deb version to install the latest pip version. +# I don't really know what this package is needed for. - name: Install PIP modules pip: - name: ['catkin_pkg','empy','coverage','catkin-lint','pylint'] + name: ['catkin_pkg','empy==3.3.4','coverage','catkin-lint','pylint'] extra_args: '--upgrade' executable: pip3 become: yes become_method: sudo become_user: root - when: - - ros_release == 'rolling' - - ros_release == 'humble' + when: (ros_release == 'rolling') or (ros_release == 'humble') - name: Install PIP modules pip: diff --git a/bin/sr-run-ci-build.sh b/bin/sr-run-ci-build.sh index 08ae157cc..2a94a6eb6 100755 --- a/bin/sr-run-ci-build.sh +++ b/bin/sr-run-ci-build.sh @@ -24,7 +24,13 @@ if [ "semaphore_docker" != $server_type ] && [ "local" != $server_type ] && [ " sudo apt-get install -y python3-dev libxml2-dev libxslt-dev python3-pip lcov wget git libssl-dev libffi-dev libyaml-dev sudo pip3 install --upgrade pip setuptools==51.1.1 gcovr - sudo pip3 install PyYAML==5.4.1 --ignore-installed + if [ "$ubuntu_version" != "jammy" ] + then + sudo pip3 install PyYAML==5.4.1 --ignore-installed + else + sudo pip3 install "PyYAML>=5.4.1" --ignore-installed + fi + git config --global user.email "build.tools@example.com" git config --global user.name "Build Tools" @@ -34,7 +40,7 @@ if [ "semaphore_docker" != $server_type ] && [ "local" != $server_type ] && [ " git clone --depth 1 https://github.com/shadow-robot/sr-build-tools.git -b "$toolset_branch" $build_tools_folder cd $build_tools_folder/ansible - sudo pip3 install -r data/requirements.txt + sudo pip3 install -r data/$ubuntu_version/requirements.txt fi export extra_variables="codecov_secure=$CODECOV_TOKEN github_login=$GITHUB_LOGIN github_password=$GITHUB_PASSWORD ros_release=$ros_release ubuntu_version_name=$ubuntu_version " @@ -83,7 +89,7 @@ case $server_type in ;; "docker_hub") echo "Docker Hub" - PYTHONUNBUFFERED=1 ansible-playbook -v -i "localhost," -c local docker_site.yml --tags "docker_hub,$tags_list" -e "ros_release=$ros_release ubuntu_version_name=$ubuntu_version" + PYTHONUNBUFFERED=1 ansible-playbook -vvv -i "localhost," -c local docker_site.yml --tags "docker_hub,$tags_list" -e "ros_release=$ros_release ubuntu_version_name=$ubuntu_version" ;; "local") echo "Local run" diff --git a/docker/ros2/humble/Dockerfile b/docker/ros2/humble/Dockerfile index 7164cdad8..d5bb36042 100644 --- a/docker/ros2/humble/Dockerfile +++ b/docker/ros2/humble/Dockerfile @@ -1,5 +1,5 @@ # -# ROS2 rolling with build tools Dockerfile +# ROS2 humble with build tools Dockerfile # # https://github.com/shadow-robot/sr-build-tools/ # @@ -18,9 +18,10 @@ ENV toolset_branch ${toolset_branch:-"master"} ENV server_type="docker_hub" ENV used_modules="check_cache,create_workspace" ENV ros_release_name=humble +ENV ubuntu_version_name=jammy ENV remote_shell_script="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/bin/sr-run-ci-build.sh" -ENV remote_additional_bashrc_cmds="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/docker/utils/additional_bashrc_cmds_rolling" +ENV remote_additional_bashrc_cmds="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/docker/utils/additional_bashrc_cmds_humble" COPY entrypoint.sh /usr/local/bin/entrypoint.sh COPY terminator_config /tmp/terminator_config diff --git a/docker/ros2/humble/buildspec.yml b/docker/ros2/humble/buildspec.yml index b070e31a4..d6e572005 100644 --- a/docker/ros2/humble/buildspec.yml +++ b/docker/ros2/humble/buildspec.yml @@ -5,7 +5,7 @@ env: variables: branch: default image: build-tools - tag: focal-humble + tag: humble dockerfile_folder: docker/ros2/humble/ exported-variables: - branch @@ -13,9 +13,6 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - | @@ -29,15 +26,10 @@ phases: else branch=$CODEBUILD_SOURCE_VERSION fi - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - git checkout $branch - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot - - docker build --build-arg toolset_branch=$branch-t $image . + - docker build --build-arg toolset_branch=$branch -t $image . - docker tag $image:latest public.ecr.aws/shadowrobot/$image:$tag - docker push public.ecr.aws/shadowrobot/$image:$tag - comm -13 <(aws ecr-public describe-image-tags --repository-name $image --region us-east-1 --output text | awk '{print $3}' | tr " " "\n" | sort -u) <(aws ecr-public describe-images --repository-name $image --region us-east-1 --output text | awk '{print $2}' | tr " " "\n" | sort -u) >> ids_to_delete.txt diff --git a/docker/ros2/rolling/Dockerfile b/docker/ros2/rolling/Dockerfile index 0ad9f6f82..8ad7e43db 100644 --- a/docker/ros2/rolling/Dockerfile +++ b/docker/ros2/rolling/Dockerfile @@ -4,7 +4,7 @@ # https://github.com/shadow-robot/sr-build-tools/ # -FROM osrf/ros2:nightly +FROM ros:rolling-ros-base LABEL Description="This image is used to make ROS2 rolling based projects build faster using build tools" Vendor="Shadow Robot" Version="1.0" @@ -18,6 +18,7 @@ ENV toolset_branch ${toolset_branch:-"master"} ENV server_type="docker_hub" ENV used_modules="check_cache,create_workspace" ENV ros_release_name=rolling +ENV ubuntu_version_name=jammy ENV remote_shell_script="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/bin/sr-run-ci-build.sh" ENV remote_additional_bashrc_cmds="https://raw.githubusercontent.com/shadow-robot/sr-build-tools/$toolset_branch/docker/utils/additional_bashrc_cmds_rolling" diff --git a/docker/ros2/rolling/buildspec.yml b/docker/ros2/rolling/buildspec.yml index f032fe310..18fb8d249 100644 --- a/docker/ros2/rolling/buildspec.yml +++ b/docker/ros2/rolling/buildspec.yml @@ -5,7 +5,7 @@ env: variables: branch: default image: build-tools - tag: focal-rolling + tag: rolling dockerfile_folder: docker/ros2/rolling/ exported-variables: - branch @@ -13,16 +13,12 @@ env: - tag - dockerfile_folder phases: - install: - runtime-versions: - docker: 19 build: commands: - | if [[ $branch != "default" ]]; then toolset_branch=$branch elif [[ $CODEBUILD_SOURCE_VERSION == "pr/"* ]]; then - pr_number=${CODEBUILD_SOURCE_VERSION#pr/} pr_number=${CODEBUILD_SOURCE_VERSION#pr/} branch=$(curl -s "https://api.github.com/repos/shadow-robot/sr-build-tools/pulls/$pr_number" | jq -r '.head.ref') elif [[ $CODEBUILD_SOURCE_VERSION =~ ^[0-9a-f]{40}$ ]]; then @@ -30,11 +26,6 @@ phases: else branch=$CODEBUILD_SOURCE_VERSION fi - - awscli_v1_executable=$(which aws) - - rm $awscli_v1_executable - - curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" - - unzip awscliv2.zip - - ./aws/install - git checkout $branch - cd $dockerfile_folder - aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/shadowrobot diff --git a/docker/utils/additional_bashrc_cmds_humble b/docker/utils/additional_bashrc_cmds_humble new file mode 100644 index 000000000..1fabf1c14 --- /dev/null +++ b/docker/utils/additional_bashrc_cmds_humble @@ -0,0 +1,49 @@ +# Print nickname for git/hg/bzr/svn version control in CWD +# Optional $1 of format string for printf, default "(%s) " +if [ -f /etc/bash_completion.d/git ]; then + . /etc/bash_completion.d/git +fi +if [ -f /etc/bash_completion.d/git-prompt ]; then + . /etc/bash_completion.d/git-prompt +fi +if [ -f /usr/share/bash-completion/completions/git ]; then + . /usr/share/bash-completion/completions/git +fi +function be_get_branch { + local dir="$PWD" + local vcs + local nick + while [[ "$dir" != "/" ]]; do + for vcs in git hg svn bzr; do + if [[ -d "$dir/.$vcs" ]] && hash "$vcs" &>/dev/null; then + case "$vcs" in + git) __git_ps1 "${1:-(%s) }"; return;; + hg) nick=$(hg branch 2>/dev/null);; + svn) nick=$(svn info 2>/dev/null\ + | grep -e '^Repository Root:'\ + | sed -e 's#.*/##');; + bzr) + local conf="${dir}/.bzr/branch/branch.conf" # normal branch + [[ -f "$conf" ]] && nick=$(grep -E '^nickname =' "$conf" | cut -d' ' -f 3) + conf="${dir}/.bzr/branch/location" # colo/lightweight branch + [[ -z "$nick" ]] && [[ -f "$conf" ]] && nick="$(basename "$(< $conf)")" + [[ -z "$nick" ]] && nick="$(basename "$(readlink -f "$dir")")";; + esac + [[ -n "$nick" ]] && printf "${1:-(%s) }" "$nick" + return 0 + fi + done + dir="$(dirname "$dir")" + done +} +# Add branch to PS1 (based on $PS1 or $1), formatted as $2 +export GIT_PS1_SHOWDIRTYSTATE=yes + +#pretty PS1: +export PS1="\$([[ \$? != 0 ]] && echo \"[\[\033[0;31m\]\342\234\227\[\033[0;37m\]]\342\224\200\")[$(if [[ ${EUID} == 0 ]]; then echo '\[\033[0;31m\]\h'; else echo '\[\033[0;33m\]\u\[\033[0;37m\]@\[\033[0;95m\]\h'; fi)\[\033[0;37m\]]\342\224\200[\[\033[0;32m\]\w\[\033[0;37m\]]\n\[\033[0;37m\]\342\224\224\342\224\200\342\224\200\342\225\274 \[\033[0m\]" +export PS1="\[\033[0;37m\]\342\224\214\342\224\200\$(be_get_branch "$2")${PS1}"; + +alias dephumble='rosdep install --from-paths . --ignore-src --rosdistro humble -y' + +export ROSCONSOLE_FORMAT='[${severity}](${node}): [${time}] ${message}' +alias python=python3