From 881c8251df13388257f742725899d881624d4f60 Mon Sep 17 00:00:00 2001 From: Nicolas Bock Date: Mon, 26 Feb 2018 11:04:39 -0700 Subject: [PATCH] Add Intel compilers Closes: #178 --- .travis.yml | 38 ++++---- install-icc.sh | 234 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 256 insertions(+), 16 deletions(-) create mode 100755 install-icc.sh diff --git a/.travis.yml b/.travis.yml index 98c63a2bf..d2a6fb28b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,6 +37,8 @@ env: # via the "travis encrypt" command using the project repo's public # key. - secure: "itC06I482XliT7b2yyoit5eWHdbIa/Wpp+cNIsZElMnWhUyBHCuKiLQHLgyovlhCR6UsL7ISDdyxxNz4Kn7Os1EdPhQ1Q+P+8vZf8pROK1LBIZLrTWM7npea3ydo3PCdpgJdr9qHAEvXhBG9mshmrBgJLdmDuNtJ7yFn2l8nLegCnmsdExkjN1ZbprRutPd9OH6iIu9C8lPAiYR1bcDEPm+fgF3VnE2JFztYvkUt+PBiwIveVFlwAQRjrBqvvF/fk2TqFmD97KFDQ6stdr6MzUmQ1m05arololkLp+y6aQxWYbwug66vorAHma+axXJClkd1Rmt/K7kyRBhRUOwE9CKKnHKL9lKYKvfyv5e7HDasg60PACErfBR1/SIyNOWATIV8KCyejn4qI8Q+QjyYGlIM3D2oUCIhIgy/FPnmb6IWGJmqE7Y2kk6BAYZv0PIXLWrffuht7L2qJnl1FI4nVsej9Kf71zJfhxfa2XY9vuFm40ldOMF0DuSOc5KNKHuIGliLhBFKws91cYqz2GnIJV2SU0Ut0ICJ935R9qz+vKE8dGlyV80RIR9+ILVCOOh/XRu5Ljb2fAT+NvlBfG5of3PwiEzQLZPHM0vtZD0wDd9N39GInYe510DwzRRPVfYpbXYSDhISEUugB4F7ASjsVNrJ7kbgZFMwU17YSe+i5DI=" + # The Intel compiler license key. + - secure: "tNJdKdOw2xyuthnRbBx2QkKvwYK0RNaxNWglasVkwJrIBX3Hp0lkzsW4ooJivxVNjj1A9B7F1JVWWBrANodP4YS5dr0/60yPcT+yrt2V7b/PU6/N3Ype/Hq3QbkaCYb99lTDIf+76hU8dbugqZLcsXAk0wNg98cT2tVxzZoFnrMqYnJW2/swhYC3Y9BKK07Y6uGynnPwG1gvfaMYdD5aFc0wrAEj8LBikoXw7kk8EFc24Yjz5tPwJ+O+h/+RtEKLTD3EriRJPCEA4wDM8pChizg13gIJjfvBPnPiZnijGmOz4vpfsIro3GvT2i6atyLEb99yQR1OD5nLK/xJTdXwqqK88a+zR/PBTpcAsQkpwU5XAWyj2NeDlFTqyDwZRJDmgFrhs4NS7bzDsbitToef5UmNrf2JeASj6Hw2QY1eYlE3PB7+R5n2QtdSTPnM4E0RXgeDNlMuSJkaBQmpCWeAAhX7pPAbcSfrix7l0iUXNTnV256zTHUQ/JpkNI1x8OWJ1VYQxNmtocT3tvf8G3DL5IvPmRN2cm2DSovN1fzhsISGIjU1C+5NtK7Nx8KejVBMv4/oVZk/u5G03fYee4LH9vl2oK6lIyFx9uUKxTmBvFSJ5YWfMpRCp3I/hCnNSUpNV8i0g4XDzrQOuWExA1DJRBtB3yu8Ew8hG0v5RiCos9I=" - COVERTY_JOB_NUMBER: 1 - DANGER_JOB_NUMBER: 1 - COVERAGE_JOB_NUMBER: 4 @@ -52,27 +54,31 @@ env: - CC=gcc-6 CXX=g++-6 FC=gfortran-6 GCOV=gcov-6 BUILD_SHARED_LIBS=yes BML_OPENMP=no BML_INTERNAL_BLAS=yes COMMAND=testing - CC=gcc-6 CXX=g++-6 FC=gfortran-6 GCOV=gcov-6 BUILD_SHARED_LIBS=yes BML_OPENMP=yes BML_INTERNAL_BLAS=no COMMAND=testing - CC=gcc-6 CXX=g++-6 FC=gfortran-6 GCOV=gcov-6 BUILD_SHARED_LIBS=yes BML_OPENMP=yes BML_INTERNAL_BLAS=yes COMMAND=testing + - CC=icc CXX=icc FC=ifort BUILD_SHARED_LIBS=no BML_OPENMP=no BML_INTERNAL_BLAS=no COMMAND=testing before_install: - # If this is a coverty scan which is not the first matrix job we will exit - # early to avoid spending unnecessary CI resources. - - if [ "${TRAVIS_BRANCH}" = "coverty_scan" -a "${TRAVIS_JOB_NUMBER##*.}" != "${COVERTY_JOB_NUMBER}" ]; then exit 0; fi - - if [ "${TRAVIS_BRANCH}" = "coverty_scan" ]; then echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-; fi - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then pip install cpp-coveralls; fi - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then pip install codecov; fi - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${DANGER_JOB_NUMBER}" ]; then bundle install; fi - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${DANGER_JOB_NUMBER}" ]; then pip install proselint; fi + # If this is a coverty scan which is not the first matrix job we will exit + # early to avoid spending unnecessary CI resources. + - if [ "${TRAVIS_BRANCH}" = "coverty_scan" -a "${TRAVIS_JOB_NUMBER##*.}" != "${COVERTY_JOB_NUMBER}" ]; then exit 0; fi + - if [ "${TRAVIS_BRANCH}" = "coverty_scan" ]; then echo -n | openssl s_client -connect scan.coverity.com:443 | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p' | sudo tee -a /etc/ssl/certs/ca-; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then pip install cpp-coveralls; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then pip install codecov; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${DANGER_JOB_NUMBER}" ]; then bundle install; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${DANGER_JOB_NUMBER}" ]; then pip install proselint; fi + - if [ "${CC}" = "icc" ]; then ./install-icc.sh; fi + - env | sort before_script: - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${DANGER_JOB_NUMBER}" ]; then bundle exec danger || echo "**NOTE** ignoring danger failure"; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${DANGER_JOB_NUMBER}" ]; then bundle exec danger || echo "**NOTE** ignoring danger failure"; fi script: - - export OMP_NUM_THREADS=4 - - export CMAKE_BUILD_TYPE=Debug - - export VERBOSE_MAKEFILE=yes - - export PARALLEL_TEST_JOBS=2 - - if [ "${COVERITY_SCAN_BRANCH}" != "${COVERTY_JOB_NUMBER}" ]; then ./build.sh ${COMMAND}; fi + - if [ "${CC}" = "icc" ]; then source ~/.bashrc; fi + - export OMP_NUM_THREADS=4 + - export CMAKE_BUILD_TYPE=Debug + - export VERBOSE_MAKEFILE=yes + - export PARALLEL_TEST_JOBS=2 + - if [ "${COVERITY_SCAN_BRANCH}" != "${COVERTY_JOB_NUMBER}" ]; then ./build.sh ${COMMAND}; fi after_success: - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then codecov --gcov-exec ${GCOV}; fi - - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then coveralls; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then codecov --gcov-exec ${GCOV}; fi + - if [ "${TRAVIS_JOB_NUMBER##*.}" = "${COVERAGE_JOB_NUMBER}" ]; then coveralls; fi diff --git a/install-icc.sh b/install-icc.sh new file mode 100755 index 000000000..75d4e2836 --- /dev/null +++ b/install-icc.sh @@ -0,0 +1,234 @@ +#!/bin/sh + +# Install Intel Parallel Studio on Travis CI +# https://github.com/nemequ/icc-travis +# +# Originally written for Squash by +# Evan Nemerson. For documentation, bug reports, support requests, +# etc. please use . +# +# To the extent possible under law, the author(s) of this script have +# waived all copyright and related or neighboring rights to this work. +# See for +# details. + +# Tools - Not very useful in non-interactive mode. +COMPONENTS_VTUNE="intel-vtune-amplifier-xe-2016-cli__x86_64;intel-vtune-amplifier-xe-2016-common__noarch;intel-vtune-amplifier-xe-2016-cli-common__noarch;intel-vtune-amplifier-xe-2016-collector-64linux__x86_64;intel-vtune-amplifier-xe-2016-sep__noarch;intel-vtune-amplifier-xe-2016-gui-common__noarch;intel-vtune-amplifier-xe-2016-gui__x86_64;intel-vtune-amplifier-xe-2016-common-pset__noarch" +COMPONENTS_INSPECTOR="intel-inspector-xe-2016-cli__x86_64;intel-inspector-xe-2016-cli-common__noarch;intel-inspector-xe-2016-gui-common__noarch;intel-inspector-xe-2016-gui__x86_64;intel-inspector-xe-2016-cli-pset__noarch" +COMPONENTS_ADVISOR="intel-advisor-xe-2016-cli__x86_64;intel-advisor-xe-2016-cli-common__noarch;intel-advisor-xe-2016-gui-common__noarch;intel-advisor-xe-2016-gui__x86_64;intel-advisor-xe-2016-cli-pset__noarch" +COMPONENTS_GDB="intel-gdb-gt__x86_64;intel-gdb-gt-src__noarch;intel-gdb-gt-libelfdwarf__x86_64;intel-gdb-gt-devel__x86_64;intel-gdb-gt-common__noarch;intel-gdb-ps-cdt__x86_64;intel-gdb-ps-cdt-source__x86_64;intel-gdb-ps-mic__x86_64;intel-gdb-ps-mpm__x86_64;intel-gdb__x86_64;intel-gdb-source__noarch;intel-gdb-python-source__noarch;intel-gdb-common__noarch;intel-gdb-ps-common__noarch" + +# Parallel libraries +COMPONENTS_MPI="intel-mpi-rt-core__x86_64;intel-mpi-rt-mic__x86_64;intel-mpi-sdk-core__x86_64;intel-mpi-sdk-mic__x86_64;intel-mpi-psxe__x86_64;intel-mpi-rt-psxe__x86_64" +COMPONENTS_TBB="intel-tbb-libs__noarch;intel-mkl-ps-tbb__x86_64;intel-mkl-ps-tbb-devel__x86_64;intel-mkl-ps-tbb-mic__x86_64;intel-mkl-ps-tbb-mic-devel__x86_64;intel-tbb-source__noarch;intel-tbb-devel__noarch;intel-tbb-common__noarch;intel-tbb-ps-common__noarch;intel-tbb-psxe__noarch" +COMPONENTS_MKL="intel-mkl__x86_64;intel-mkl-ps__x86_64;intel-mkl-common__noarch;intel-mkl-ps-common__noarch;intel-mkl-devel__x86_64;intel-mkl-ps-mic-devel__x86_64;intel-mkl-ps-f95-devel__x86_64;intel-mkl-gnu-devel__x86_64;intel-mkl-ps-gnu-devel__x86_64;intel-mkl-ps-pgi-devel__x86_64;intel-mkl-sp2dp-devel__x86_64;intel-mkl-ps-cluster-devel__x86_64;intel-mkl-ps-cluster-common__noarch;intel-mkl-ps-f95-common__noarch;intel-mkl-ps-cluster__x86_64;intel-mkl-gnu__x86_64;intel-mkl-ps-gnu__x86_64;intel-mkl-ps-pgi__x86_64;intel-mkl-sp2dp__x86_64;intel-mkl-ps-mic__x86_64;intel-mkl-ps-tbb__x86_64;intel-mkl-ps-tbb-devel__x86_64;intel-mkl-ps-tbb-mic__x86_64;intel-mkl-ps-tbb-mic-devel__x86_64;intel-mkl-psxe__noarch" +COMPONENTS_IPP="intel-ipp-l-common__noarch;intel-ipp-l-ps-common__noarch;intel-ipp-l-st__x86_64;intel-ipp-l-mt__x86_64;intel-ipp-l-st-devel__x86_64;intel-ipp-l-ps-st-devel__x86_64;intel-ipp-l-mt-devel__x86_64;intel-ipp-psxe__noarch" +COMPONENTS_IPP_CRYPTO="intel-crypto-ipp-st-devel__x86_64;intel-crypto-ipp-ps-st-devel__x86_64;intel-crypto-ipp-st__x86_64;intel-crypto-ipp-mt-devel__x86_64;intel-crypto-ipp-mt__x86_64;intel-crypto-ipp-ss-st-devel__x86_64;intel-crypto-ipp-common__noarch" +COMPONENTS_DAAL="intel-daal__x86_64;intel-daal-common__noarch" + +# Compilers +COMPONENTS_OPENMP="intel-openmp-l-all__x86_64;intel-openmp-l-ps-mic__x86_64;intel-openmp-l-ps__x86_64;intel-openmp-l-ps-ss__x86_64;intel-openmp-l-all-devel__x86_64;intel-openmp-l-ps-mic-devel__x86_64;intel-openmp-l-ps-devel__x86_64;intel-openmp-l-ps-ss-devel__x86_64" +COMPONENTS_COMPILER_COMMON="intel-comp-l-all-vars__noarch;intel-comp-l-all-common;intel-comp-l-ps-common;intel-comp-l-all-devel;intel-comp-l-ps-ss-devel;intel-comp-l-ps-ss-wrapper;intel-comp-l-all-devel;intel-comp-l-ps-devel__x86_64;intel-comp-l-ps-ss-devel__x86_64;intel-psf-intel__x86_64;intel-icsxe-pset;intel-ipsf__noarch;intel-ccompxe__noarch;${COMPONENTS_OPENMP}" +COMPONENTS_IFORT="intel-ifort-l-ps__x86_64;intel-ifort-l-ps-vars__noarch;intel-ifort-l-ps-common__noarch;intel-ifort-l-ps-devel__x86_64;${COMPONENTS_COMPILER_COMMON}" +COMPONENTS_ICC="intel-icc-l-all__x86_64;intel-icc-l-ps-ss__x86_64;intel-icc-l-all-vars__noarch;intel-icc-l-all-common__noarch;intel-icc-l-ps-common__noarch;intel-icc-l-all-devel__x86_64;intel-icc-l-ps-devel__x86_64;intel-icc-l-ps-ss-devel__x86_64;${COMPONENTS_COMPILER_COMMON}" + +DESTINATION="${HOME}/intel" +TEMPORARY_FILES="/tmp" +PHONE_INTEL="no" +COMPONENTS="" + +add_components() { + if [ ! -z "${COMPONENTS}" ]; then + COMPONENTS="${COMPONENTS};$1" + else + COMPONENTS="$1" + fi +} + +while [ $# != 0 ]; do + case "$1" in + "--dest") + DESTINATION="$(realpath "$2")"; shift + ;; + "--tmpdir") + TEMPORARY_FILES="$2"; shift + ;; + "--components") + shift + OLD_IFS="${IFS}" + IFS="," + for component in $1; do + case "$component" in + "icc") + add_components "${COMPONENTS_ICC}" + ;; + "mpi") + add_components "${COMPONENTS_MPI}" + ;; + "vtune") + add_components "${COMPONENTS_VTUNE}" + ;; + "inspector") + add_components "${COMPONENTS_INSPECTOR}" + ;; + "advisor") + add_components "${COMPONENTS_ADVISOR}" + ;; + "tbb") + add_components "${COMPONENTS_TBB}" + ;; + "ifort") + add_components "${COMPONENTS_IFORT}" + ;; + "mkl") + add_components "${COMPONENTS_MKL}" + ;; + "openmp") + # Noop, here for compatability. OpenMP is installed with icc and ifort now. + ;; + "ipp") + add_components "${COMPONENTS_IPP}" + ;; + "ipp-crypto") + add_components "${COMPONENTS_IPP_CRYPTO}" + ;; + "gdb") + add_components "${COMPONENTS_GDB}" + ;; + "daal") + add_components "${COMPONENTS_DAAL}" + ;; + *) + echo "Unknown component '$component'" + exit 1 + ;; + esac + done + IFS="${OLD_IFS}" + ;; + *) + echo "Unrecognized argument '$1'" + exit 1 + ;; + esac + shift +done + +if [ -z "${COMPONENTS}" ]; then + COMPONENTS="${COMPONENTS_ICC}" +fi + +INSTALLER_SCRIPT="parallel_studio_xe_2016_update3_online.sh" +INSTALLER="${TEMPORARY_FILES}/${INSTALLER_SCRIPT}" +INSTALLER_URL="http://registrationcenter-download.intel.com/akdlm/irc_nas/9061/${INSTALLER_SCRIPT}" +SILENT_CFG="${TEMPORARY_FILES}/silent.cfg" +SUCCESS_INDICATOR="${TEMPORARY_FILES}/icc-travis-success" + +if [ ! -e "${TEMPORARY_FILES}" ]; then + echo "${TEMPORARY_FILES} does not exist, creating..." + mkdir -p "${TEMPORARY_FILES}" || (sudo mkdir -p "${TEMPORARY_FILES}" && sudo chown -R "${USER}:${USER}" "${TEMPORARY_FILES}") +fi + +if [ ! -e "${DESTINATION}" ]; then + echo "${DESTINATION} does not exist, creating..." + mkdir -p "${DESTINATION}" || (sudo mkdir -p "${DESTINATION}" && sudo chown -R "${USER}:${USER}" "${DESTINATION}") +fi + +if [ ! -e "${INSTALLER}" ]; then + wget -O "${INSTALLER}" "${INSTALLER_URL}" || exit 1 +fi +chmod u+x "${INSTALLER}" + +# See https://software.intel.com/en-us/articles/intel-composer-xe-2015-silent-installation-guide +echo "# Generated silent configuration file" > "${SILENT_CFG}" +echo "ACCEPT_EULA=accept" >> "${SILENT_CFG}" +echo "INSTALL_MODE=NONRPM" >> "${SILENT_CFG}" +echo "CONTINUE_WITH_OPTIONAL_ERROR=yes" >> "${SILENT_CFG}" +echo "PSET_INSTALL_DIR=${DESTINATION}" >> "${SILENT_CFG}" +echo "CONTINUE_WITH_INSTALLDIR_OVERWRITE=yes" >> "${SILENT_CFG}" +echo "COMPONENTS=${COMPONENTS}" >> "${SILENT_CFG}" +echo "PSET_MODE=install" >> "${SILENT_CFG}" +echo "PHONEHOME_SEND_USAGE_DATA=${PHONE_INTEL}" >> "${SILENT_CFG}" +if [ "x" != "x${INTEL_SERIAL_NUMBER}" ]; then + echo "ACTIVATION_SERIAL_NUMBER=${INTEL_SERIAL_NUMBER}" >> "${SILENT_CFG}" + echo "ACTIVATION_TYPE=serial_number" >> "${SILENT_CFG}" +else + echo "ACTIVATION_TYPE=trial_lic" >> "${SILENT_CFG}" +fi + +attempt=1; +while [ $attempt -le 3 ]; do + if [ ! -e "${TEMPORARY_FILES}/parallel-studio-install-data" ]; then + mkdir -p "${TEMPORARY_FILES}/parallel-studio-install-data" || (sudo mkdir -p "${TEMPORARY_FILES}/parallel-studio-install-data" && sudo chown -R "${USER}:${USER}" "${TEMPORARY_FILES}") + fi + + ("${INSTALLER}" \ + -t "${TEMPORARY_FILES}/parallel-studio-install-data" \ + -s "${SILENT_CFG}" \ + --cli-mode \ + --user-mode && \ + touch "${SUCCESS_INDICATOR}") & + + # So Travis doesn't die in case of a long download/installation. + # + # NOTE: a watched script never terminates. + elapsed=0; + while kill -0 $! 2>/dev/null; do + sleep 1 + elapsed=$(expr $elapsed + 1) + if [ $(expr $elapsed % 60) -eq 0 ]; then + mins_elapsed=$(expr $elapsed / 60) + if [ $mins_elapsed = 1 ]; then + minute_string="minute" + else + minute_string="minutes" + fi + echo "Still running... (about $(expr $elapsed / 60) ${minute_string} so far)." + fi + done + + if [ ! -e "${SUCCESS_INDICATOR}" ]; then + echo "Installation failed." + exit 1 + fi + + if [ ! -e "${DESTINATION}/bin/compilervars.sh" ]; then + # Sometimes the installer returns successfully without actually + # installing anything. Let's try again… + echo "Installation attempt #${attempt} completed, but unable to find compilervars.sh." + find "${DESTINATION}" + else + break + fi + + rm -vrf "${TEMPORARY_FILES}/parallel-studio-install-data" + echo "Trying again..." + + attempt=$(expr $attempt + 1) +done + +if [ ! -e "${DESTINATION}/bin/compilervars.sh" ]; then + echo "Installation failed." + exit 1 +fi + +# Apparently the installer drops the license file in a location it +# doesn't know to check. +ln -s "${DESTINATION}"/licenses ~/Licenses + +# Add configuration information to ~/.bashrc. Unfortunately this will +# not be picked up automatically by Travis, so you'll still need to +# source ~/.bashrc in your .travis.yml +# +# Container-based builds include a `[ -z "$PS1" ] && return` near the +# beginning of the file, so appending won't work, we'll need to +# prepend. +echo "export INTEL_INSTALL_PATH=\"${DESTINATION}\"" >> ~/.bashrc-intel +echo ". \"\${INTEL_INSTALL_PATH}/bin/compilervars.sh\" intel64" >> ~/.bashrc-intel +echo "export LD_LIBRARY_PATH=\"\${INTEL_INSTALL_PATH}/ism/bin/intel64:\${INTEL_INSTALL_PATH}/lib/intel64_lin:\$LD_LIBRARY_PATH\"" >> ~/.bashrc-intel +echo "export PATH=\"\${INTEL_INSTALL_PATH}/bin:\$PATH\"" >> ~/.bashrc-intel +echo "function uninstall_intel_software {" >> ~/.bashrc-intel +echo " find \"\${INTEL_INSTALL_PATH}\" -name 'uninstall.sh' -not -path '*/ism/uninstall.sh' -exec {} -s \;" >> ~/.bashrc-intel +echo "}" >> ~/.bashrc-intel +cat ~/.bashrc >> ~/.bashrc-intel +mv ~/.bashrc-intel ~/.bashrc