From 7e118062c4e62f54d2d2010c07cdf967b90aaceb Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 00:29:22 -0500 Subject: [PATCH 01/11] Add new dockerfiles for raspberry pi --- docker/dbmongo/Dockerfile_raspberrypi | 34 ++++ docker/lab/Dockerfile_raspberrypi | 240 ++++++++++++++++++++++++++ docker/machine/Dockerfile_raspberrypi | 97 +++++++++++ 3 files changed, 371 insertions(+) create mode 100644 docker/dbmongo/Dockerfile_raspberrypi create mode 100644 docker/lab/Dockerfile_raspberrypi create mode 100644 docker/machine/Dockerfile_raspberrypi diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/Dockerfile_raspberrypi new file mode 100644 index 000000000..89f3330b0 --- /dev/null +++ b/docker/dbmongo/Dockerfile_raspberrypi @@ -0,0 +1,34 @@ +FROM arm64v8/mongo:bionic +WORKDIR /opt/ + +ARG docker_filepath=docker/dbmongo/files + +#add repo for mongodb +#RUN echo "deb http://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.2.list +#RUN apt-get update --fix-missing && \ +# apt-get install -y wget gnupg +#RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.2.asc | apt-key add - + +#RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.2 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-4.2.list + +RUN apt-get update --fix-missing && \ + apt-get install -y --allow-unauthenticated \ + vim npm openssh-client htop dos2unix \ + net-tools iputils-ping \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +#check if we need to rebuild +COPY ${docker_filepath}/mongod.conf /etc/ +COPY ${docker_filepath}/users.json /root/ +COPY ${docker_filepath}/projects.json /root/ +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/sync.sh /root/ +RUN dos2unix /root/entrypoint.sh /root/sync.sh +RUN dos2unix /root/users.json /root/projects.json +RUN dos2unix /etc/mongod.conf + +EXPOSE 27017 + +CMD ["/bin/bash", "/root/entrypoint.sh"] diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi new file mode 100644 index 000000000..70d83b22b --- /dev/null +++ b/docker/lab/Dockerfile_raspberrypi @@ -0,0 +1,240 @@ +FROM ubuntu:bionic + +#RUN apt-get update --fix-missing && apt-get install -y wget + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN groupadd --gid 1000 node \ + && useradd --uid 1000 --gid node --shell /bin/bash --create-home node + +ENV NODE_VERSION 12.20.0 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ENV YARN_VERSION 1.22.5 + +RUN set -ex \ + && savedAptMark="$(apt-mark showmanual)" \ + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 6A010C5166006599AA17F08146C2130DFD2497F5 \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" \ + && curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz.asc" \ + && gpg --batch --verify yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && mkdir -p /opt \ + && tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn \ + && ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg \ + && rm yarn-v$YARN_VERSION.tar.gz.asc yarn-v$YARN_VERSION.tar.gz \ + && apt-mark auto '.*' > /dev/null \ + && { [ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; } \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + # smoke test + && yarn --version + +ARG docker_filepath=docker/lab/files + +RUN apt-get update --fix-missing && apt-get install -y \ + vim openssh-client openssh-server telnet apache2 \ + net-tools iputils-ping xz-utils \ + screen ngrep ca-cacert \ + mercurial subversion \ + build-essential cmake lsb-core cpio mesa-common-dev \ + dos2unix \ + curl \ + git \ + libatlas-base-dev \ + gfortran \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + +# ensure local python is preferred over distribution python +ENV PATH /usr/local/bin:$PATH + +# http://bugs.python.org/issue19846 +# > At the moment, setting "LANG=C" on a Linux system *fundamentally breaks Python 3*, and that's not OK. +ENV LANG C.UTF-8 + +ENV TZ=America/New_York +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone +ARG DEBIAN_FRONTEND=noninteractive +RUN apt-get update && apt-get install -y --no-install-recommends \ + libbluetooth-dev \ + tk-dev \ + uuid-dev \ + && rm -rf /var/lib/apt/lists/* +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget python3.7-venv python3-numpy \ + && apt-get clean \ + && rm -rf /var/lib/apt/list/* + +RUN python3 --version +RUN python --version +RUN python -m venv /venv +ENV PATH=/venv/bin:$PATH +RUN python --version +RUN pip --version + +# setup python environment +COPY ${docker_filepath}/requirements.txt /root/ +RUN mkdir /root/wheel +COPY ${docker_filepath}/wheel /root/wheel +RUN pip3 install --no-cache-dir cython +#RUN pip3 install --no-cache-dir -r /root/requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt +### bill's surprise fork +# NOTE: For some reason, this install doesn't seem to recognize the installed numpy +# when run on Docker. As a workaround, the `wheel` directory includes a wheel +# for this install. +#RUN pip3 install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip3 install --no-index --find-links=/root/wheel scikit_surprise + +## Webserver + +RUN rm /etc/apache2/sites-enabled/* +COPY ${docker_filepath}/ports.conf /etc/apache2/ +RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/ssl* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/socache* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy.* /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_wstunnel.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/proxy_http.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/headers.load /etc/apache2/mods-enabled \ + && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +COPY lab/webapp/package-lock.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/*.json +RUN npm install --silent --progress=false + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +COPY lab/package-lock.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/*.json +RUN npm install --silent --progress=false + +RUN apt-get install -y libgfortran5 liblapack-dev + +COPY ${docker_filepath}/001-pennai.conf /etc/apache2/sites-enabled/ +COPY ${docker_filepath}/htpasswd /etc/apache2/htpasswd +COPY ${docker_filepath}/certs/* /usr/lib/ssl/private/ + +WORKDIR /root/ + +# Webserver - paiwww +COPY ${docker_filepath}/start.sh /root/ + +## Utility script, used when starting ai +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + + +## PennAI Lab server +COPY ${docker_filepath}/entrypoint.sh /root/ +RUN ["chmod", "+x", "/root/entrypoint.sh"] + +RUN dos2unix /root/start.sh \ + && dos2unix /root/wait-for-it.sh \ + && dos2unix /root/entrypoint.sh + +# set version and build environment; tag.sh is sourced in entrypoint.sh +ENV BUILD_ENV='dev' +COPY .env /etc/profile.d/ +RUN cp '/etc/profile.d/.env' '/etc/profile.d/tag.sh' +RUN dos2unix /etc/profile.d/tag.sh +RUN sed -i "s/TAG=/export TAG=/g" /etc/profile.d/tag.sh + + +# Start the webserver +CMD ["/bin/bash", "/root/start.sh"] + +# EXPOSE 443 +EXPOSE 5080 +WORKDIR /appsrc/lab/ +ENTRYPOINT ["/root/entrypoint.sh"] diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi new file mode 100644 index 000000000..b1941fde6 --- /dev/null +++ b/docker/machine/Dockerfile_raspberrypi @@ -0,0 +1,97 @@ +FROM python:3.7-stretch + +#nodejs +#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +# tar -xvf ~/node.tar.xz -C /opt/ && \ +# rm ~/node.tar.xz +#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +ENV NODE_VERSION 12.20.0 + +RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ + && case "${dpkgArch##*-}" in \ + amd64) ARCH='x64';; \ + ppc64el) ARCH='ppc64le';; \ + s390x) ARCH='s390x';; \ + arm64) ARCH='arm64';; \ + armhf) ARCH='armv7l';; \ + i386) ARCH='x86';; \ + *) echo "unsupported architecture"; exit 1 ;; \ + esac \ + && set -ex \ + # libatomic1 for arm + && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* \ + && for key in \ + 4ED778F539E3634C779C87C6D7062848A1AB005C \ + 94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \ + 1C050899334244A8AF75E53792EF661D867B9DFA \ + 71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \ + 8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \ + C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \ + C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \ + DD8F2338BAE7501E3DD5AC78C273792F7D83545D \ + A48C2BEE680E841632CD4E44F07496B3EB3C1762 \ + 108F52B48DB57BB0CC439B2997B01419BD92F80A \ + B9E2F5981AA6E0CD28160D9FF13993A75599653C \ + ; do \ + gpg --batch --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys "$key" || \ + gpg --batch --keyserver hkp://ipv4.pool.sks-keyservers.net --recv-keys "$key" || \ + gpg --batch --keyserver hkp://pgp.mit.edu:80 --recv-keys "$key" ; \ + done \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \ + && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \ + && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \ + && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \ + && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \ + && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \ + && apt-mark auto '.*' > /dev/null \ + && find /usr/local -type f -executable -exec ldd '{}' ';' \ + | awk '/=>/ { print $(NF-1) }' \ + | sort -u \ + | xargs -r dpkg-query --search \ + | cut -d: -f1 \ + | sort -u \ + | xargs -r apt-mark manual \ + && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \ + && ln -s /usr/local/bin/node /usr/local/bin/nodejs \ + # smoke tests + && node --version \ + && npm --version + +ARG docker_filepath=docker/machine/files + +RUN apt-get update --fix-missing && apt-get install -y \ + vim openssh-client openssh-server graphviz \ + net-tools iputils-ping xz-utils \ + screen ngrep ca-cacert \ + mercurial subversion \ + build-essential cmake lsb-core cpio mesa-common-dev \ + libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ + --no-install-recommends && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# setup python environment +COPY ${docker_filepath}/requirements.txt /root/ +RUN mkdir /root/wheel +COPY ${docker_filepath}/wheel /root/wheel +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt + +# setup node environment +RUN npm install -g pm2 --silent --progress=false + +# install node_modules to an anon volume +WORKDIR /appsrc/machine + +COPY machine/package.json /appsrc/machine/ +COPY machine/package-lock.json /appsrc/machine/ +RUN dos2unix /appsrc/machine/*.json +RUN npm install --silent --progress=false + +COPY ${docker_filepath}/entrypoint.sh /root/ +COPY ${docker_filepath}/wait-for-it.sh /root/ +RUN dos2unix /root/wait-for-it.sh && dos2unix /root/entrypoint.sh +RUN ["chmod", "+x", "/root/wait-for-it.sh"] + +CMD ["/bin/bash", "/root/entrypoint.sh"] From 4c56e102666f1c25b23016be535431e6a6cd3438 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 15:33:32 -0500 Subject: [PATCH 02/11] Create new docker compose configuration for rpi --- docker-compose-raspberrypi.yml | 48 ++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 docker-compose-raspberrypi.yml diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml new file mode 100644 index 000000000..7b40dd500 --- /dev/null +++ b/docker-compose-raspberrypi.yml @@ -0,0 +1,48 @@ +version: '3' + +services: + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile_raspberrypi + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + - "5080:5080" + env_file: + - ./config/common.env + - ./config/ai.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile_raspberrypi + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + env_file: ./config/common.env + environment: + - MACHINE_PORT=5081 + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile_raspberrypi + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env From bde6cb511336286e6b95b2c8eab23637ecce0f25 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:06:02 -0500 Subject: [PATCH 03/11] Add rpi dependency wheels as a Git submodule --- .gitmodules | 3 +++ docker/pennai-arm64-deps | 1 + 2 files changed, 4 insertions(+) create mode 100644 .gitmodules create mode 160000 docker/pennai-arm64-deps diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..363522775 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "docker/pennai-arm64-deps"] + path = docker/pennai-arm64-deps + url = https://github.com/EpistasisLab/pennai-arm64-deps diff --git a/docker/pennai-arm64-deps b/docker/pennai-arm64-deps new file mode 160000 index 000000000..83b7045aa --- /dev/null +++ b/docker/pennai-arm64-deps @@ -0,0 +1 @@ +Subproject commit 83b7045aa973cda00a830af46ef0d6dc1f52f04b From e9f7da92b1ca2f8edbe9e29c6ea7a7a8ed429700 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:33:16 -0500 Subject: [PATCH 04/11] Make Dockerfiles for rpi see wheel files --- docker/dbmongo/Dockerfile_raspberrypi | 2 +- docker/lab/Dockerfile_raspberrypi | 17 ++++++----------- docker/machine/Dockerfile_production | 1 + docker/machine/Dockerfile_raspberrypi | 13 ++++--------- 4 files changed, 12 insertions(+), 21 deletions(-) diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/Dockerfile_raspberrypi index 89f3330b0..8bf6e0133 100644 --- a/docker/dbmongo/Dockerfile_raspberrypi +++ b/docker/dbmongo/Dockerfile_raspberrypi @@ -13,7 +13,7 @@ ARG docker_filepath=docker/dbmongo/files RUN apt-get update --fix-missing && \ apt-get install -y --allow-unauthenticated \ - vim npm openssh-client htop dos2unix \ + npm openssh-client htop dos2unix \ net-tools iputils-ping \ --no-install-recommends && \ apt-get clean && \ diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi index 70d83b22b..a0d864aa8 100644 --- a/docker/lab/Dockerfile_raspberrypi +++ b/docker/lab/Dockerfile_raspberrypi @@ -99,18 +99,14 @@ RUN set -ex \ && yarn --version ARG docker_filepath=docker/lab/files +ARG python_wheel_directory=docker/pennai-arm64-deps/lab RUN apt-get update --fix-missing && apt-get install -y \ - vim openssh-client openssh-server telnet apache2 \ + openssh-client openssh-server telnet apache2 \ net-tools iputils-ping xz-utils \ - screen ngrep ca-cacert \ - mercurial subversion \ + ngrep ca-cacert \ build-essential cmake lsb-core cpio mesa-common-dev \ - dos2unix \ - curl \ - git \ - libatlas-base-dev \ - gfortran \ + dos2unix curl git libatlas-base-dev gfortran \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* @@ -158,9 +154,8 @@ RUN pip --version # setup python environment COPY ${docker_filepath}/requirements.txt /root/ RUN mkdir /root/wheel -COPY ${docker_filepath}/wheel /root/wheel +COPY ${python_wheel_directory}/lab /root/wheel RUN pip3 install --no-cache-dir cython -#RUN pip3 install --no-cache-dir -r /root/requirements.txt RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt ### bill's surprise fork # NOTE: For some reason, this install doesn't seem to recognize the installed numpy @@ -183,7 +178,7 @@ RUN cp /etc/apache2/mods-available/rewrite* /etc/apache2/mods-enabled \ && cp /etc/apache2/mods-available/expires.load /etc/apache2/mods-enabled # setup node environment -RUN npm install -g pm2 --silent --progress=false +RUN npm install -g pm2 webpack --silent --progress=false # install lab/webapp/node_modules to an anon volume WORKDIR /appsrc/lab/webapp diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production index 570a6c481..c0cbb2f46 100644 --- a/docker/machine/Dockerfile_production +++ b/docker/machine/Dockerfile_production @@ -11,6 +11,7 @@ RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH ARG docker_filepath=docker/machine/files +ARG python_wheels=docker/pennai-arm64-deps/lab/ RUN apt-get update --fix-missing && apt-get install -y \ vim openssh-client openssh-server graphviz \ diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index b1941fde6..b044f5e07 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -1,11 +1,5 @@ FROM python:3.7-stretch -#nodejs -#RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ -# tar -xvf ~/node.tar.xz -C /opt/ && \ -# rm ~/node.tar.xz -#ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH - ENV NODE_VERSION 12.20.0 RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ @@ -60,12 +54,12 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && npm --version ARG docker_filepath=docker/machine/files +ARG python_wheel_directory=docker/pennai-arm64-deps RUN apt-get update --fix-missing && apt-get install -y \ - vim openssh-client openssh-server graphviz \ + openssh-client openssh-server graphviz \ net-tools iputils-ping xz-utils \ - screen ngrep ca-cacert \ - mercurial subversion \ + ngrep ca-cacert \ build-essential cmake lsb-core cpio mesa-common-dev \ libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ --no-install-recommends && \ @@ -76,6 +70,7 @@ RUN apt-get update --fix-missing && apt-get install -y \ COPY ${docker_filepath}/requirements.txt /root/ RUN mkdir /root/wheel COPY ${docker_filepath}/wheel /root/wheel +COPY ${python_wheel_directory}/machine /root/wheel RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt # setup node environment From add2dd78ed9367cc921f8eb119d41833ab08aeb7 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Fri, 15 Jan 2021 16:46:47 -0500 Subject: [PATCH 05/11] Revert machine Dockerfile_production after accidental change --- docker/machine/Dockerfile_production | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/machine/Dockerfile_production b/docker/machine/Dockerfile_production index c0cbb2f46..570a6c481 100644 --- a/docker/machine/Dockerfile_production +++ b/docker/machine/Dockerfile_production @@ -11,7 +11,6 @@ RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH ARG docker_filepath=docker/machine/files -ARG python_wheels=docker/pennai-arm64-deps/lab/ RUN apt-get update --fix-missing && apt-get install -y \ vim openssh-client openssh-server graphviz \ From e5cadd404bc64a58ea026a93ee936b1e38a49093 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sat, 16 Jan 2021 13:28:11 -0500 Subject: [PATCH 06/11] Builds all docker containers --- docker/lab/Dockerfile_raspberrypi | 8 ++++---- docker/lab/files/requirements-raspberrypi.txt | 9 +++++++++ docker/machine/Dockerfile_raspberrypi | 9 ++++----- docker/machine/files/requirements-raspberrypi.txt | 11 +++++++++++ 4 files changed, 28 insertions(+), 9 deletions(-) create mode 100644 docker/lab/files/requirements-raspberrypi.txt create mode 100644 docker/machine/files/requirements-raspberrypi.txt diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/Dockerfile_raspberrypi index a0d864aa8..f3236b30c 100644 --- a/docker/lab/Dockerfile_raspberrypi +++ b/docker/lab/Dockerfile_raspberrypi @@ -99,7 +99,7 @@ RUN set -ex \ && yarn --version ARG docker_filepath=docker/lab/files -ARG python_wheel_directory=docker/pennai-arm64-deps/lab +ARG python_wheel_directory=docker/pennai-arm64-deps/lab/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server telnet apache2 \ @@ -152,11 +152,11 @@ RUN python --version RUN pip --version # setup python environment -COPY ${docker_filepath}/requirements.txt /root/ +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ RUN mkdir /root/wheel -COPY ${python_wheel_directory}/lab /root/wheel +COPY ${python_wheel_directory}/*.whl /root/wheel/ RUN pip3 install --no-cache-dir cython -RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt ### bill's surprise fork # NOTE: For some reason, this install doesn't seem to recognize the installed numpy # when run on Docker. As a workaround, the `wheel` directory includes a wheel diff --git a/docker/lab/files/requirements-raspberrypi.txt b/docker/lab/files/requirements-raspberrypi.txt new file mode 100644 index 000000000..b33b0a57b --- /dev/null +++ b/docker/lab/files/requirements-raspberrypi.txt @@ -0,0 +1,9 @@ +scikit-learn==0.22.1 +tqdm==4.32.2 +pymongo==3.8.0 +pandas==1.0.3 +cython==0.29.2 +xgboost==0.90 +simplejson==3.16.0 +requests==2.22.0 +joblib==0.16.0 diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index b044f5e07..cf083bd7c 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -54,7 +54,7 @@ RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \ && npm --version ARG docker_filepath=docker/machine/files -ARG python_wheel_directory=docker/pennai-arm64-deps +ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server graphviz \ @@ -67,11 +67,10 @@ RUN apt-get update --fix-missing && apt-get install -y \ rm -rf /var/lib/apt/lists/* # setup python environment -COPY ${docker_filepath}/requirements.txt /root/ +COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ RUN mkdir /root/wheel -COPY ${docker_filepath}/wheel /root/wheel -COPY ${python_wheel_directory}/machine /root/wheel -RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements.txt +COPY ${python_wheel_directory}/*.whl /root/wheel/ +RUN python -m pip install --no-index --find-links=/root/wheel -r /root/requirements-raspberrypi.txt # setup node environment RUN npm install -g pm2 --silent --progress=false diff --git a/docker/machine/files/requirements-raspberrypi.txt b/docker/machine/files/requirements-raspberrypi.txt new file mode 100644 index 000000000..1bcbe8cfd --- /dev/null +++ b/docker/machine/files/requirements-raspberrypi.txt @@ -0,0 +1,11 @@ +scikit-learn==0.22.1 +tqdm==4.32.2 +pymongo==3.8.0 +pandas==1.0.3 +matplotlib==3.1.0 +mlxtend==0.16.0 +pydot==1.4.1 +requests==2.22.0 +xgboost==0.90 +shap==0.34.0 +joblib==0.16.0 From 611f55ee4706ce21739e6b4acc98eefc2aa0f0ce Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sat, 16 Jan 2021 15:38:35 -0500 Subject: [PATCH 07/11] Create mongodb data directory in dbmongo container if it doesn't exist --- docker/dbmongo/files/entrypoint.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker/dbmongo/files/entrypoint.sh b/docker/dbmongo/files/entrypoint.sh index 0c9a68dc3..5f24460e3 100644 --- a/docker/dbmongo/files/entrypoint.sh +++ b/docker/dbmongo/files/entrypoint.sh @@ -3,6 +3,9 @@ wget localhost:51678/v1/metadata -t 1 -qO- &> /dev/null if [ ! -v PROJECTS_FILE ]; then PROJECTS_FILE='projects.json'; fi +if [ ! -d /var/lib/mongodb ]; then + mkdir -p /var/lib/mongodb; +fi mongod -f /etc/mongod.conf --fork #check to see if db was loaded if [ ! -f '/root/forum' ]; then From f4315239a3cf74d95dd55f0c714d6a3af2977ba8 Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:21:27 -0500 Subject: [PATCH 08/11] Bump machine (rpi) to buster to fix arm64 glibc distribution --- docker/machine/Dockerfile_raspberrypi | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/Dockerfile_raspberrypi index cf083bd7c..c38a3f358 100644 --- a/docker/machine/Dockerfile_raspberrypi +++ b/docker/machine/Dockerfile_raspberrypi @@ -1,4 +1,4 @@ -FROM python:3.7-stretch +FROM python:3.7-buster ENV NODE_VERSION 12.20.0 @@ -59,12 +59,14 @@ ARG python_wheel_directory=docker/pennai-arm64-deps/machine/wheel RUN apt-get update --fix-missing && apt-get install -y \ openssh-client openssh-server graphviz \ net-tools iputils-ping xz-utils \ - ngrep ca-cacert \ - build-essential cmake lsb-core cpio mesa-common-dev \ + ngrep \ + libatlas-base-dev \ + build-essential cmake cpio mesa-common-dev \ libglib2.0-0 libxext6 libsm6 libxrender1 dos2unix \ --no-install-recommends && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Removed: lsb-core ca-cacert glibc # setup python environment COPY ${docker_filepath}/requirements-raspberrypi.txt /root/ From ed25cd6bac319740c9b7fb2257a27c21a2816a3a Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:37:20 -0500 Subject: [PATCH 09/11] Improve naming convention for RPi dockerfiles and create RPi test dockerfiles --- docker-compose-int-test-rpi.yml | 70 +++++++++++++++++++ docker-compose-raspberrypi.yml | 6 +- docker-compose-unit-test-rpi.yml | 15 ++++ ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 ...{Dockerfile_raspberrypi => rpi.Dockerfile} | 0 tests/integration/rpi.Dockerfile | 19 +++++ tests/unit/rpi.Dockerfile | 44 ++++++++++++ 8 files changed, 151 insertions(+), 3 deletions(-) create mode 100644 docker-compose-int-test-rpi.yml create mode 100644 docker-compose-unit-test-rpi.yml rename docker/dbmongo/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) rename docker/lab/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) rename docker/machine/{Dockerfile_raspberrypi => rpi.Dockerfile} (100%) create mode 100644 tests/integration/rpi.Dockerfile create mode 100644 tests/unit/rpi.Dockerfile diff --git a/docker-compose-int-test-rpi.yml b/docker-compose-int-test-rpi.yml new file mode 100644 index 000000000..380a0e168 --- /dev/null +++ b/docker-compose-int-test-rpi.yml @@ -0,0 +1,70 @@ +version: '3' + +services: + + tester: + build: ./tests/integration + depends_on: + - lab + - machine + - dbmongo + tty: true + stdin_open: true + command: bash -c "sh /root/wait_pennai.sh && sh /root/int_test_runner.sh" + volumes: + - "./tests/integration/jest:/src/jest" + - "./data/datasets/test/integration:/appsrc/data/datasets/test/integration" + - "./target:/target" + - "./ai/__pycache__:/appsrc/ai/__pycache__" + - "./ai/metalearning/__pycache__:/appsrc/ai/metalearning/__pycache__" + + lab: + build: + context: . + dockerfile: docker/lab/Dockerfile + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" + ports: + #- "443:443" + - "5080:5080" + environment: + - AI_RECOMMENDER=random + - AI_NUMRECOMMEND=1 + - AI_AUTOSTART=1 + - STARTUP_DATASET_PATH=/appsrc/data/datasets/test/test_mixed + env_file: ./config/common.env + depends_on: + - dbmongo + + machine: + build: + context: . + dockerfile: docker/machine/Dockerfile + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/machine/node_modules" + ports: + - "5081:5081" + environment: + - STARTUP_DATASET_PATH=/appsrc/data/datasets/test/test_mixed + - MACHINE_PORT=5081 + env_file: ./config/common.env + depends_on: + - lab + - dbmongo + + dbmongo: + build: + context: . + dockerfile: docker/dbmongo/Dockerfile + tty: true + stdin_open: true + ports: + - "27017:27017" + env_file: ./config/common.env diff --git a/docker-compose-raspberrypi.yml b/docker-compose-raspberrypi.yml index 7b40dd500..397a5ac43 100644 --- a/docker-compose-raspberrypi.yml +++ b/docker-compose-raspberrypi.yml @@ -4,7 +4,7 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile_raspberrypi + dockerfile: docker/lab/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -22,7 +22,7 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile_raspberrypi + dockerfile: docker/machine/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -40,7 +40,7 @@ services: dbmongo: build: context: . - dockerfile: docker/dbmongo/Dockerfile_raspberrypi + dockerfile: docker/dbmongo/rpi.Dockerfile tty: true stdin_open: true ports: diff --git a/docker-compose-unit-test-rpi.yml b/docker-compose-unit-test-rpi.yml new file mode 100644 index 000000000..1993eac34 --- /dev/null +++ b/docker-compose-unit-test-rpi.yml @@ -0,0 +1,15 @@ +version: '3' + +services: + unit_tester: + build: + context: . + dockerfile: tests/unit/Dockerfile + working_dir: /appsrc + command: bash -c "dos2unix tests/unit/unit_test_runner.sh && sh tests/unit/unit_test_runner.sh" + tty: true + stdin_open: true + volumes: + - "./:/appsrc" + - "/appsrc/lab/webapp/node_modules" + - "/appsrc/lab/node_modules" diff --git a/docker/dbmongo/Dockerfile_raspberrypi b/docker/dbmongo/rpi.Dockerfile similarity index 100% rename from docker/dbmongo/Dockerfile_raspberrypi rename to docker/dbmongo/rpi.Dockerfile diff --git a/docker/lab/Dockerfile_raspberrypi b/docker/lab/rpi.Dockerfile similarity index 100% rename from docker/lab/Dockerfile_raspberrypi rename to docker/lab/rpi.Dockerfile diff --git a/docker/machine/Dockerfile_raspberrypi b/docker/machine/rpi.Dockerfile similarity index 100% rename from docker/machine/Dockerfile_raspberrypi rename to docker/machine/rpi.Dockerfile diff --git a/tests/integration/rpi.Dockerfile b/tests/integration/rpi.Dockerfile new file mode 100644 index 000000000..39bbd5e26 --- /dev/null +++ b/tests/integration/rpi.Dockerfile @@ -0,0 +1,19 @@ +FROM node:8.15.1-slim +WORKDIR /src + +RUN apt-get update && apt-get install -y vim dos2unix + +COPY package.json . +COPY tsconfig.json . +# RUN npm install --save es6-promise isomorphic-fetch +# RUN npm install --save-dev typescript jest ts-jest @types/jest +# RUN npm install --save-dev babel-jest babel-preset-react react-test-renderer babel-preset-es2015 +# RUN npm install --save-dev babel-jest babel-preset-react react-test-renderer babel-preset-env +RUN npm install --no-optional --progress=false + +COPY wait-for-it.sh /opt +RUN dos2unix /opt/wait-for-it.sh +RUN ["chmod", "+x", "/opt/wait-for-it.sh"] +COPY wait_pennai.sh /root +COPY int_test_runner.sh /root +RUN dos2unix /root/*.sh diff --git a/tests/unit/rpi.Dockerfile b/tests/unit/rpi.Dockerfile new file mode 100644 index 000000000..f04fee9e6 --- /dev/null +++ b/tests/unit/rpi.Dockerfile @@ -0,0 +1,44 @@ +FROM python:3.7.4-stretch + +#nodejs +RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ + tar -xvf ~/node.tar.xz -C /opt/ && \ + rm ~/node.tar.xz +ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH + +RUN apt-get update --fix-missing \ + && apt-get install -y --no-install-recommends \ + graphviz dos2unix && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN npm install -g mocha + +RUN pip install --upgrade pip + +## tests +COPY /tests/unit/requirements.txt /root/test_requirements.txt +RUN pip install --no-cache-dir -r /root/test_requirements.txt + +## machine and lab container +COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt +RUN pip install --no-cache-dir -r /root/lab_requirements.txt + +COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt +RUN pip install --no-cache-dir -r /root/mach_requirements.txt + +### Bill's surprise fork +RUN pip install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 + + +# install lab/node_modules to an anon volume +WORKDIR /appsrc/lab +COPY lab/package.json /appsrc/lab/ +RUN dos2unix /appsrc/lab/package.json +RUN npm install --silent --progress=false + +# install lab/webapp/node_modules to an anon volume +WORKDIR /appsrc/lab/webapp +COPY lab/webapp/package.json /appsrc/lab/webapp/ +RUN dos2unix /appsrc/lab/webapp/package.json +RUN npm install --silent --progress=false From b63c2f584654924dd59eced24f89a88e8193c7ea Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Sun, 24 Jan 2021 14:40:44 -0500 Subject: [PATCH 10/11] Make new test docker-compose files point to the right Dockerfiles --- docker-compose-int-test-rpi.yml | 6 +++--- docker-compose-unit-test-rpi.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose-int-test-rpi.yml b/docker-compose-int-test-rpi.yml index 380a0e168..666a42daf 100644 --- a/docker-compose-int-test-rpi.yml +++ b/docker-compose-int-test-rpi.yml @@ -21,7 +21,7 @@ services: lab: build: context: . - dockerfile: docker/lab/Dockerfile + dockerfile: docker/lab/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -43,7 +43,7 @@ services: machine: build: context: . - dockerfile: docker/machine/Dockerfile + dockerfile: docker/machine/rpi.Dockerfile tty: true stdin_open: true volumes: @@ -62,7 +62,7 @@ services: dbmongo: build: context: . - dockerfile: docker/dbmongo/Dockerfile + dockerfile: docker/dbmongo/rpi.Dockerfile tty: true stdin_open: true ports: diff --git a/docker-compose-unit-test-rpi.yml b/docker-compose-unit-test-rpi.yml index 1993eac34..b2cf26966 100644 --- a/docker-compose-unit-test-rpi.yml +++ b/docker-compose-unit-test-rpi.yml @@ -4,7 +4,7 @@ services: unit_tester: build: context: . - dockerfile: tests/unit/Dockerfile + dockerfile: tests/unit/rpi.Dockerfile working_dir: /appsrc command: bash -c "dos2unix tests/unit/unit_test_runner.sh && sh tests/unit/unit_test_runner.sh" tty: true From 6157f68666d57632f83a685ab5521145d6c923fc Mon Sep 17 00:00:00 2001 From: Joe Romano Date: Mon, 25 Jan 2021 14:24:22 -0500 Subject: [PATCH 11/11] Pass all unit tests on Raspberry Pi --- tests/unit/requirements-rpi.txt | 4 ++++ tests/unit/rpi.Dockerfile | 39 ++++++++++++++++++++++++--------- 2 files changed, 33 insertions(+), 10 deletions(-) create mode 100644 tests/unit/requirements-rpi.txt diff --git a/tests/unit/requirements-rpi.txt b/tests/unit/requirements-rpi.txt new file mode 100644 index 000000000..a7fa3f931 --- /dev/null +++ b/tests/unit/requirements-rpi.txt @@ -0,0 +1,4 @@ +coverage==5.3 +nose==1.3.7 +nose-htmloutput==0.6.0 +parameterized==0.7.0 \ No newline at end of file diff --git a/tests/unit/rpi.Dockerfile b/tests/unit/rpi.Dockerfile index f04fee9e6..0b951e24b 100644 --- a/tests/unit/rpi.Dockerfile +++ b/tests/unit/rpi.Dockerfile @@ -1,34 +1,53 @@ -FROM python:3.7.4-stretch +FROM ubuntu:bionic + +RUN apt-get update && apt-get install -y wget xz-utils curl #nodejs -RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-x64.tar.xz -O ~/node.tar.xz && \ +RUN wget --quiet https://nodejs.org/dist/v11.14.0/node-v11.14.0-linux-arm64.tar.xz -O ~/node.tar.xz && \ tar -xvf ~/node.tar.xz -C /opt/ && \ rm ~/node.tar.xz -ENV PATH /opt/node-v11.14.0-linux-x64/bin:$PATH +ENV PATH /opt/node-v11.14.0-linux-arm64/bin:$PATH RUN apt-get update --fix-missing \ && apt-get install -y --no-install-recommends \ - graphviz dos2unix && \ + graphviz dos2unix python3-numpy libatlas-base-dev gfortran libgfortran5 && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Ensure specific python version +# See: https://stackoverflow.com/a/58562728/1730417 +# and: https://askubuntu.com/a/1176271/260220 +RUN apt-get update && apt-get install -y software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt-get update +RUN apt-get install -y python3.7 python3.7-dev python3.7-distutils +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.7 1 +RUN update-alternatives --set python /usr/bin/python3.7 +RUN curl -s https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py --force-reinstall && \ + rm get-pip.py + RUN npm install -g mocha RUN pip install --upgrade pip ## tests -COPY /tests/unit/requirements.txt /root/test_requirements.txt +COPY /tests/unit/requirements-rpi.txt /root/test_requirements.txt RUN pip install --no-cache-dir -r /root/test_requirements.txt ## machine and lab container -COPY /docker/lab/files/requirements.txt /root/lab_requirements.txt -RUN pip install --no-cache-dir -r /root/lab_requirements.txt +RUN mkdir /root/wheel +COPY /docker/pennai-arm64-deps/lab/wheel/*.whl /root/wheel/ +COPY /docker/pennai-arm64-deps/machine/wheel/*.whl /root/wheel/ +COPY /docker/lab/files/requirements-raspberrypi.txt /root/lab_requirements.txt +RUN pip3 install --no-index --find-links=/root/wheel -r /root/lab_requirements.txt -COPY /docker/machine/files/requirements.txt /root/mach_requirements.txt -RUN pip install --no-cache-dir -r /root/mach_requirements.txt +COPY /docker/machine/files/requirements-raspberrypi.txt /root/mach_requirements.txt +RUN pip install --no-index --find-links=/root/wheel -r /root/mach_requirements.txt ### Bill's surprise fork -RUN pip install --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +#RUN pip install --verbose --verbose --verbose --no-cache-dir git+https://github.com/lacava/surprise.git@1.0.8.3 +RUN pip install --no-index --find-links=/root/wheel scikit_surprise # install lab/node_modules to an anon volume