From 3a991f3ad1478ab5233b53e10daad2088673080a Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Mon, 25 Mar 2024 15:26:40 -0300 Subject: [PATCH] Update ARM foundation image and minor improvements --- DockerfileLeanFoundation | 20 +-- DockerfileLeanFoundationARM | 255 +++++++++++++--------------- Tests/Python/PythonPackagesTests.cs | 21 ++- 3 files changed, 140 insertions(+), 156 deletions(-) diff --git a/DockerfileLeanFoundation b/DockerfileLeanFoundation index 099299b3e1b0..7e6158597781 100644 --- a/DockerfileLeanFoundation +++ b/DockerfileLeanFoundation @@ -90,6 +90,7 @@ RUN pip install --no-cache-dir \ gym==0.26.2 \ ipywidgets==8.1.2 \ deap==1.4.1 \ + pykalman==0.9.7 \ cvxpy==1.4.2 \ pyportfolioopt==1.5.5 \ pmdarima==2.0.4 \ @@ -119,7 +120,7 @@ RUN pip install --no-cache-dir \ seaborn==0.13.2 \ optuna==3.5.0 \ findiff==0.10.0 \ - sktime==0.27.0 \ + sktime==0.27.1 \ hyperopt==0.2.7 \ bayesian-optimization==1.4.3 \ pingouin==0.5.4 \ @@ -192,7 +193,7 @@ RUN pip install --no-cache-dir \ DoubleML==0.7.1 \ jupyter-bokeh==4.0.0 \ imbalanced-learn==0.12.0 \ - openai==1.14.0 \ + openai==1.14.3 \ lazypredict-nightly==0.3.0 \ darts==0.28.0 \ fastparquet==2024.2.0 \ @@ -231,7 +232,10 @@ RUN pip install --no-cache-dir \ jupyter-ai==2.12.0 \ keras-tcn==3.5.0 \ neuralprophet[live]==0.8.0 \ - Riskfolio-Lib==6.0.0 + Riskfolio-Lib==6.0.0 \ + fuzzy-c-means==1.7.2 \ + EMD-signal==1.6.0 \ + dask[complete]==2024.3.1 RUN conda install -c conda-forge -y cudatoolkit=11.8.0 && conda install -c nvidia -y cuda-compiler=12.2.2 && conda clean -y --all ENV XLA_FLAGS=--xla_gpu_cuda_data_dir=/opt/miniconda3/ @@ -284,16 +288,6 @@ RUN wget -q https://cdn.quantconnect.com/ta-lib/ta-lib-0.4.0-src.tar.gz && \ cd .. && rm -rf ta-lib && rm ta-lib-0.4.0-src.tar.gz && \ pip install --no-cache-dir TA-Lib==0.4.28 -# Install PyEMD -RUN wget -q https://cdn.quantconnect.com/pyemd/PyEMD-master-9f42c57.zip && \ - unzip -q PyEMD-master-9f42c57.zip && cd PyEMD-master && \ - pip install . && cd .. && rm -rf PyEMD-master && rm PyEMD-master-9f42c57.zip - -# Install fuzzy-c-means -RUN wget -q https://cdn.quantconnect.com/fuzzy-c-means/fuzzy-c-means-master-0c4b407.zip && \ - unzip -q fuzzy-c-means-master-0c4b407.zip && cd fuzzy-c-means-master && \ - pip install . && cd .. && rm -rf fuzzy-c-means-master && rm fuzzy-c-means-master-0c4b407.zip - RUN echo "{\"argv\":[\"python\",\"-m\",\"ipykernel_launcher\",\"-f\",\"{connection_file}\"],\"display_name\":\"Foundation-Py-Default\",\"language\":\"python\",\"metadata\":{\"debugger\":true}}" > /opt/miniconda3/share/jupyter/kernels/python3/kernel.json # Install wkhtmltopdf and xvfb to support HTML to PDF conversion of reports diff --git a/DockerfileLeanFoundationARM b/DockerfileLeanFoundationARM index ebb437f73eef..fa4fbd6cb770 100644 --- a/DockerfileLeanFoundationARM +++ b/DockerfileLeanFoundationARM @@ -12,7 +12,7 @@ CMD ["/sbin/my_init"] # Install OS Packages: # Misc tools for running Python.NET and IB inside a headless container. RUN add-apt-repository ppa:ubuntu-toolchain-r/test && apt-get update \ - && apt-get install -y git libgtk2.0.0 cmake bzip2 curl unzip wget python3-pip python3-opengl zlib1g-dev \ + && apt-get install -y git libgtk2.0.0 bzip2 curl unzip wget python3-pip python3-opengl zlib1g-dev \ xvfb libxrender1 libxtst6 libxi6 libglib2.0-dev libopenmpi-dev libstdc++6 openmpi-bin \ r-base pandoc libcurl4-openssl-dev \ openjdk-11-jdk openjdk-11-jre bbe \ @@ -29,10 +29,10 @@ RUN wget https://dot.net/v1/dotnet-install.sh && \ ENV DOTNET_ROOT="/root/.dotnet" # Set PythonDLL variable for PythonNet -ENV PYTHONNET_PYDLL="/opt/miniconda3/lib/libpython3.8.so" +ENV PYTHONNET_PYDLL="/opt/miniconda3/lib/libpython3.11.so" # Install miniconda -ENV CONDA="Miniconda3-py38_23.1.0-1-Linux-aarch64.sh" +ENV CONDA="Miniconda3-py311_24.1.2-0-Linux-aarch64.sh" ENV PATH="/opt/miniconda3/bin:${PATH}" RUN wget -q https://cdn.quantconnect.com/miniconda/${CONDA} && \ bash ${CONDA} -b -p /opt/miniconda3 && rm -rf ${CONDA} @@ -47,175 +47,171 @@ RUN apt-get update && apt-get install -y alien dpkg-dev debhelper build-essentia ENV PIP_DEFAULT_TIMEOUT=120 # Install numpy first to avoid it not being resolved when installing libraries that depend on it next -RUN pip install --no-cache-dir numpy==1.23.5 +RUN pip install --no-cache-dir numpy==1.26.4 + +# Install newer (than provided by ubuntu) cmake required by scikit build process +RUN conda install -c conda-forge cmake==3.28.4 && conda clean -y --all # The list of packages in this image is shorter than the list in the AMD images # This list only includes packages that can be installed within 2 minutes on ARM RUN pip install --no-cache-dir \ - cython==0.29.36 \ - pandas==1.5.3 \ - scipy==1.10.1 \ - numpy==1.23.5 \ - wrapt==1.14.1 \ - astropy==5.2.2 \ - beautifulsoup4==4.12.2 \ - dill==0.3.7 \ - jsonschema==4.19.1 \ - lxml==4.9.3 \ - msgpack==1.0.7 \ - numba==0.56.4 \ - xarray==2023.1.0 \ - plotly==5.17.0 \ - jupyterlab==3.4.4 \ - tensorflow==2.13.1 \ + cython==3.0.9 \ + pandas==2.1.4 \ + scipy==1.11.4 \ + numpy==1.26.4 \ + wrapt==1.16.0 \ + astropy==6.0.0 \ + beautifulsoup4==4.12.3 \ + dill==0.3.8 \ + jsonschema==4.21.1 \ + lxml==5.1.0 \ + msgpack==1.0.8 \ + numba==0.59.0 \ + xarray==2024.2.0 \ + plotly==5.20.0 \ + jupyterlab==4.1.5 \ + tensorflow==2.16.1 \ docutils==0.20.1 \ gensim==4.3.2 \ - keras==2.13.1 \ - lightgbm==4.1.0 \ - mpi4py==3.1.5 \ + keras==3.0.5 \ + lightgbm==4.3.0 \ nltk==3.8.1 \ graphviz==0.20.1 \ - cmdstanpy==1.2.0 \ - copulae==0.7.8 \ - featuretools==1.27.0 \ - PuLP==2.7.0 \ - pymc==5.6.1 \ + cmdstanpy==1.2.1 \ + copulae==0.7.9 \ + featuretools==1.30.0 \ + PuLP==2.8.0 \ + pymc==5.10.4 \ rauth==0.7.3 \ - scikit-learn==1.3.2 \ - scikit-multiflow==0.5.3 \ - scikit-optimize==0.9.0 \ - aesara==2.9.2 \ - tsfresh==0.20.1 \ - tslearn==0.6.2 \ + scikit-learn==1.4.1.post1 \ + scikit-optimize==0.10.0 \ + aesara==2.9.3 \ + tsfresh==0.20.2 \ + tslearn==0.6.3 \ tweepy==4.14.0 \ - PyWavelets==1.4.1 \ - umap-learn==0.5.3 \ - fastai==2.7.13 \ - arch==5.6.0 \ - copulas==0.9.2 \ + PyWavelets==1.5.0 \ + umap-learn==0.5.5 \ + fastai==2.7.14 \ + arch==6.3.0 \ + copulas==0.10.1 \ cufflinks==0.17.3 \ gym==0.26.2 \ - ipywidgets==8.1.1 \ + ipywidgets==8.1.2 \ deap==1.4.1 \ - cvxpy==1.4.1 \ - pykalman==0.9.5 \ - pyro-ppl==1.8.6 \ + pykalman==0.9.7 \ + cvxpy==1.4.2 \ + pyro-ppl==1.9.0 \ sklearn-json==0.1.0 \ - dtw-python==1.3.0 \ - gluonts==0.13.7 \ + dtw-python==1.3.1 \ + gluonts==0.14.4 \ gplearn==0.4.2 \ - jax==0.4.12 \ - pennylane==0.32.0 \ - PennyLane-Lightning==0.32.0 \ - pennylane-qiskit==0.32.0 \ + jax==0.4.25 \ + pennylane==0.35.1 \ + PennyLane-Lightning==0.35.1 \ + pennylane-qiskit==0.35.1 \ mplfinance==0.12.10b0 \ - hmmlearn==0.3.0 \ - ta==0.10.2 \ - seaborn==0.13.0 \ - optuna==3.4.0 \ + hmmlearn==0.3.2 \ + ta==0.11.0 \ + seaborn==0.13.2 \ + optuna==3.5.0 \ findiff==0.10.0 \ - sktime==0.24.0 \ + sktime==0.27.1 \ hyperopt==0.2.7 \ bayesian-optimization==1.4.3 \ - matplotlib==3.7.3 \ + matplotlib==3.7.5 \ sdeint==0.3.0 \ - pandas_market_calendars==4.3.1 \ - ruptures==1.1.8 \ - simpy==4.0.2 \ + pandas_market_calendars==4.4.0 \ + ruptures==1.1.9 \ + simpy==4.1.1 \ scikit-learn-extra==0.3.0 \ - ray==2.7.1 \ - "ray[tune]"==2.7.1 \ - "ray[rllib]"==2.7.1 \ + ray==2.9.3 \ + "ray[tune]"==2.9.3 \ + "ray[rllib]"==2.9.3 \ fastText==0.9.2 \ - h2o==3.44.0.1 \ + h2o==3.46.0.1 \ prophet==1.1.5 \ - Riskfolio-Lib==4.0.3 \ - torch==2.1.0 \ - torchvision==0.16.0 \ - ax-platform==0.3.3 \ + Riskfolio-Lib==6.0.0 \ + torch==2.2.1 \ + torchvision==0.17.1 \ + ax-platform==0.3.7 \ alphalens-reloaded==0.4.3 \ pyfolio-reloaded==0.9.5 \ - altair==5.1.2 \ - stellargraph==1.2.1 \ - modin==0.22.3 \ - persim==0.3.1 \ - ripser==0.6.4 \ - pydmd==0.4.1.post2308 \ - EMD-signal==1.5.2 \ - spacy==3.7.2 \ + altair==5.2.0 \ + modin==0.26.1 \ + persim==0.3.5 \ + ripser==0.6.8 \ + pydmd==1.0.0 \ + EMD-signal==1.6.0 \ + spacy==3.7.4 \ pandas-ta==0.3.14b \ - pytorch-ignite==0.4.12 \ + pytorch-ignite==0.4.13 \ tensorly==0.8.1 \ - mlxtend==0.23.0 \ - shap==0.43.0 \ + mlxtend==0.23.1 \ + shap==0.45.0 \ lime==0.2.0.1 \ mpmath==1.3.0 \ - polars==0.19.11 \ - stockstats==0.5.4 \ + polars==0.20.15 \ + stockstats==0.6.2 \ QuantStats==0.0.62 \ hurst==0.0.5 \ - numerapi==2.16.1 \ + numerapi==2.18.0 \ pymdptoolbox==4.0-b3 \ - panel==1.2.3 \ - hvplot==0.9.0 \ + panel==1.3.8 \ + hvplot==0.9.2 \ py-heat==0.0.6 \ py-heat-magic==0.0.2 \ - bokeh==3.1.1 \ - river==0.14.0 \ + bokeh==3.3.4 \ + river==0.21.0 \ stumpy==1.12.0 \ - pyvinecopulib==0.6.3 \ + pyvinecopulib==0.6.4 \ ijson==3.2.3 \ - jupyter-resource-usage==0.7.2 \ + jupyter-resource-usage==1.0.2 \ injector==0.21.0 \ openpyxl==3.1.2 \ xlrd==2.0.1 \ - mljar-supervised==1.0.2 \ + mljar-supervised==1.1.6 \ dm-tree==0.1.8 \ - lz4==4.3.2 \ - ortools==9.6.2534 \ + lz4==4.3.3 \ + ortools==9.9.3963 \ py_vollib==1.0.1 \ thundergbm==0.3.17 \ yellowbrick==1.5 \ livelossplot==0.5.5 \ gymnasium==0.28.1 \ - interpret==0.4.4 \ - DoubleML==0.7.0 \ - jupyter-bokeh==3.0.7 \ - imbalanced-learn==0.11.0 \ - scikeras==0.12.0 \ - openai==1.3.5 \ - lazypredict==0.2.12 \ - fracdiff==0.9.0 \ - darts==0.24.0 \ - fastparquet==2023.8.0 \ - tables==3.8.0 \ - dimod==0.12.3 \ - dwave-samplers==1.0.0 \ + interpret==0.5.1 \ + DoubleML==0.7.1 \ + jupyter-bokeh==4.0.0 \ + imbalanced-learn==0.12.0 \ + openai==1.14.3 \ + lazypredict-nightly==0.3.0 \ + darts==0.28.0 \ + fastparquet==2024.2.0 \ + tables==3.9.2 \ + dimod==0.12.14 \ + dwave-samplers==1.2.0 \ python-statemachine==2.1.2 \ pymannkendall==1.4.3 \ - Pyomo==6.6.2 \ - gpflow==2.9.0 \ - pyarrow==13.0.0 \ - dwave-ocean-sdk==6.1.1 \ + Pyomo==6.7.1 \ + gpflow==2.9.1 \ + pyarrow==15.0.1 \ + dwave-ocean-sdk==6.9.0 \ chardet==5.2.0 \ - stable-baselines3==2.1.0 \ + stable-baselines3==2.2.1 \ Shimmy==1.3.0 \ FixedEffectModel==0.0.5 \ - transformers==4.34.0 \ - langchain==0.0.341 \ - tensorflow-ranking==0.5.3 \ - pomegranate==1.0.3 \ - tigramite==5.2.3.1 \ - MAPIE==0.7.0 \ - mlforecast==0.9.3 \ - x-transformers==1.26.0 \ - Werkzeug==2.3.8 + transformers==4.38.2 \ + langchain==0.1.12 \ + pomegranate==1.0.4 \ + MAPIE==0.8.3 \ + mlforecast==0.12.0 \ + x-transformers==1.27.19 \ + Werkzeug==3.0.1 # Install dwave tool RUN dwave install --all -y # Install 'ipopt' solver for 'Pyomo' -RUN conda install -c conda-forge ipopt==3.14.13 \ +RUN conda install -c conda-forge ipopt==3.14.14 \ && conda clean -y --all # We install need to install separately else fails to find numpy @@ -224,8 +220,8 @@ RUN pip install --no-cache-dir iisignature==0.24 # Install spacy models RUN python -m spacy download en_core_web_md && python -m spacy download en_core_web_sm -RUN conda install -y -c conda-forge \ - openmpi=4.1.6 \ +RUN conda config --set solver classic && conda install -y -c conda-forge \ + openmpi=5.0.2 \ && conda clean -y --all # Install nltk data @@ -234,16 +230,6 @@ RUN python -m nltk.downloader -d /usr/share/nltk_data punkt && \ python -m nltk.downloader -d /usr/share/nltk_data stopwords && \ python -m nltk.downloader -d /usr/share/nltk_data wordnet -# Install ppscore -RUN wget -q https://cdn.quantconnect.com/ppscore/ppscore-master-ce93fa3.zip && \ - unzip -q ppscore-master-ce93fa3.zip && cd ppscore-master && \ - pip install . && cd .. && rm -rf ppscore-master && rm ppscore-master-ce93fa3.zip - -# Install DX Analytics -RUN wget -q https://cdn.quantconnect.com/dx/dx-master-69922c0.zip && \ - unzip -q dx-master-69922c0.zip && cd dx-master && \ - pip install . && cd .. && rm -rf dx-master && rm dx-master-69922c0.zip - # Install Pyrb RUN wget -q https://cdn.quantconnect.com/pyrb/pyrb-master-250054e.zip && \ unzip -q pyrb-master-250054e.zip && cd pyrb-master && \ @@ -254,17 +240,6 @@ RUN wget -q https://cdn.quantconnect.com/ssm/ssm-master-646e188.zip && \ unzip -q ssm-master-646e188.zip && cd ssm-master && \ pip install . && cd .. && rm -rf ssm-master && rm ssm-master-646e188.zip -# Due to conflicts install 'pomegranate' virtual environment package -RUN python -m venv /Foundation-Pomegranate --system-site-packages && . /Foundation-Pomegranate/bin/activate \ - && pip install --no-cache-dir \ - pomegranate==0.14.8 \ - mxnet==1.9.1 \ - nbeats-keras==1.8.0 \ - nbeats-pytorch==1.8.0 \ - neuralprophet[live]==0.6.2 \ - && python -m ipykernel install --name=Foundation-Pomegranate \ - && deactivate - RUN echo "{\"argv\":[\"python\",\"-m\",\"ipykernel_launcher\",\"-f\",\"{connection_file}\"],\"display_name\":\"Foundation-Py-Default\",\"language\":\"python\",\"metadata\":{\"debugger\":true}}" > /opt/miniconda3/share/jupyter/kernels/python3/kernel.json # Install wkhtmltopdf and xvfb to support HTML to PDF conversion of reports diff --git a/Tests/Python/PythonPackagesTests.cs b/Tests/Python/PythonPackagesTests.cs index b13d5cd27235..06722fb506d4 100644 --- a/Tests/Python/PythonPackagesTests.cs +++ b/Tests/Python/PythonPackagesTests.cs @@ -56,10 +56,10 @@ import control public void PyCaret() { AssertCode(@" -def RunTest(): - from pycaret.datasets import get_data - from pycaret.classification import * +from pycaret.datasets import get_data +from pycaret.classification import setup +def RunTest(): data = get_data('diabetes') s = setup(data, target = 'Class variable', session_id = 123)"); } @@ -1017,6 +1017,21 @@ def RunTest(): ); } + [Test] + public void PykalmanTest() + { + AssertCode( + @" +import numpy +from pykalman import KalmanFilter +def RunTest(): + kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]]) + measurements = numpy.asarray([[1,0], [0,0], [0,1]]) # 3 observations + kf = kf.em(measurements, n_iter=5) + return kf.filter(measurements)" + ); + } + [Test] public void AesaraTest() {