From 12e3bc883fae3c8e7c22ffb04cf9a320c896da68 Mon Sep 17 00:00:00 2001 From: lucasew Date: Sun, 28 May 2023 11:27:08 -0300 Subject: [PATCH 01/10] checkpoint Signed-off-by: lucasew --- flake.lock | 18 ++++++++++ flake.nix | 5 +++ projects/nerfstudio/default.nix | 41 +++++++++++++++++++++++ projects/nerfstudio/package.nix | 58 +++++++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+) create mode 100644 projects/nerfstudio/default.nix create mode 100644 projects/nerfstudio/package.nix diff --git a/flake.lock b/flake.lock index 299f98e2..f0765892 100644 --- a/flake.lock +++ b/flake.lock @@ -150,6 +150,23 @@ "type": "github" } }, + "nerfstudio-src": { + "flake": false, + "locked": { + "lastModified": 1684777532, + "narHash": "sha256-9X/DyxSe4isIjlFmgTnk6brayqe/GKPzOcRcKcHWdBg=", + "owner": "nerfstudio-project", + "repo": "nerfstudio", + "rev": "bd04ceeb503658bbf0439261a6148fe26fdefa48", + "type": "github" + }, + "original": { + "owner": "nerfstudio-project", + "ref": "v0.3.1", + "repo": "nerfstudio", + "type": "github" + } + }, "nix-darwin": { "inputs": { "nixpkgs": [ @@ -251,6 +268,7 @@ "hercules-ci-effects": "hercules-ci-effects", "invokeai-src": "invokeai-src", "koboldai-src": "koboldai-src", + "nerfstudio-src": "nerfstudio-src", "nixpkgs": "nixpkgs_2" } } diff --git a/flake.nix b/flake.nix index 58a8d014..1931b770 100644 --- a/flake.nix +++ b/flake.nix @@ -18,6 +18,10 @@ url = "github:koboldai/koboldai-client/1.19.2"; flake = false; }; + nerfstudio-src = { + url = "github:nerfstudio-project/nerfstudio/v0.3.1"; + flake = false; + }; flake-parts = { url = "github:hercules-ci/flake-parts"; inputs.nixpkgs-lib.follows = "nixpkgs"; @@ -38,6 +42,7 @@ ./modules/aipython3 ./projects/invokeai ./projects/koboldai + ./projects/nerfstudio ./website ]; }; diff --git a/projects/nerfstudio/default.nix b/projects/nerfstudio/default.nix new file mode 100644 index 00000000..ff51ff90 --- /dev/null +++ b/projects/nerfstudio/default.nix @@ -0,0 +1,41 @@ +{ config, inputs, lib, withSystem, ... }: +{ + perSystem = { config, pkgs, ... }: let + inherit (config.dependencySets) aipython3-amd aipython3-nvidia; + + src = inputs.nerfstudio-src; + + mkNerfstudioVariant = args: pkgs.callPackage ./package.nix ({ inherit src; } // args); + in { + packages = { + nerfstudio-amd = mkNerfstudioVariant { + aipython3 = aipython3-amd; + }; + nerfstudio-nvidia = mkNerfstudioVariant { + aipython3 = aipython3-nvidia; + }; + }; + }; + + flake.nixosModules = let + packageModule = pkgAttrName: { pkgs, ... }: { + services.nerfstudio.package = withSystem pkgs.system ( + { config, ... }: lib.mkOptionDefault config.packages.${pkgAttrName} + ); + }; + in { + invokeai = ./nixos; + invokeai-amd = { + imports = [ + config.flake.nixosModules.nerfstudio + (packageModule "nerfstudio-amd") + ]; + }; + invokeai-nvidia = { + imports = [ + config.flake.nixosModules.nerfstudio + (packageModule "nerfstudio-nvidia") + ]; + }; + }; +} diff --git a/projects/nerfstudio/package.nix b/projects/nerfstudio/package.nix new file mode 100644 index 00000000..3a98a7d4 --- /dev/null +++ b/projects/nerfstudio/package.nix @@ -0,0 +1,58 @@ +{ aipython3 +# misc +, lib +, src +# extra deps +, libdrm +}: + +let + getVersion = lib.flip lib.pipe [ + (src: builtins.readFile "${src}/pyproject.toml") + (builtins.match ".*version = \"([^\"]+)\".*") + builtins.head + ]; +in + +aipython3.buildPythonPackage { + pname = "nerfstudio"; + format = "pyproject"; + + version = getVersion src; + inherit src; + + propagatedBuildInputs = with aipython3; [ + setuptools + ]; + + makeWrapperArgs = [ + '' --run ' + if [ -d "/usr/lib/wsl/lib" ] + then + echo "Running via WSL (Windows Subsystem for Linux), setting LD_LIBRARY_PATH=/usr/lib/wsl/lib" + set -x + export LD_LIBRARY_PATH="/usr/lib/wsl/lib" + set +x + fi + ' + '' + ] ++ lib.optionals (aipython3.torch.rocmSupport or false) [ + '' --run ' + if [ ! -e /tmp/nix-pytorch-rocm___/amdgpu.ids ] + then + mkdir -p /tmp/nix-pytorch-rocm___ + ln -s ${libdrm}/share/libdrm/amdgpu.ids /tmp/nix-pytorch-rocm___/amdgpu.ids + fi + ' + '' + # See note about consumer GPUs: + # https://docs.amd.com/bundle/ROCm-Deep-Learning-Guide-v5.4.3/page/Troubleshooting.html + " --set-default HSA_OVERRIDE_GFX_VERSION 10.3.0" + ]; + + meta = { + description = "All-in-one repository for state-of-the-art NeRFs"; + homepage = "https://github.com/nerfstudio-project/nerfstudio"; + maintainers = [ lib.maintainers.lucasew ]; + }; +} From eb66bf67d3ef3538bc917a6e0c6ec64c3f6b0cd6 Mon Sep 17 00:00:00 2001 From: lucasew Date: Mon, 29 May 2023 15:02:53 -0300 Subject: [PATCH 02/10] checkpoint Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 5 + packages/nerfacc/default.nix | 68 ++++++++++++ packages/open3d/default.nix | 184 ++++++++++++++++++++++++++++++++ packages/pymeshlab/default.nix | 37 +++++++ packages/tyro/default.nix | 25 +++++ packages/viser/default.nix | 34 ++++++ projects/nerfstudio/package.nix | 15 +++ 7 files changed, 368 insertions(+) create mode 100644 packages/nerfacc/default.nix create mode 100644 packages/open3d/default.nix create mode 100644 packages/pymeshlab/default.nix create mode 100644 packages/tyro/default.nix create mode 100644 packages/viser/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 02ad03e9..4ecd2dd0 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -82,6 +82,11 @@ pkgs: { clip-anytorch = callPackage ../../packages/clip-anytorch { }; clean-fid = callPackage ../../packages/clean-fid { }; getpass-asterisk = callPackage ../../packages/getpass-asterisk { }; + pymeshlab = callPackage ../../packages/pymeshlab { }; + viser = callPackage ../../packages/viser { }; + tyro = callPackage ../../packages/tyro { }; + nerfacc = callPackage ../../packages/nerfacc { }; + open3d = callPackage ../../packages/open3d { }; }; torchRocm = final: prev: rec { diff --git a/packages/nerfacc/default.nix b/packages/nerfacc/default.nix new file mode 100644 index 00000000..9c428ace --- /dev/null +++ b/packages/nerfacc/default.nix @@ -0,0 +1,68 @@ +{ buildPythonPackage +, fetchPypi +, lib +, torch +, which +, hip +, rocthrust +, rocrand +, hipsparse +, hipblas +, rocblas +, cudaPackages +, gcc11Stdenv +, runCommand +, ninja +, rich +, typing-extensions +}: + +let + isRocm = torch.rocmSupport or false; + package = buildPythonPackage rec { + pname = "nerfacc"; + version = "0.5.2"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-kEl3g7G4Rt62x/BwRSDqezSCo2PzouI+M7326LzmvaU="; + }; + + preConfigure = '' + export HOME=$(mktemp -d) + ''; + + enableParallelBuilding = true; + + + TORCH_CUDA_ARCH_LIST = "3.5;5.2;6.0;6.1;7.0+PTX"; + + nativeBuildInputs = [ which ninja ] + ++ (if isRocm + then [ + hip + ] else [ + cudaPackages.cudatoolkit + ] + ); + + buildInputs = [ ] + ++ (if isRocm + then [ + rocthrust + rocrand + hipsparse + (runCommand "rocblas-link" {} "mkdir $out -p; ln -s ${rocblas}/include/rocblas $out/include") + ] + else [ + cudaPackages.cuda_cudart + ]) + ; + + propagatedBuildInputs = [ torch rich typing-extensions ]; + }; +in package.overrideDerivation (old: { + stdenv = gcc11Stdenv; +}) + + diff --git a/packages/open3d/default.nix b/packages/open3d/default.nix new file mode 100644 index 00000000..7815358e --- /dev/null +++ b/packages/open3d/default.nix @@ -0,0 +1,184 @@ +{ buildPythonPackage +, lib +, fetchFromGitHub +, cmake +, assimp +, boringssl +, cudaPackages +, curl +, eigen +, gitMinimal +, glfw +, glew +, ispc +, jsoncpp +, libcxx +, libcxxabi +, libjpeg +, nanoflann +, nasm +, ninja +, pybind11 +, torch +, vulkan-headers +, vulkan-loader +, xorg +, tinygltf +, tinyobjloader +, qhull +, fmt +, imgui +, python-lzf +, python +, fetchurl +, msgpack +, vtk +, tbb +, mkl +, tensorflow +, useOldCXXAbi +, gtest +, librealsense +, zeromq +, pkg-config +, zlib +}: + +let + isRocm = torch.rocmSupport or false; +in + +buildPythonPackage rec { + pname = "open3d"; + version = "0.17.0"; + + format = "other"; + + src = fetchFromGitHub { + owner = "isl-org"; + repo = "Open3D"; + rev = "v${version}"; + sha256 = "sha256-dGdDnHch71O7wAbK8Sg+0uH0p99avUtrG/lFmpsx45Y="; + }; + + prePatch = '' + substituteAllInPlace 3rdparty/uvatlas/uvatlas.cmake \ + --replace '${"$"}{INSTALL_DIR}/' + # --replace '${"$"}{Open3D_INSTALL_LIB_DIR}/${"$"}{CMAKE_STATIC_LIBRARY_PREFIX}uvatlas' "" + cat 3rdparty/uvatlas/uvatlas.cmake + ''; + + cmakeFlags = [ + "-DOPEN3D_VERSION=${version}" + "-DCMAKE_ISPC_COMPILER=${ispc}/bin/ispc" + "-DCPP_LIBRARY=${libcxx}/lib" + "-DVULKAN_INCLUDE_DIR=${vulkan-headers}/include" + "-Dmsgpack_DIR=${msgpack}" + # "-DBORINGSSL_ROOT_DIR=${boringssl}" + # "-DImGui_LIBRARY=${imgui}/lib" + # "-Dliblzf_DIR=${python-lzf}" + # "-DTensorflow_CXX11_ABI=OFF" + # "-DPytorch_CXX11_ABI=OFF" + "-DBUILD_COMMON_CUDA_ARCHS=ON" + "-DGLIBCXX_USE_CXX11_ABI=ON" + "-DBUILD_SHARED_LIBS=ON" + "-DUSE_SYSTEM_ASSIMP=ON" + "-DUSE_SYSTEM_CURL=ON" + "-DUSE_SYSTEM_EIGEN3=ON" + "-DUSE_SYSTEM_FILAMENT=ON" + "-DUSE_SYSTEM_FMT=ON" + "-DUSE_SYSTEM_GLEW=ON" + "-DUSE_SYSTEM_GLFW=ON" + "-DUSE_SYSTEM_GOOGLETEST=ON" + "-DUSE_SYSTEM_IMGUI=ON" + "-DUSE_SYSTEM_JPEG=ON" + "-DUSE_SYSTEM_JSONCPP=ON" + "-DUSE_SYSTEM_LIBLZF=ON" + "-DUSE_SYSTEM_MSGPACK=ON" + "-DUSE_SYSTEM_NANOFLANN=ON" + "-DUSE_SYSTEM_OPENSSL=ON" + "-DUSE_SYSTEM_PNG=ON" + "-DUSE_SYSTEM_PYBIND11=ON" + "-DUSE_SYSTEM_QHULLCPP=ON" + "-DUSE_SYSTEM_TBB=ON" + "-DUSE_SYSTEM_TINYGLTF=ON" + "-DUSE_SYSTEM_TINYOBJLOADER=ON" + "-DUSE_SYSTEM_VTK=ON" + "-DUSE_SYSTEM_ZEROMQ=ON" + # "-DBUILD_TENSORFLOW_OPS=ON" + # "-DBUILD_PYTORCH_OPS=ON" + "-DBUILD_JUPYTER_EXTENSION=ON" + # "-DBUNDLE_OPEN3D_ML=ON" + # "-GNinja" + ] + ++ (if isRocm + then [ ] + else [ "-DBUILD_CUDA_MODULE=ON" ] + ); + + MESA_TARBALL = fetchurl { + url = "https://github.com/isl-org/open3d_downloads/releases/download/mesa-libgl/mesa_libGL_22.1.4.tar.bz2"; + sha256 = "sha256-VzK/tw6PzHRwGIILyP0xzRhn665aoJuvZUgrQsE01Fo="; + }; + + SPHINX_THEME_TARBALL = fetchurl { + url = "https://github.com/isl-org/open3d_sphinx_theme/archive/c71d2728eb5afd1aeeb20dc27a5a0d42bb402d83.tar.gz"; + sha256 = "sha256-mK+Lf9t1p0KAthh9u1jqYB25eNTz+JVtPYfFnCB4b3M="; + }; + + preConfigure = '' + mkdir -p 3rdparty_downloads/{mesa,open3d_sphinx_theme} + ln -s $MESA_TARBALL 3rdparty_downloads/mesa/mesa_libGL_22.1.4.tar.bz2 + ln -s $SPHINX_THEME_TARBALL 3rdparty_downloads/open3d_sphinx_theme/c71d2728eb5afd1aeeb20dc27a5a0d42bb402d83.tar.gz + ''; + + + buildInputs = [ + assimp + curl + eigen + glfw + glew + jsoncpp + libcxx + libcxxabi + libjpeg + nanoflann + pybind11 + vulkan-loader + tinygltf + tinyobjloader + qhull + fmt + imgui + python-lzf + python + msgpack + vtk + tbb + mkl + tensorflow + torch + librealsense + zeromq + zlib + ] + ++ (if isRocm + then [] + else [ + cudaPackages.cudatoolkit + cudaPackages.cuda_cudart + cudaPackages.cudnn + ] + ); + + nativeBuildInputs = [ + cmake + nasm + gitMinimal + # ninja + useOldCXXAbi + gtest + pkg-config + ]; +} diff --git a/packages/pymeshlab/default.nix b/packages/pymeshlab/default.nix new file mode 100644 index 00000000..6ad29b55 --- /dev/null +++ b/packages/pymeshlab/default.nix @@ -0,0 +1,37 @@ +# WARNING: This file was automatically generated. You should avoid editing it. +# If you run pynixify again, the file will be either overwritten or +# deleted, and you will lose the changes you made to it. + +{ buildPythonPackage +, lib +, fetchPypi +, fetchFromGitHub +, numpy +, pyngrok +, plotly +, jupyterlab +, viser +}: + +buildPythonPackage rec { + pname = "PyMeshLab"; + version = "2022.2.post4"; + + src = fetchFromGitHub { + owner = "cnr-isti-vclab"; + repo = pname; + rev = "v${version}"; + sha256 = "sha256-NmnEp0ThF/YLhuVmjWsscP5l6V8TIrn2I47qC/owQCo="; + }; + + propagatedBuildInputs = [ numpy pyngrok plotly jupyterlab viser ]; + + # TODO FIXME + doCheck = false; + + meta = with lib; { + description = + "A Python interface to MeshLab"; + homepage = "https://github.com/cnr-isti-vclab/PyMeshLab"; + }; +} diff --git a/packages/tyro/default.nix b/packages/tyro/default.nix new file mode 100644 index 00000000..848d7fef --- /dev/null +++ b/packages/tyro/default.nix @@ -0,0 +1,25 @@ +{ buildPythonPackage +, lib +, fetchPypi +, poetry-core +, frozendict +, rich +, typing-extensions +, shtab +, docstring-parser +, pyyaml +}: + +buildPythonPackage rec { + pname = "tyro"; + version = "0.5.3"; + + format = "pyproject"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-ygdNkRr4bjDDHioXoMWPZ1c0IamIkqGyvAuvJx3Bhis="; + }; + + propagatedBuildInputs = [ poetry-core frozendict rich typing-extensions shtab docstring-parser pyyaml ]; +} diff --git a/packages/viser/default.nix b/packages/viser/default.nix new file mode 100644 index 00000000..ecf87246 --- /dev/null +++ b/packages/viser/default.nix @@ -0,0 +1,34 @@ +{ buildPythonPackage +, lib +, fetchPypi +, poetry-core +, tyro +, gdown +, rich +, numpy +, liblzfse +, websockets +, msgpack +, scipy +, scikit-image +, pythonRelaxDepsHook +}: + +buildPythonPackage rec { + pname = "viser"; + version = "0.0.13"; + + format = "pyproject"; + + src = fetchPypi { + inherit pname version; + sha256 = "sha256-PTrr4CjtuD/XyaEp4xhUaZ5u8Ap1Jbkzk5jZ5MdjW7o="; + }; + + nativeBuildInputs = [ poetry-core pythonRelaxDepsHook ]; + + pythonRelaxDeps = [ "gdown" "rich" ]; + pythonRemoveDeps = [ "pre-commit" ]; + + propagatedBuildInputs = [ tyro gdown rich numpy liblzfse websockets msgpack scipy scikit-image ]; +} diff --git a/projects/nerfstudio/package.nix b/projects/nerfstudio/package.nix index 3a98a7d4..3e91bc91 100644 --- a/projects/nerfstudio/package.nix +++ b/projects/nerfstudio/package.nix @@ -4,6 +4,7 @@ , src # extra deps , libdrm +, ninja }: let @@ -21,8 +22,22 @@ aipython3.buildPythonPackage { version = getVersion src; inherit src; + nativeBuildInputs = [ aipython3.pythonRelaxDepsHook ninja ]; + + pythonRemoveDeps = [ "ninja" ]; + propagatedBuildInputs = with aipython3; [ setuptools + matplotlib + pymeshlab + pyquaternion + appdirs + python-socketio + msgpack-numpy + av + nerfacc + ipywidgets + open3d ]; makeWrapperArgs = [ From e49bf41381b8ceb1846493f9ee6665f8ae48a822 Mon Sep 17 00:00:00 2001 From: lucasew Date: Mon, 29 May 2023 16:13:08 -0300 Subject: [PATCH 03/10] checkpoint Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 2 ++ packages/liblzf/default.nix | 14 ++++++++++++ packages/open3d/default.nix | 41 +++++++++++++++++++++------------- 3 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 packages/liblzf/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 4ecd2dd0..8dd287ce 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -87,6 +87,7 @@ pkgs: { tyro = callPackage ../../packages/tyro { }; nerfacc = callPackage ../../packages/nerfacc { }; open3d = callPackage ../../packages/open3d { }; + liblzf = callPackage ../../packages/liblzf { }; }; torchRocm = final: prev: rec { @@ -120,4 +121,5 @@ pkgs: { torch = final.torch-bin; torchvision = final.torchvision-bin; }; + } diff --git a/packages/liblzf/default.nix b/packages/liblzf/default.nix new file mode 100644 index 00000000..cbe28c0a --- /dev/null +++ b/packages/liblzf/default.nix @@ -0,0 +1,14 @@ +{ stdenv +, lib +, fetchurl +}: + +stdenv.mkDerivation rec { + pname = "liblzf"; + version = "3.6"; + + src = fetchurl { + url = "http://dist.schmorp.de/liblzf/liblzf-${version}.tar.gz"; + sha256 = "sha256-nF3gH3ucyuQMP2GdJqer7JmGwGw20mDBec7dBLiftGo="; + }; +} diff --git a/packages/open3d/default.nix b/packages/open3d/default.nix index 7815358e..a8670156 100644 --- a/packages/open3d/default.nix +++ b/packages/open3d/default.nix @@ -15,6 +15,7 @@ , libcxx , libcxxabi , libjpeg +, libpng , nanoflann , nasm , ninja @@ -39,9 +40,11 @@ , useOldCXXAbi , gtest , librealsense +, libsodium , zeromq , pkg-config , zlib +, liblzf }: let @@ -74,6 +77,8 @@ buildPythonPackage rec { "-DCPP_LIBRARY=${libcxx}/lib" "-DVULKAN_INCLUDE_DIR=${vulkan-headers}/include" "-Dmsgpack_DIR=${msgpack}" + "-DTinyGLTF_DIR=${tinygltf}" + "-Dvtk_DIR=${vtk}" # "-DBORINGSSL_ROOT_DIR=${boringssl}" # "-DImGui_LIBRARY=${imgui}/lib" # "-Dliblzf_DIR=${python-lzf}" @@ -116,20 +121,26 @@ buildPythonPackage rec { else [ "-DBUILD_CUDA_MODULE=ON" ] ); - MESA_TARBALL = fetchurl { - url = "https://github.com/isl-org/open3d_downloads/releases/download/mesa-libgl/mesa_libGL_22.1.4.tar.bz2"; - sha256 = "sha256-VzK/tw6PzHRwGIILyP0xzRhn665aoJuvZUgrQsE01Fo="; - }; - - SPHINX_THEME_TARBALL = fetchurl { - url = "https://github.com/isl-org/open3d_sphinx_theme/archive/c71d2728eb5afd1aeeb20dc27a5a0d42bb402d83.tar.gz"; - sha256 = "sha256-mK+Lf9t1p0KAthh9u1jqYB25eNTz+JVtPYfFnCB4b3M="; - }; + downloadDepsHook = let + deps = { + "3rdparty_downloads/mesa/mesa_libGL_22.1.4.tar.bz2" = fetchurl { + url = "https://github.com/isl-org/open3d_downloads/releases/download/mesa-libgl/mesa_libGL_22.1.4.tar.bz2"; + sha256 = "sha256-VzK/tw6PzHRwGIILyP0xzRhn665aoJuvZUgrQsE01Fo="; + }; + "3rdparty_downloads/open3d_sphinx_theme/c71d2728eb5afd1aeeb20dc27a5a0d42bb402d83.tar.gz" = fetchurl { + url = "https://github.com/isl-org/open3d_sphinx_theme/archive/c71d2728eb5afd1aeeb20dc27a5a0d42bb402d83.tar.gz"; + sha256 = "sha256-mK+Lf9t1p0KAthh9u1jqYB25eNTz+JVtPYfFnCB4b3M="; + }; + }; + downloadElement = name: value: '' + mkdir -p $(dirname ${name}) + ln -s ${value} ${name} + ''; + downloadCommands = builtins.mapAttrs (downloadElement) deps; + in builtins.concatStringsSep "\n" (builtins.attrValues downloadCommands); preConfigure = '' - mkdir -p 3rdparty_downloads/{mesa,open3d_sphinx_theme} - ln -s $MESA_TARBALL 3rdparty_downloads/mesa/mesa_libGL_22.1.4.tar.bz2 - ln -s $SPHINX_THEME_TARBALL 3rdparty_downloads/open3d_sphinx_theme/c71d2728eb5afd1aeeb20dc27a5a0d42bb402d83.tar.gz + runHook downloadDepsHook ''; @@ -142,6 +153,7 @@ buildPythonPackage rec { jsoncpp libcxx libcxxabi + libsodium libjpeg nanoflann pybind11 @@ -151,7 +163,7 @@ buildPythonPackage rec { qhull fmt imgui - python-lzf + liblzf python msgpack vtk @@ -162,6 +174,7 @@ buildPythonPackage rec { librealsense zeromq zlib + libpng ] ++ (if isRocm then [] @@ -174,8 +187,6 @@ buildPythonPackage rec { nativeBuildInputs = [ cmake - nasm - gitMinimal # ninja useOldCXXAbi gtest From af9c1671e3f03e3564e6c908f34ae9f62c11718a Mon Sep 17 00:00:00 2001 From: lucasew Date: Tue, 30 May 2023 14:03:39 -0300 Subject: [PATCH 04/10] checkpoint Signed-off-by: lucasew --- flake.nix | 1 + modules/aipython3/overlays.nix | 1 + packages/zipnerf-pytorch/default.nix | 106 +++++++++++++++++++++++++++ projects/pkgs/default.nix | 6 ++ 4 files changed, 114 insertions(+) create mode 100644 packages/zipnerf-pytorch/default.nix create mode 100644 projects/pkgs/default.nix diff --git a/flake.nix b/flake.nix index 1931b770..279d34c6 100644 --- a/flake.nix +++ b/flake.nix @@ -43,6 +43,7 @@ ./projects/invokeai ./projects/koboldai ./projects/nerfstudio + ./projects/pkgs ./website ]; }; diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 8dd287ce..ad47b55e 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -87,6 +87,7 @@ pkgs: { tyro = callPackage ../../packages/tyro { }; nerfacc = callPackage ../../packages/nerfacc { }; open3d = callPackage ../../packages/open3d { }; + zipnerf-pytorch = callPackage ../../packages/zipnerf-pytorch { }; liblzf = callPackage ../../packages/liblzf { }; }; diff --git a/packages/zipnerf-pytorch/default.nix b/packages/zipnerf-pytorch/default.nix new file mode 100644 index 00000000..957c2ef3 --- /dev/null +++ b/packages/zipnerf-pytorch/default.nix @@ -0,0 +1,106 @@ +{ buildPythonPackage +, fetchFromGitHub +, lib +, python +, torch +, cudaPackages +, which +, hip +, hipsparse +, rocblas +, rocthrust +, runCommand +, gcc11Stdenv +, stdenv +, wrapPython +, makeWrapper +, accelerate +# , nvidia-thrust +}: + +let + isRocm = torch.rocmSupport or false; + + version = "2023.05.28-unstable"; + + src = fetchFromGitHub { + owner = "SuLvXiangXin"; + repo = "zipnerf-pytorch"; + rev = "20866db84b31addc9d7fb2aa57e2b50fd33ce15b"; + sha256 = "sha256-IglTczgFqnpWw/Ck36G308yFpgHL3YOSt7o1lwkwJFE="; + }; + + rocblas-header = runCommand "rocblas-header" {} '' + mkdir $out + ln -s ${rocblas}/include/rocblas $out/include + ''; + + gridencoder = (buildPythonPackage { + pname = "gridencoder"; + inherit version; + + src = "${src}/gridencoder"; + + propagatedBuildInputs = [ torch ]; + + TORCH_CUDA_ARCH_LIST = "7.0+PTX"; + + buildInputs = [ ] + ++ (if isRocm + then [ + hipsparse + rocblas + rocblas-header + rocthrust + ] + else [ cudaPackages.cudatoolkit ]); + + nativeBuildInputs = [ which ] + ++ (if isRocm + then [ hip ] + else [ cudaPackages.cuda_nvcc ]); + + }).overrideDerivation (old: { + stdenv = if isRocm then stdenv else gcc11Stdenv; + }); + +in + +buildPythonPackage { + pname = "zipnerf"; + inherit version; + + format = "other"; + + inherit src; + + buildInputs = [ gridencoder ]; + + nativeBuildInputs = [ wrapPython makeWrapper which ]; + + wrappedPython = (python.withPackages (p: with p; [ + numpy + absl-py + accelerate + gin-config + matplotlib + trimesh + opencv3 + scipy + scikitimage + ])).interpreter; + + installPhase = '' + export INSTALL_DIR=$out/${python.sitePackages}/zipnerf-pytorch + mkdir -p $INSTALL_DIR + cp -r * $INSTALL_DIR + + mkdir -p $out/bin + + for bin in eval extract render train; do + makeWrapper $wrappedPython $out/bin/zipnerf-$bin \ + --prefix PYTHONPATH : $INSTALL_DIR \ + --add-flags $INSTALL_DIR/$bin.py + done + ''; +} diff --git a/projects/pkgs/default.nix b/projects/pkgs/default.nix new file mode 100644 index 00000000..77e22c66 --- /dev/null +++ b/projects/pkgs/default.nix @@ -0,0 +1,6 @@ +{ ... }: +{ + perSystem = { config, ...}: { + legacyPackages.pkgs = config.dependencySets; + }; +} From d67a805cf1d8339457ffd655c9129a1b1b4b00d6 Mon Sep 17 00:00:00 2001 From: lucasew Date: Wed, 31 May 2023 00:10:36 -0300 Subject: [PATCH 05/10] checkpoint Signed-off-by: lucasew --- packages/zipnerf-pytorch/default.nix | 56 +++++++++++++++++++++++----- 1 file changed, 46 insertions(+), 10 deletions(-) diff --git a/packages/zipnerf-pytorch/default.nix b/packages/zipnerf-pytorch/default.nix index 957c2ef3..877c7b63 100644 --- a/packages/zipnerf-pytorch/default.nix +++ b/packages/zipnerf-pytorch/default.nix @@ -16,6 +16,15 @@ , makeWrapper , accelerate # , nvidia-thrust +, numpy +, absl-py +, gin-config +, matplotlib +, trimesh +, opencv3 +, scipy +, scikitimage +, ninja }: let @@ -55,30 +64,47 @@ let ] else [ cudaPackages.cudatoolkit ]); - nativeBuildInputs = [ which ] + nativeBuildInputs = [ which ninja ] ++ (if isRocm then [ hip ] else [ cudaPackages.cuda_nvcc ]); + postInstall = '' + ls -R $out + ''; + + pythonImportsCheck = [ "_gridencoder" ]; + }).overrideDerivation (old: { stdenv = if isRocm then stdenv else gcc11Stdenv; }); in -buildPythonPackage { - pname = "zipnerf"; +buildPythonPackage rec { + pname = "zipnerf_pytorch"; inherit version; format = "other"; inherit src; - buildInputs = [ gridencoder ]; + _INSTALL_PATH = "${python.sitePackages}/zipnerf_pytorch"; + + prePatch = '' + substituteInPlace internal/configs.py \ + --replace "gin.add_config_file_search_path('configs/')" "gin.add_config_file_search_path('$out/$_INSTALL_PATH/configs/')" + substituteInPlace $(find -type f | grep .py) \ + --replace 'from internal' 'from zipnerf_pytorch.internal' + ''; + + buildInputs = [ + gridencoder + ]; nativeBuildInputs = [ wrapPython makeWrapper which ]; - wrappedPython = (python.withPackages (p: with p; [ + propagatedBuildInputs = [ numpy absl-py accelerate @@ -88,19 +114,29 @@ buildPythonPackage { opencv3 scipy scikitimage - ])).interpreter; + ]; + + pythonImportsCheck = [ pname ]; installPhase = '' - export INSTALL_DIR=$out/${python.sitePackages}/zipnerf-pytorch + export INSTALL_DIR=$out/$_INSTALL_PATH mkdir -p $INSTALL_DIR cp -r * $INSTALL_DIR + buildPythonPath + echo pythonpath + echo $program_PYTHONPATH + + touch $INSTALL_DIR/__init__.py + mkdir -p $out/bin for bin in eval extract render train; do - makeWrapper $wrappedPython $out/bin/zipnerf-$bin \ - --prefix PYTHONPATH : $INSTALL_DIR \ - --add-flags $INSTALL_DIR/$bin.py + makeWrapper ${(python.withPackages (p: propagatedBuildInputs)).interpreter} $out/bin/zipnerf_$bin \ + --prefix PYTHONPATH : $out/${python.sitePackages} \ + --add-flags $INSTALL_DIR/$bin.py \ + --prefix PYTHONPATH : "$program_PYTHONPATH" \ + --prefix PATH : "$program_PATH" done ''; } From ce55f3dbfe57f602c5da7518f68323efa30b9a15 Mon Sep 17 00:00:00 2001 From: lucasew Date: Wed, 31 May 2023 01:08:40 -0300 Subject: [PATCH 06/10] checkpoint Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 1 + packages/rawpy/default.nix | 38 +++++++++++++ packages/zipnerf-pytorch/default.nix | 79 +++++++++++++++++----------- 3 files changed, 86 insertions(+), 32 deletions(-) create mode 100644 packages/rawpy/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index ad47b55e..464750d4 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -89,6 +89,7 @@ pkgs: { open3d = callPackage ../../packages/open3d { }; zipnerf-pytorch = callPackage ../../packages/zipnerf-pytorch { }; liblzf = callPackage ../../packages/liblzf { }; + rawpy = callPackage ../../packages/rawpy { }; }; torchRocm = final: prev: rec { diff --git a/packages/rawpy/default.nix b/packages/rawpy/default.nix new file mode 100644 index 00000000..bf5edd4e --- /dev/null +++ b/packages/rawpy/default.nix @@ -0,0 +1,38 @@ +{ buildPythonPackage +, fetchFromGitHub +, lib +, numpy +, cython +, libraw +, runCommand +, pkg-config +, scikitimage +}: + +buildPythonPackage rec { + pname = "rawpy"; + version = "0.18.1"; + + src = fetchFromGitHub { + owner = "letmaik"; + repo = "rawpy"; + rev = "refs/tags/v${version}"; + sha256 = "sha256-ErQSVtv+pxKIgqCPrh74PbuWv4BKwqhLlBxtljmTCFM="; + }; + + propagatedBuildInputs = [ numpy ]; + + checkInputs = [ scikitimage ]; + + nativeBuildInputs = [ cython pkg-config ]; + + buildInputs = [ + libraw + # (runCommand "libraw-headers" {} '' + # mkdir $out + # ln -s ${libraw.dev}/include/libraw $out/include + # '') + ]; + + pythonImportsCheck = [ "rawpy" ]; +} diff --git a/packages/zipnerf-pytorch/default.nix b/packages/zipnerf-pytorch/default.nix index 877c7b63..1c9534f6 100644 --- a/packages/zipnerf-pytorch/default.nix +++ b/packages/zipnerf-pytorch/default.nix @@ -12,19 +12,22 @@ , runCommand , gcc11Stdenv , stdenv -, wrapPython , makeWrapper , accelerate # , nvidia-thrust , numpy , absl-py , gin-config +, wheel , matplotlib , trimesh , opencv3 , scipy , scikitimage , ninja +, rawpy +, addOpenGLRunpath +, pillow }: let @@ -54,7 +57,7 @@ let TORCH_CUDA_ARCH_LIST = "7.0+PTX"; - buildInputs = [ ] + buildInputs = [] ++ (if isRocm then [ hipsparse @@ -62,15 +65,17 @@ let rocblas-header rocthrust ] - else [ cudaPackages.cudatoolkit ]); + else [ + cudaPackages.cudatoolkit + ]); - nativeBuildInputs = [ which ninja ] + nativeBuildInputs = [ which ninja addOpenGLRunpath ] ++ (if isRocm then [ hip ] - else [ cudaPackages.cuda_nvcc ]); + else [ cudaPackages.cudatoolkit ]); - postInstall = '' - ls -R $out + postFixup= '' + # addOpenGLRunpath $out/${python.sitePackages}/*.so ''; pythonImportsCheck = [ "_gridencoder" ]; @@ -79,6 +84,14 @@ let stdenv = if isRocm then stdenv else gcc11Stdenv; }); + ourScikitImage = scikitimage.overrideAttrs (old: { + prePatch = (old.prePatch or "") + '' + echo resolvendo buxa + substituteInPlace skimage/color/colorconv.py \ + --replace 'from scipy import linalg' 'from numpy import linalg' + ''; + }); + in buildPythonPackage rec { @@ -89,7 +102,9 @@ buildPythonPackage rec { inherit src; - _INSTALL_PATH = "${python.sitePackages}/zipnerf_pytorch"; + PYTHONFAULTHANDLER=1; + DEBUG=1; + OPENBLAS_CORETYPE="haswell"; prePatch = '' substituteInPlace internal/configs.py \ @@ -98,45 +113,45 @@ buildPythonPackage rec { --replace 'from internal' 'from zipnerf_pytorch.internal' ''; - buildInputs = [ - gridencoder - ]; - - nativeBuildInputs = [ wrapPython makeWrapper which ]; + nativeBuildInputs = [ python.pkgs.wrapPython makeWrapper which ]; propagatedBuildInputs = [ - numpy - absl-py - accelerate + gridencoder + # accelerate gin-config - matplotlib - trimesh opencv3 - scipy - scikitimage + pillow + matplotlib + # scipy + ourScikitImage + absl-py + torch + rawpy ]; - pythonImportsCheck = [ pname ]; + pythonPath = propagatedBuildInputs; + + # the second import is to check if something is conflicting with scikit + pythonImportsCheck = [ pname "${pname}.internal.datasets" ]; installPhase = '' export INSTALL_DIR=$out/$_INSTALL_PATH - mkdir -p $INSTALL_DIR - cp -r * $INSTALL_DIR - - buildPythonPath - echo pythonpath - echo $program_PYTHONPATH + mkdir -p $out/${python.sitePackages}/$pname + cp -r * $out/${python.sitePackages}/$pname - touch $INSTALL_DIR/__init__.py + touch $out/${python.sitePackages}/$pname mkdir -p $out/bin + ''; + preFixup = '' + buildPythonPath "$out $pythonPath" for bin in eval extract render train; do - makeWrapper ${(python.withPackages (p: propagatedBuildInputs)).interpreter} $out/bin/zipnerf_$bin \ - --prefix PYTHONPATH : $out/${python.sitePackages} \ - --add-flags $INSTALL_DIR/$bin.py \ + makeWrapper ${python.interpreter} $out/bin/zipnerf_$bin \ --prefix PYTHONPATH : "$program_PYTHONPATH" \ - --prefix PATH : "$program_PATH" + --prefix PATH : "$program_PATH" \ + --add-flags $out/${python.sitePackages}/$pname/$bin.py done ''; + } From ef31297c47110c9bf2414cdb1da6025fe78344da Mon Sep 17 00:00:00 2001 From: lucasew Date: Thu, 1 Jun 2023 18:20:25 -0300 Subject: [PATCH 07/10] checkpoint Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 9 +++ packages/pycolmap/default.nix | 82 ++++++++++++++++++++++++++++ packages/pymeshlab/default.nix | 31 ++++++++++- packages/torch_scatter/default.nix | 28 ++++++++++ packages/zipnerf-pytorch/default.nix | 34 ++++++++++-- 5 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 packages/pycolmap/default.nix create mode 100644 packages/torch_scatter/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 464750d4..f5f2049b 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -29,6 +29,13 @@ pkgs: { broken = false; }; }); + scikitimage = prev.scikitimage.overrideAttrs (old: { + # fix weird crash + prePatch = (old.prePatch or "") + '' + substituteInPlace skimage/color/colorconv.py \ + --replace 'from scipy import linalg' 'from numpy import linalg' + ''; + }); streamlit = let streamlit = final.callPackage (pkgs.path + "/pkgs/applications/science/machine-learning/streamlit") { protobuf3 = final.protobuf; @@ -90,6 +97,8 @@ pkgs: { zipnerf-pytorch = callPackage ../../packages/zipnerf-pytorch { }; liblzf = callPackage ../../packages/liblzf { }; rawpy = callPackage ../../packages/rawpy { }; + pycolmap = callPackage ../../packages/pycolmap { }; + torch_scatter = callPackage ../../packages/torch_scatter { }; }; torchRocm = final: prev: rec { diff --git a/packages/pycolmap/default.nix b/packages/pycolmap/default.nix new file mode 100644 index 00000000..b5b2b086 --- /dev/null +++ b/packages/pycolmap/default.nix @@ -0,0 +1,82 @@ +{ buildPythonPackage +, fetchFromGitHub +, lib +, python +, colmap +, cmake +, ceres-solver +, boost +, freeimage +, libGL +, glew +, qt5 +, torch +, hip +, cudatoolkit +, cgal +, pybind11 +, tbb +, flann +, numpy +}: + +let + isRocm = torch.rocmSupport or false; + + customColmap = colmap.overrideDerivation (old: rec { + pname = "colmap"; + version = "3.8"; + src = fetchFromGitHub { + owner = "colmap"; + repo = "colmap"; + rev = "refs/tags/${version}"; + sha256 = "sha256-1uUbUZdz49TloEaPJijNwa51DxIPjgz/fthnbWLfgS8="; + }; + + buildInputs = old.buildInputs ++ [ flann ]; + }); +in + +buildPythonPackage rec { + pname = "pycolmap"; + version = "0.4.0"; + format = "other"; + + src = fetchFromGitHub { + owner = "colmap"; + repo = pname; + rev = "refs/tags/v${version}"; + sha256 = "sha256-W3d+uHZXkH1/QlER1HV8t1MOBOrHIXYsVeYv1zbsbW4="; + fetchSubmodules = true; + }; + + cmakeFlags = [ "-DCOLMAP_CGAL_ENABLED=ON" ] + ++ (if isRocm then [ ] else [ "-DCOLMAP_CUDA_ENABLED=ON"]) + ; + + buildInputs = [ + customColmap + ceres-solver + boost + freeimage + libGL + glew + qt5.qtbase + cgal + tbb + flann + ]; + + propagatedBuildInputs = [ numpy ]; + + nativeBuildInputs = [ cmake qt5.wrapQtAppsHook pybind11 ] + ++ (if isRocm then [ hip ] else [ cudatoolkit ]) + ; + + installPhase = '' + mkdir -p $out/${python.sitePackages} + install *.so $out/${python.sitePackages} + ''; + + pythonImportsCheck = [ pname ]; +} diff --git a/packages/pymeshlab/default.nix b/packages/pymeshlab/default.nix index 6ad29b55..4e7a3ce2 100644 --- a/packages/pymeshlab/default.nix +++ b/packages/pymeshlab/default.nix @@ -11,6 +11,17 @@ , plotly , jupyterlab , viser +, cmake +, meshlab +, libGL +, qt5 +, eigen +, boost +, cgal +, levmar +, lib3ds +, muparser +, nexus }: buildPythonPackage rec { @@ -21,11 +32,29 @@ buildPythonPackage rec { owner = "cnr-isti-vclab"; repo = pname; rev = "v${version}"; - sha256 = "sha256-NmnEp0ThF/YLhuVmjWsscP5l6V8TIrn2I47qC/owQCo="; + fetchSubmodules = true; + sha256 = "sha256-CtzyKymM/SMoiw413Y+r89R6FEHvEaBS36iDcuRkDCo="; }; propagatedBuildInputs = [ numpy pyngrok plotly jupyterlab viser ]; + nativeBuildInputs = [ cmake qt5.wrapQtAppsHook ]; + + buildInputs = [ + meshlab + libGL + qt5.qtbase + eigen + boost + cgal + levmar + lib3ds + muparser + nexus + ]; + + pythonImportsCheck = [ "pymeshlab" "pymeshlab.pmeshlab" ]; + # TODO FIXME doCheck = false; diff --git a/packages/torch_scatter/default.nix b/packages/torch_scatter/default.nix new file mode 100644 index 00000000..faaec9af --- /dev/null +++ b/packages/torch_scatter/default.nix @@ -0,0 +1,28 @@ +{ buildPythonPackage +, fetchFromGitHub +, lib +, torch +, which +, pytest +, pytest-cov +}: + +buildPythonPackage rec { + pname = "torch_scatter"; + version = "2.1.1"; + + src = fetchFromGitHub { + owner = "rusty1s"; + repo = "pytorch_scatter"; + rev = version; + sha256 = "sha256-oAxWTX412dWFb2DYo9UbN+N1BUvg4nB/JL1cMoJIkjw="; + }; + + nativeBuildInputs = [ which pytest pytest-cov ]; + + propagatedBuildInputs = [ torch ]; + + setuptoolsCheckPhase = "pytest"; + + pythonImportsCheck = [ pname ]; +} diff --git a/packages/zipnerf-pytorch/default.nix b/packages/zipnerf-pytorch/default.nix index 1c9534f6..c58a9579 100644 --- a/packages/zipnerf-pytorch/default.nix +++ b/packages/zipnerf-pytorch/default.nix @@ -20,7 +20,6 @@ , gin-config , wheel , matplotlib -, trimesh , opencv3 , scipy , scikitimage @@ -28,6 +27,12 @@ , rawpy , addOpenGLRunpath , pillow +, torch_scatter +, tensorboardx +, tensorboard +, trimesh +, mediapy +, pymeshlab }: let @@ -86,7 +91,6 @@ let ourScikitImage = scikitimage.overrideAttrs (old: { prePatch = (old.prePatch or "") + '' - echo resolvendo buxa substituteInPlace skimage/color/colorconv.py \ --replace 'from scipy import linalg' 'from numpy import linalg' ''; @@ -110,23 +114,40 @@ buildPythonPackage rec { substituteInPlace internal/configs.py \ --replace "gin.add_config_file_search_path('configs/')" "gin.add_config_file_search_path('$out/$_INSTALL_PATH/configs/')" substituteInPlace $(find -type f | grep .py) \ - --replace 'from internal' 'from zipnerf_pytorch.internal' + --replace 'from internal' 'from ${pname}.internal' + + substituteInPlace internal/pycolmap/pycolmap/__init__.py \ + --replace 'from ' 'from ${pname}.internal.pycolmap.' + + substituteInPlace internal/pycolmap/pycolmap/scene_manager.py \ + --replace 'from camera' 'from ${pname}.internal.pycolmap.camera' \ + --replace 'from image' 'from ${pname}.internal.pycolmap.image' \ + --replace 'from rotation' 'from ${pname}.internal.pycolmap.rotation' + + substituteInPlace internal/datasets.py \ + --replace 'import pycolmap' 'from ${pname}.internal import pycolmap' ''; nativeBuildInputs = [ python.pkgs.wrapPython makeWrapper which ]; propagatedBuildInputs = [ gridencoder - # accelerate + accelerate gin-config opencv3 pillow matplotlib - # scipy - ourScikitImage + scikitimage + # ourScikitImage absl-py torch rawpy + torch_scatter + tensorboardx + tensorboard + trimesh + mediapy + pymeshlab ]; pythonPath = propagatedBuildInputs; @@ -138,6 +159,7 @@ buildPythonPackage rec { export INSTALL_DIR=$out/$_INSTALL_PATH mkdir -p $out/${python.sitePackages}/$pname cp -r * $out/${python.sitePackages}/$pname + mv $out/${python.sitePackages}/$pname/internal/pycolmap/pycolmap/* $out/${python.sitePackages}/$pname/internal/pycolmap touch $out/${python.sitePackages}/$pname From 3438b46085fec03dd21ace090a8c68a309bd3d6a Mon Sep 17 00:00:00 2001 From: lucasew Date: Fri, 2 Jun 2023 08:11:10 -0300 Subject: [PATCH 08/10] checkpoint Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 1 + packages/libigl/default.nix | 35 ++++++++++++++++++++++++++++++++++ packages/pymeshlab/default.nix | 34 +++++++++++++++++++++------------ 3 files changed, 58 insertions(+), 12 deletions(-) create mode 100644 packages/libigl/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index f5f2049b..62375f64 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -99,6 +99,7 @@ pkgs: { rawpy = callPackage ../../packages/rawpy { }; pycolmap = callPackage ../../packages/pycolmap { }; torch_scatter = callPackage ../../packages/torch_scatter { }; + libigl = callPackage ../../packages/libigl { }; }; torchRocm = final: prev: rec { diff --git a/packages/libigl/default.nix b/packages/libigl/default.nix new file mode 100644 index 00000000..ba09b870 --- /dev/null +++ b/packages/libigl/default.nix @@ -0,0 +1,35 @@ +{ stdenv +, fetchurl +, lib +, unzip +, cmake +, gmp +, mpfr +, eigen2 +, pkg-config +, embree +, libGL +, glfw +, boost +, tetgen +, cgal +, catch2 +}: + +stdenv.mkDerivation rec { + pname = "libigl"; + version = "2.4.0"; + + src = fetchurl { + url = "https://www.meshlab.net/data/libs/libigl-${version}.zip"; + sha256 = "sha256-sg2AyaP6fQxrw63uWORPDNscu0kijfbWf53b/AjuFAM="; + }; + + cmakeFlags = [ + "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" + ]; + + nativeBuildInputs = [ unzip cmake gmp pkg-config catch2 ]; + + buildInputs = [ mpfr eigen2 embree libGL glfw boost tetgen cgal catch2 ]; +} diff --git a/packages/pymeshlab/default.nix b/packages/pymeshlab/default.nix index 4e7a3ce2..da5ed7a0 100644 --- a/packages/pymeshlab/default.nix +++ b/packages/pymeshlab/default.nix @@ -9,6 +9,7 @@ , numpy , pyngrok , plotly +, pybind11 , jupyterlab , viser , cmake @@ -18,10 +19,16 @@ , eigen , boost , cgal +, glew , levmar , lib3ds , muparser , nexus +, python +, pkg-config +, structure-synth +, libigl +, tinygltf }: buildPythonPackage rec { @@ -32,26 +39,29 @@ buildPythonPackage rec { owner = "cnr-isti-vclab"; repo = pname; rev = "v${version}"; + # sha256 = "sha256-CtzyKymM/SMoiw413Y+r89R6FEHvEaBS36iDcuRkDCo="; + sha256 = "sha256-nyXnqZkeHNEwoGAhJSFhgUUCvRKVRtEEKRQtzSZjXz0="; fetchSubmodules = true; - sha256 = "sha256-CtzyKymM/SMoiw413Y+r89R6FEHvEaBS36iDcuRkDCo="; + deepClone = true; }; + cmakeFlags = [ + ] + ++ meshlab.cmakeFlags + ; + propagatedBuildInputs = [ numpy pyngrok plotly jupyterlab viser ]; - nativeBuildInputs = [ cmake qt5.wrapQtAppsHook ]; + nativeBuildInputs = [ cmake qt5.wrapQtAppsHook pkg-config ]; buildInputs = [ + pybind11 meshlab - libGL - qt5.qtbase - eigen - boost - cgal - levmar - lib3ds - muparser - nexus - ]; + structure-synth + tinygltf + libigl + ] + ++ meshlab.buildInputs; pythonImportsCheck = [ "pymeshlab" "pymeshlab.pmeshlab" ]; From 86bc6db3937760ea77c93862c38d27243c247040 Mon Sep 17 00:00:00 2001 From: lucasew Date: Fri, 2 Jun 2023 23:54:43 -0300 Subject: [PATCH 09/10] libigl builds yay Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 1 + packages/glad/default.nix | 20 +++ packages/libigl/default.nix | 216 +++++++++++++++++++++++++++++++-- packages/pymeshlab/default.nix | 1 - 4 files changed, 224 insertions(+), 14 deletions(-) create mode 100644 packages/glad/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index 62375f64..bfed4024 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -100,6 +100,7 @@ pkgs: { pycolmap = callPackage ../../packages/pycolmap { }; torch_scatter = callPackage ../../packages/torch_scatter { }; libigl = callPackage ../../packages/libigl { }; + glad = callPackage ../../packages/glad { }; }; torchRocm = final: prev: rec { diff --git a/packages/glad/default.nix b/packages/glad/default.nix new file mode 100644 index 00000000..281eb449 --- /dev/null +++ b/packages/glad/default.nix @@ -0,0 +1,20 @@ +{ buildPythonPackage +, fetchFromGitHub +, lib +, jinja2 +, cmake +}: + +buildPythonPackage rec { + pname = "glad"; + version = "2.0.4"; + + src = fetchFromGitHub { + owner = "Dav1dde"; + repo = "glad"; + rev = "v${version}"; + sha256 = "sha256-Sq7VUhRnGvHtVlLv1b7pyA+x8u3/cxNxxp5VBms0Vd0="; + }; + + propagatedBuildInputs = [ jinja2 ]; +} diff --git a/packages/libigl/default.nix b/packages/libigl/default.nix index ba09b870..a18f731a 100644 --- a/packages/libigl/default.nix +++ b/packages/libigl/default.nix @@ -1,35 +1,225 @@ -{ stdenv -, fetchurl +{ buildPythonPackage +, fetchzip +, fetchgit , lib , unzip , cmake , gmp , mpfr -, eigen2 +, eigen , pkg-config , embree , libGL , glfw , boost -, tetgen , cgal , catch2 +, glad +, ispc +, tbb +, imgui +, tinyxml2 +, libpng +, xorg +, stb }: -stdenv.mkDerivation rec { - pname = "libigl"; - version = "2.4.0"; +let + getFetchContent = { cmakeFile, hashes ? {} }: let + inherit (builtins) readFile split filter match typeOf head tail foldl'; + inherit (lib) pipe flatten; + + data = pipe (readFile cmakeFile) [ + (split "\n") + (filter (x: typeOf x == "string")) + (flatten) + (map (match " *(.*GIT.*) *")) + (flatten) + (filter (x: x != null)) + (map (x: let line = lib.pipe x [ + (split " ") + (filter (v: typeOf v == "string" && v != "")) + ]; in { "${head line}" = "${head (tail line)}";})) + (foldl' (x: y: x // y) {}) + ]; - src = fetchurl { + rev = builtins.replaceStrings [ "tags/" ] [ "" ] data.GIT_TAG; + in fetchgit { + url = "${data.GIT_REPOSITORY}"; + inherit rev; + sha256 = hashes.${rev} or (builtins.trace "getFetchContent: fallback hash for '${data.GIT_REPOSITORY}' '${data.GIT_TAG}'" lib.fakeSha256); + }; + + version = "2.4.0"; + src = fetchzip { url = "https://www.meshlab.net/data/libs/libigl-${version}.zip"; - sha256 = "sha256-sg2AyaP6fQxrw63uWORPDNscu0kijfbWf53b/AjuFAM="; + sha256 = "sha256-HRD854Y/F2jwVKEvewbYYy/rgf3tmEb8ZFCtbtG9lmI="; }; - cmakeFlags = [ - "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" + inputs = { + embree = { + "v3.13.3" = "sha256-g6BsXMNUvx17hgAq0PewtBLgtWqpp03M0k6vWNapDKs="; + }; + tinyxml2 = { + "d175e9de0be0d4db75d0a8cf065599a435a87eb6" = "sha256-g5D88Vl2EhtEp+jh5J0+T+P5hM7raM/41qh3Ud9Sg6o="; + }; + stb = { + "f67165c2bb2af3060ecae7d20d6f731173485ad0" = "sha256-b/T+8omHlUVmA6a+66FbgH++iXy4cZJD3Q5HTXApGa0="; + }; + libigl_imgui_fonts = { + "7e1053e750b0f4c129b046f4e455243cb7f804f3" = "sha256-3+rscFLD0OvsjRJIDRvENSf4Vorons6QDV6pXBVCYRM="; + }; + catch2 = { + "v2.13.8" = "sha256-jOA2TxDgaJUJ2Jn7dVGZUbjmphTDuVZahzSaxfJpRqE="; + }; + eigen = { + "3.3.7" = "sha256-oXJ4V5rakL9EPtQF0Geptl0HMR8700FdSrOB09DbbMQ="; + }; + glad = { + "09b4969c56779f7ddf8e6176ec1873184aec890f" = "sha256-k6s8Ct3FGPq43+5bAvq56bGrqPsys7P7M5f6EcrRmWc="; + }; + glfw = { + "3327050ca66ad34426a82c217c2d60ced61526b7" = "sha256-XBVxtkL29WqQooM2+wkorjjWIpXj0g3X8rkGhSiNoxI="; + }; + imguizmo = { + "a23567269f6617342bcc112394bdad937b54b2d7" = "sha256-l5BOeBkJlX6VAGNDM+Ouc8YnwooRqzZbcmRFyeO/ZCU="; + }; + imgui = { + "v1.85" = "sha256-HQsGlsvmf3ikqhGnJHf/d6SRCY/QDeW7XUTwXQH/JYE="; + }; + libigl_tests_data = { + "name" = "libigl_tests_tata"; + "19cedf96d70702d8b3a83eb27934780c542356fe" = "sha256-IhLlfe89utbIks/z1Re1dbgUDqpX1iqJIryTw3umr/M="; + }; + }; + + inputsCMakeVars = lib.pipe inputs [ + (builtins.attrNames) + (map (dir: "-DFETCHCONTENT_SOURCE_DIR_${lib.toUpper (inputs.${dir}.name or dir)}=../.${dir}")) + ]; + + inputsDerivations = builtins.mapAttrs (k: v: getFetchContent { + cmakeFile = "${src}/cmake/recipes/external/${k}.cmake"; + hashes = inputs.${k}; + }) inputs; + + inputsPatchPhase = lib.pipe inputsDerivations [ + (builtins.attrNames) + (map (k: "cp -r ${inputsDerivations.${k}} -r .${k}; chmod +w .${k} -R")) + (builtins.concatStringsSep "\n") ]; - nativeBuildInputs = [ unzip cmake gmp pkg-config catch2 ]; +in - buildInputs = [ mpfr eigen2 embree libGL glfw boost tetgen cgal catch2 ]; +buildPythonPackage rec { + pname = "libigl"; + + inherit src version; + + format = "other"; + + cmakeSuffix = '' + # include(FindPkgConfig) + # find_package(eigen REQUIRED eigen-populate) + # find_package(glfw3 REQUIRED glfw) + # find_package(glad REQUIRED) + ''; + + prePatch = '' + ${inputsPatchPhase} + + echo "$cmakeSuffix" >> CMakeLists.txt + ''; + + cmakeFlags = inputsCMakeVars ++ [ + # "-DFETCHCONTENT_FULLY_DISCONNECTED=ON" + "-DLIBGL_EMBREE=OFF" + "-DFETCHCONTENT_TRY_FIND_PACKAGE_MODE=ALWAYS" + "-DLIBIGL_BUILD_TUTORIALS=OFF" + "-DLIBIGL_COPYLEFT_CGAL=OFF" + "-DLIBIGL_COPYLEFT_COMISO=OFF" + "-DLIBIGL_COPYLEFT_CORE=OFF" + "-DLIBIGL_COPYLEFT_CORK=OFF" + "-DLIBIGL_COPYLEFT_TETGEN=OFF" + "-DLIBIGL_PREDICATES=OFF" + "-DLIBIGL_RESTRICTED_MATLAB=OFF" + "-DLIBIGL_RESTRICTED_MOSEK=OFF" + "-DLIBIGL_RESTRICTED_TRIANGLE=OFF" + "-DTBB_DIR=${tbb.src}" + # "-DFETCHCONTENT_SOURCE_DIR_EMBREE=../.embree" # embree needs to be in a mutable location + # "-DFETCHCONTENT_SOURCE_DIR_TINYXML2=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/tinyxml2.cmake"; + # hashes = { + # "d175e9de0be0d4db75d0a8cf065599a435a87eb6" = "sha256-g5D88Vl2EhtEp+jh5J0+T+P5hM7raM/41qh3Ud9Sg6o="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_STB=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/stb.cmake"; + # hashes = { + # "f67165c2bb2af3060ecae7d20d6f731173485ad0" = "sha256-b/T+8omHlUVmA6a+66FbgH++iXy4cZJD3Q5HTXApGa0="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_LIBIGL_IMGUI_FONTS=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/libigl_imgui_fonts.cmake"; + # hashes = { + # "7e1053e750b0f4c129b046f4e455243cb7f804f3" = "sha256-3+rscFLD0OvsjRJIDRvENSf4Vorons6QDV6pXBVCYRM="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_CATCH2=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/catch2.cmake"; + # hashes = { + # "v2.13.8" = "sha256-jOA2TxDgaJUJ2Jn7dVGZUbjmphTDuVZahzSaxfJpRqE="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_EIGEN=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/eigen.cmake"; + # hashes = { + # "3.3.7" = "sha256-oXJ4V5rakL9EPtQF0Geptl0HMR8700FdSrOB09DbbMQ="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_GLAD=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/glad.cmake"; + # hashes = { + # "09b4969c56779f7ddf8e6176ec1873184aec890f" = "sha256-k6s8Ct3FGPq43+5bAvq56bGrqPsys7P7M5f6EcrRmWc="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_GLFW=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/glfw.cmake"; + # hashes = { + # "3327050ca66ad34426a82c217c2d60ced61526b7" = "sha256-XBVxtkL29WqQooM2+wkorjjWIpXj0g3X8rkGhSiNoxI="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_IMGUIZMO=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/imguizmo.cmake"; + # hashes = { + # "a23567269f6617342bcc112394bdad937b54b2d7" = "sha256-l5BOeBkJlX6VAGNDM+Ouc8YnwooRqzZbcmRFyeO/ZCU="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_IMGUI=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/imgui.cmake"; + # hashes = { + # "v1.85" = "sha256-HQsGlsvmf3ikqhGnJHf/d6SRCY/QDeW7XUTwXQH/JYE="; + # }; + # }}" + # "-DFETCHCONTENT_SOURCE_DIR_LIBIGL_TESTS_TATA=${getFetchContent { + # cmakeFile = "${src}/cmake/recipes/external/libigl_tests_data.cmake"; + # hashes = { + # "19cedf96d70702d8b3a83eb27934780c542356fe" = "sha256-IhLlfe89utbIks/z1Re1dbgUDqpX1iqJIryTw3umr/M="; + # }; + # }}" + ]; + + nativeBuildInputs = [ unzip cmake gmp pkg-config ]; + + buildInputs = [ + mpfr + libGL + libpng + xorg.libX11 + xorg.libXrandr + xorg.libXinerama + xorg.libXcursor + xorg.libXi + stb + ]; } diff --git a/packages/pymeshlab/default.nix b/packages/pymeshlab/default.nix index da5ed7a0..7410a2f8 100644 --- a/packages/pymeshlab/default.nix +++ b/packages/pymeshlab/default.nix @@ -39,7 +39,6 @@ buildPythonPackage rec { owner = "cnr-isti-vclab"; repo = pname; rev = "v${version}"; - # sha256 = "sha256-CtzyKymM/SMoiw413Y+r89R6FEHvEaBS36iDcuRkDCo="; sha256 = "sha256-nyXnqZkeHNEwoGAhJSFhgUUCvRKVRtEEKRQtzSZjXz0="; fetchSubmodules = true; deepClone = true; From 07546036274b385409a8cdb3f15057d483a74c76 Mon Sep 17 00:00:00 2001 From: lucasew Date: Sat, 3 Jun 2023 14:40:03 -0300 Subject: [PATCH 10/10] checkpoint Signed-off-by: lucasew --- modules/aipython3/overlays.nix | 2 ++ packages/openctm/default.nix | 38 +++++++++++++++++++++++++++++ packages/pymeshlab/default.nix | 44 +++++++++++++++++++++++++++------- packages/u3d/default.nix | 19 +++++++++++++++ 4 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 packages/openctm/default.nix create mode 100644 packages/u3d/default.nix diff --git a/modules/aipython3/overlays.nix b/modules/aipython3/overlays.nix index bfed4024..134d87a4 100644 --- a/modules/aipython3/overlays.nix +++ b/modules/aipython3/overlays.nix @@ -101,6 +101,8 @@ pkgs: { torch_scatter = callPackage ../../packages/torch_scatter { }; libigl = callPackage ../../packages/libigl { }; glad = callPackage ../../packages/glad { }; + openctm = callPackage ../../packages/openctm { }; + u3d = callPackage ../../packages/u3d { }; }; torchRocm = final: prev: rec { diff --git a/packages/openctm/default.nix b/packages/openctm/default.nix new file mode 100644 index 00000000..e482c478 --- /dev/null +++ b/packages/openctm/default.nix @@ -0,0 +1,38 @@ +{ stdenv +, lib +, fetchzip +, tinyxml +, pkg-config +, gtk2 +, glew +, freeglut +, libGLU +}: + +stdenv.mkDerivation rec { + pname = "OpenCTM"; + version = "1.0.3"; + + prePatch = '' + echo 'include Makefile.linux' > Makefile + substituteInPlace Makefile.linux \ + --replace /usr/local $out \ + --replace /usr $out + ''; + + preInstall = '' + mkdir -p $out/{bin,lib,include,share/man/man1} + ''; + + NIX_LDFLAGS = [ "-lGL" "-lGLU"]; + NIX_CFLAGS_COMPILE = [ "-Wformat" "-Wl,--copy-dt-needed-entries" ]; + + buildInputs = [ gtk2 tinyxml freeglut libGLU ]; + + nativeBuildInputs = [ pkg-config ]; + + src = fetchzip { + url = "https://www.meshlab.net/data/libs/OpenCTM-${version}-src.zip"; + sha256 = "sha256-zG3GxSurEuoSU8AoFFfguCgXyZJ2CCdRlsFeZH7/YW8="; + }; +} diff --git a/packages/pymeshlab/default.nix b/packages/pymeshlab/default.nix index 7410a2f8..86dd1a73 100644 --- a/packages/pymeshlab/default.nix +++ b/packages/pymeshlab/default.nix @@ -17,8 +17,8 @@ , libGL , qt5 , eigen -, boost -, cgal +, boost17x +, cgal_5 , glew , levmar , lib3ds @@ -26,9 +26,13 @@ , nexus , python , pkg-config -, structure-synth , libigl , tinygltf +, openctm +, xercesc +, gmp +, mpfr +, u3d }: buildPythonPackage rec { @@ -45,22 +49,44 @@ buildPythonPackage rec { }; cmakeFlags = [ - ] - ++ meshlab.cmakeFlags - ; + "-DALLOW_BUNDLED_EIGEN=OFF" + "-DALLOW_BUNDLED_GLEW=OFF" + "-DALLOW_BUNDLED_LIB3DS=OFF" + "-DALLOW_BUNDLED_MUPARSER=OFF" + "-DALLOW_BUNDLED_QHULL=OFF" + "-DALLOW_BUNDLED_OPENCTM=OFF" + "-DALLOW_BUNDLED_BOOST=OFF" + "-DALLOW_BUNDLED_NEWUOA=OFF" + "-DALLOW_BUNDLED_LEVMAR=OFF" + "-DALLOW_BUNDLED_SSYNTH=OFF" + "-DALLOW_BUNDLED_U3D=OFF" + # "-DSSYNTH_DIR=${structure-synth.src}" + "-DLEVMAR_DIR=${levmar.src}" + + # "-DFETCHCONTENT_SOURCE_DIR_Teste=" + ]; propagatedBuildInputs = [ numpy pyngrok plotly jupyterlab viser ]; nativeBuildInputs = [ cmake qt5.wrapQtAppsHook pkg-config ]; buildInputs = [ + gmp + xercesc + openctm pybind11 meshlab - structure-synth tinygltf libigl - ] - ++ meshlab.buildInputs; + lib3ds + boost17x + cgal_5 + libGL + eigen + glew + mpfr + muparser + ]; pythonImportsCheck = [ "pymeshlab" "pymeshlab.pmeshlab" ]; diff --git a/packages/u3d/default.nix b/packages/u3d/default.nix new file mode 100644 index 00000000..25feba82 --- /dev/null +++ b/packages/u3d/default.nix @@ -0,0 +1,19 @@ +{ buildPythonPackage +, fetchzip +, lib +, cmake +}: + +buildPythonPackage rec { + pname = "u3d"; + version = "1.5.1"; + + format = "other"; + + nativeBuildInputs = [ cmake ]; + + src = fetchzip { + url = "https://www.meshlab.net/data/libs/u3d-1.5.1.zip"; + sha256 = "sha256-2MlhGmMsECjJIamXiarHb8MV8DxSxXo/Z+aB4ZBBbJA="; + }; +}