From cb45b55489221df3605d2d0afc62d5cbd3c8822d Mon Sep 17 00:00:00 2001 From: Alexander Raistrick Date: Wed, 20 Nov 2024 16:28:16 -0500 Subject: [PATCH 01/11] Fix scenetype.gin crashes for underwater/kelpforest, lift 1hr timelimit, make integration reuslts group-writable for easy admin --- infinigen/datagen/configs/base.gin | 4 ++++ tests/integration/launch.sh | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/infinigen/datagen/configs/base.gin b/infinigen/datagen/configs/base.gin index 08eaeb909..90655f845 100644 --- a/infinigen/datagen/configs/base.gin +++ b/infinigen/datagen/configs/base.gin @@ -11,4 +11,8 @@ sample_scene_spec.config_distribution = [ ("coast", 4), ("arctic", 1), ("snowy_mountain", 1), + + # keep these here even if 0, otherwise sample_config_distribution will clash with them if they are provided manually by the user + ("under_water", 0), + ("kelp_forest", 0), ] diff --git a/tests/integration/launch.sh b/tests/integration/launch.sh index 9f095b473..fbb1ba22a 100644 --- a/tests/integration/launch.sh +++ b/tests/integration/launch.sh @@ -5,6 +5,9 @@ if [ -z "$OUTPUT_PATH" ]; then exit 1 fi +# make outputs group-writable +umask 0002 + # Environment Variables for Opting In/Out RUN_INDOOR=${RUN_INDOOR:-1} RUN_NATURE=${RUN_NATURE:-1} @@ -27,7 +30,7 @@ if [ "$RUN_INDOOR" -eq 1 ]; then for indoor_type in DiningRoom Bathroom Bedroom Kitchen LivingRoom; do python -m infinigen.datagen.manage_jobs --output_folder $OUTPUT_PATH/${JOBTAG}_scene_indoor_$indoor_type \ --num_scenes 3 --cleanup big_files --configs singleroom.gin fast_solve.gin --overwrite \ - --pipeline_configs slurm_1h monocular indoor_background_configs.gin \ + --pipeline_configs slurm monocular indoor_background_configs.gin \ --pipeline_overrides get_cmd.driver_script=infinigen_examples.generate_indoors sample_scene_spec.seed_range=[0,100] slurm_submit_cmd.slurm_nodelist=$NODECONF \ --overrides compose_indoors.terrain_enabled=True restrict_solving.restrict_parent_rooms=\[\"$indoor_type\"\] & done @@ -39,7 +42,7 @@ if [ "$RUN_NATURE" -eq 1 ]; then python -m infinigen.datagen.manage_jobs --output_folder $OUTPUT_PATH/${JOBTAG}_scene_nature_$nature_type \ --num_scenes 3 --cleanup big_files --overwrite \ --configs $nature_type.gin dev.gin \ - --pipeline_configs slurm_1h monocular \ + --pipeline_configs slurm monocular \ --pipeline_overrides sample_scene_spec.seed_range=[0,100] & done fi From 503cf67ad798b68b2a089e52811ccd50ce260fd4 Mon Sep 17 00:00:00 2001 From: Alexander Raistrick Date: Wed, 4 Dec 2024 15:38:49 -0500 Subject: [PATCH 02/11] Implement tool to parse and group nature scene crash reasons --- .../tools/results/analyze_crash_reasons.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 infinigen/tools/results/analyze_crash_reasons.py diff --git a/infinigen/tools/results/analyze_crash_reasons.py b/infinigen/tools/results/analyze_crash_reasons.py new file mode 100644 index 000000000..9438836ea --- /dev/null +++ b/infinigen/tools/results/analyze_crash_reasons.py @@ -0,0 +1,117 @@ +# Copyright (C) 2024, Princeton University. +# This source code is licensed under the BSD 3-Clause license found in the LICENSE file in the root directory of this source tree. + +# Authors: Alexander Raistrick + +import argparse +import re +from pathlib import Path + +import numpy as np +import pandas as pd + + +def canonicalize_reason(reason): + reason = reason[:60] + return reason + + +def get_configs(log_path, stage): + scene_folder = log_path.parent.parent + run_pipeline = scene_folder / "run_pipeline.sh" + + if not run_pipeline.exists(): + return None + + text = run_pipeline.read_text() + + for line in text.split("\n"): + if "--task " + stage not in line: + continue + + regex = re.compile("-g(.*)-p") + match = regex.search(line) + + if not match: + continue + + return match.groups()[0] + + +def main(args): + crash_reasons = (args.input_folder / "crash_summaries.txt").read_text().split("\n") + + regex = re.compile( + ".*\s.*\s(.*\/([a-zA-Z0-9]*)\/logs\/(.*))\sreason=[\"'](.*)[\"']\snode='(.*)'" + ) + + records = [] + for x in crash_reasons: + if not x: + continue + + match = regex.match(x) + if match: + log_path, job_id, stage, reason, node = match.groups() + stage = stage.split(".")[0].split("_")[0] + configs = get_configs(Path(log_path), stage) + record = { + "log_path": log_path, + "job_id": job_id, + "stage": stage, + "reason": reason, + "node": node, + "configs": configs, + "scenetype": configs.split()[0] if configs else None, + } + records.append(record) + else: + print(f"Could not match: {x}") + + df = pd.DataFrame.from_records(records) + + df["reason_canonical"] = df["reason"].apply(canonicalize_reason) + + print("COMMON CRASH REASONS") + print(df.value_counts("reason_canonical")) + print("") + + print("CRASH SCENETYPES") + print(df.value_counts("scenetype")) + print("") + + for x in df["reason_canonical"].unique(): + print(f"REASON: {x}") + + examples = df[df["reason_canonical"] == x] + sample_idxs = np.random.choice( + len(examples), min(10, len(examples)), replace=False + ) + + stages = examples.value_counts("stage") + if len(stages) > 1: + print(stages) + + scenetype = examples.value_counts("scenetype") + if len(scenetype) > 1: + print(scenetype) + + for idx in sample_idxs: + row = " ".join( + [ + str(x) + for x in examples.iloc[idx][ + ["log_path", "stage", "scenetype"] + ].values + ] + ) + print(f" {row}") + print("") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--input_folder", type=Path, required=True) + args = parser.parse_args() + + main(args) From 27f943308c8544085a8661a58e135e23528de213 Mon Sep 17 00:00:00 2001 From: Alexander Raistrick Date: Mon, 9 Dec 2024 13:22:50 -0500 Subject: [PATCH 03/11] Increase max_full_retries to 40 for camera and other animations, to increase sucessrate --- docs/CHANGELOG.md | 6 +++++- infinigen/__init__.py | 2 +- infinigen/assets/objects/creatures/bird.py | 1 - infinigen/assets/objects/creatures/util/boid_swarm.py | 4 +--- infinigen/core/placement/animation_policy.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 4b94da768..661751fbf 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -154,4 +154,8 @@ v1.11.1 - Make `bnurbs` CPython module optional and not installed by default v1.11.2 -- Fix opengl_gt input file symlink missing \ No newline at end of file +- Fix opengl_gt input file symlink missing + +v1.11.3 +- Increase max camera / object animation random walk trials +- Fix scenetype gin recognition causing crashes for underwater.gin / kelpforest.gin \ No newline at end of file diff --git a/infinigen/__init__.py b/infinigen/__init__.py index ff0548a6e..82213320b 100644 --- a/infinigen/__init__.py +++ b/infinigen/__init__.py @@ -6,7 +6,7 @@ import logging from pathlib import Path -__version__ = "1.11.2" +__version__ = "1.11.3" def repo_root(): diff --git a/infinigen/assets/objects/creatures/bird.py b/infinigen/assets/objects/creatures/bird.py index 75080b015..0a90b5c7b 100644 --- a/infinigen/assets/objects/creatures/bird.py +++ b/infinigen/assets/objects/creatures/bird.py @@ -420,7 +420,6 @@ def create_placeholder(self, i, loc, rot): self.bvh, self.policy, retry_rotation=True, - max_full_retries=30, fatal=True, ) curve.name = f"animhelper:{self}.create_placeholder({i}).path" diff --git a/infinigen/assets/objects/creatures/util/boid_swarm.py b/infinigen/assets/objects/creatures/util/boid_swarm.py index 5c6f0bcb0..e02499b8b 100644 --- a/infinigen/assets/objects/creatures/util/boid_swarm.py +++ b/infinigen/assets/objects/creatures/util/boid_swarm.py @@ -71,9 +71,7 @@ def create_placeholder(self, loc, rot, **kwargs) -> bpy.types.Object: step_range=step_size_range, yaw_dist=("normal", 0, 70), ) - animation_policy.animate_trajectory( - p, self.bvh, policy, retry_rotation=True, max_full_retries=20 - ) + animation_policy.animate_trajectory(p, self.bvh, policy, retry_rotation=True) return p diff --git a/infinigen/core/placement/animation_policy.py b/infinigen/core/placement/animation_policy.py index ec3ef7d65..bfdc47e65 100644 --- a/infinigen/core/placement/animation_policy.py +++ b/infinigen/core/placement/animation_policy.py @@ -578,7 +578,7 @@ def animate_trajectory( policy_func, validate_pose_func=None, max_step_tries=25, - max_full_retries=10, + max_full_retries=40, retry_rotation=False, verbose=True, fatal=False, From 4001dda564eafc2f0f0186c9f0539aba84ad7758 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Wed, 27 Nov 2024 15:24:37 -0500 Subject: [PATCH 04/11] minor bug fixes; not using pytest-ordering; cleanup in autouse fixture --- infinigen/core/init.py | 10 ++++++++-- infinigen/core/rendering/post_render.py | 1 + pyproject.toml | 21 ++++++++++----------- tests/assets/test_scatters_basic.py | 2 +- tests/conftest.py | 10 ++++++++++ 5 files changed, 30 insertions(+), 14 deletions(-) create mode 100644 tests/conftest.py diff --git a/infinigen/core/init.py b/infinigen/core/init.py index aff79258c..eba849cb4 100644 --- a/infinigen/core/init.py +++ b/infinigen/core/init.py @@ -297,12 +297,18 @@ def report_fail(msg): builtin_local_addons = set(a.__name__ for a in addon_utils.modules(refresh=True)) - if addon not in builtin_local_addons and not allow_online: + if ( + (addon not in builtin_local_addons) + and (long not in builtin_local_addons) + and (not allow_online) + ): report_fail(f"{addon=} not found and online install is disabled") try: if long in builtin_local_addons: - logger.info(f"Addon {addon} already in blender local addons, attempt to enable it.") + logger.info( + f"Addon {addon} already in blender local addons, attempt to enable it." + ) bpy.ops.preferences.addon_enable(module=long) else: bpy.ops.extensions.userpref_allow_online() diff --git a/infinigen/core/rendering/post_render.py b/infinigen/core/rendering/post_render.py index 8a960e6df..7dd7f1aec 100644 --- a/infinigen/core/rendering/post_render.py +++ b/infinigen/core/rendering/post_render.py @@ -49,6 +49,7 @@ def load_single_channel(p): def load_depth(p): return load_single_channel(p) + def load_normals(p): return load_exr(p)[..., [2, 0, 1]] * np.array([-1.0, 1.0, 1.0]) diff --git a/pyproject.toml b/pyproject.toml index add0c473a..9233ae48a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,8 +43,8 @@ dependencies = [ "geomdl", # can be removed so long as you do not generate any creatures # used by trimesh, we could specify "trimesh[easy]" but this brings more packages - "python-fcl", - "rtree", + "python-fcl", + "rtree", ] [project.optional-dependencies] @@ -60,7 +60,6 @@ vis = [ ] dev = [ "pytest", - "pytest-ordering", "pytest-cov", "pytest-xdist", "pytest-timeout", @@ -68,7 +67,7 @@ dev = [ "ruff", "isort", "tabulate", - "rapidfuzz", + "rapidfuzz", "pre-commit" ] @@ -117,7 +116,7 @@ filterwarnings = [ "ignore:The value of the smallest subnormal for Date: Wed, 11 Dec 2024 13:29:05 -0500 Subject: [PATCH 05/11] added setuptools as a dependency for terrain extra --- .github/workflows/checks.yml | 2 +- infinigen/OcMesher | 2 +- pyproject.toml | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 97b46fc08..e6ebf6220 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -51,4 +51,4 @@ jobs: - name: Check for copyright statements run: | - ruff check --output-format=github --preview --select CPY001 . \ No newline at end of file + ruff check --output-format=github --preview --select CPY001 . diff --git a/infinigen/OcMesher b/infinigen/OcMesher index d3d1441ab..0f11f5f91 160000 --- a/infinigen/OcMesher +++ b/infinigen/OcMesher @@ -1 +1 @@ -Subproject commit d3d1441ab57c48db3ec40c621fc3d0c323579e8a +Subproject commit 0f11f5f91c3b0aeb779e634606406645fd2d4bcd diff --git a/pyproject.toml b/pyproject.toml index 9233ae48a..d08981a37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,9 @@ dependencies = [ terrain = [ "landlab==2.6.0", "pyrender", + # landlab depends on `setuptools` but doesn't explicitly list it as a + # dependency; We include it to avoid problems with newer Python versions. + "setuptools" ] vis = [ "einops", From 941ad54f19657b93ca95ee41ffe21d27a33aed0d Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Wed, 11 Dec 2024 16:30:29 -0500 Subject: [PATCH 06/11] increasing timeout on account of one failed test --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d08981a37..bb2b857be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -112,7 +112,7 @@ version = {attr = "infinigen.__version__"} testpaths = "tests" junit_family = "xunit2" markers = ["nature", "indoors", "skip_for_ci"] -timeout = 240 +timeout = 480 filterwarnings = [ From d7a2b070af505b3b6f0cae3bbabda4055e8cfe4f Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Wed, 11 Dec 2024 16:32:12 -0500 Subject: [PATCH 07/11] tweaks to get generate_indoors to not segfault --- infinigen/core/constraints/checks.py | 3 +- .../constraints/example_solver/__init__.py | 1 - .../constraints/example_solver/annealing.py | 28 ++++++++++--------- .../greedy/all_substitutions.py | 14 +++++----- .../core/constraints/example_solver/solve.py | 8 +++--- infinigen_examples/generate_indoors.py | 2 +- 6 files changed, 29 insertions(+), 27 deletions(-) diff --git a/infinigen/core/constraints/checks.py b/infinigen/core/constraints/checks.py index eda0695ed..48cc1e18d 100644 --- a/infinigen/core/constraints/checks.py +++ b/infinigen/core/constraints/checks.py @@ -26,7 +26,7 @@ logger = logging.getLogger(__name__) -def iter_domains(node: cl.Node) -> typing.Iterator[r.Domain]: +def iter_domains(node: "cl.Node") -> typing.Iterator["r.Domain"]: match node: case cl.ObjectSetExpression(): yield node, r.constraint_domain(node) @@ -93,6 +93,7 @@ def check_problem_greedy_coverage(prob: cl.Problem, stages: dict[str, r.Domain]) check_coverage_errors(b, coverage, stages) +# def check_unfinalized_constraints(prob: cl.Problem): # TODO return [] diff --git a/infinigen/core/constraints/example_solver/__init__.py b/infinigen/core/constraints/example_solver/__init__.py index 7be9b05bc..620a5888b 100644 --- a/infinigen/core/constraints/example_solver/__init__.py +++ b/infinigen/core/constraints/example_solver/__init__.py @@ -1,3 +1,2 @@ from . import room -from .solve import Solver from .state_def import State diff --git a/infinigen/core/constraints/example_solver/annealing.py b/infinigen/core/constraints/example_solver/annealing.py index 7074a3a0d..5ffac5df7 100644 --- a/infinigen/core/constraints/example_solver/annealing.py +++ b/infinigen/core/constraints/example_solver/annealing.py @@ -131,10 +131,10 @@ def checkpoint(self, state): def validate_lazy_eval( self, - state: State, - consgraph: cl.Problem, - prop_result: evaluate.EvalResult, - filter_domain: r.Domain, + state: "State", + consgraph: "cl.Problem", + prop_result: "evaluate.EvalResult", + filter_domain: "r.Domain", ): test_memo = {} impl_util.DISABLE_BVH_CACHE = True @@ -162,10 +162,10 @@ def validate_lazy_eval( @gin.configurable def _move( self, - consgraph: cl.Node, - state: State, - move: Move, - filter_domain: r.Domain, + consgraph: "cl.Node", + state: "State", + move: "Move", + filter_domain: "r.Domain", do_lazy_eval=True, validate_lazy_eval=False, ): @@ -188,11 +188,11 @@ def _move( def retry_attempt_proposals( self, propose_func: typing.Callable, - consgraph: cl.Node, - state: State, + consgraph: "cl.Node", + state: "State", temp: float, - filter_domain: r.Domain, - ) -> typing.Tuple[Move, evaluate.EvalResult, int]: + filter_domain: "r.Domain", + ) -> typing.Tuple["Move", "evaluate.EvalResult", int]: move_gen = propose_func(consgraph, state, filter_domain, temp) move = None @@ -224,7 +224,9 @@ def curr_temp(self) -> float: temp = np.clip(temp, self.final_temp, self.initial_temp) return temp - def metrop_hastings_with_viol(self, prop_result: evaluate.EvalResult, temp: float): + def metrop_hastings_with_viol( + self, prop_result: "evaluate.EvalResult", temp: float + ): prop_viol = prop_result.viol_count() curr_viol = self.curr_result.viol_count() diff --git a/infinigen/core/constraints/example_solver/greedy/all_substitutions.py b/infinigen/core/constraints/example_solver/greedy/all_substitutions.py index 0d79c970f..c81b8c43f 100644 --- a/infinigen/core/constraints/example_solver/greedy/all_substitutions.py +++ b/infinigen/core/constraints/example_solver/greedy/all_substitutions.py @@ -18,9 +18,9 @@ def _resolve_toplevel_var( - dom: r.Domain, - state: state_def.State, - limits: dict[t.Variable, int] = None, + dom: "r.Domain", + state: "state_def.State", + limits: dict["t.Variable", int] = None, ) -> typing.Iterator[str]: """ Find and yield all valid substitutions of a toplevel VariableTag in a given dom @@ -58,11 +58,11 @@ def _resolve_toplevel_var( def substitutions( - dom: r.Domain, - state: state_def.State, - limits: dict[t.Variable, int] | None = None, + dom: "r.Domain", + state: "state_def.State", + limits: dict["t.Variable", int] | None = None, nonempty: bool = False, -) -> typing.Iterator[r.Domain]: +) -> typing.Iterator["r.Domain"]: """Find all t.Variable in d's tags or relations, and return one Domain for each possible assignment limits cuts off enumeration of each varname with some integer count diff --git a/infinigen/core/constraints/example_solver/solve.py b/infinigen/core/constraints/example_solver/solve.py index f060c327a..30cb448f1 100644 --- a/infinigen/core/constraints/example_solver/solve.py +++ b/infinigen/core/constraints/example_solver/solve.py @@ -131,15 +131,15 @@ def choose_move_type( weights = np.array([s if isinstance(s, (float, int)) else s(t) for s in scheds]) return np.random.choice(funcs, p=weights / weights.sum()) - def solve_rooms(self, scene_seed, consgraph: cl.Problem, filter: r.Domain): + def solve_rooms(self, scene_seed, consgraph: "cl.Problem", filter: "r.Domain"): self.state, _, _ = self.room_solver_fn(scene_seed, consgraph).solve() return self.state @gin.configurable def solve_objects( self, - consgraph: cl.Problem, - filter_domain: r.Domain, + consgraph: "cl.Problem", + filter_domain: "r.Domain", var_assignments: dict[str, str], n_steps: int, desc: str, @@ -216,6 +216,6 @@ def solve_objects( return self.state - def get_bpy_objects(self, domain: r.Domain) -> list[bpy.types.Object]: + def get_bpy_objects(self, domain: "r.Domain") -> list[bpy.types.Object]: objkeys = domain_contains.objkeys_in_dom(domain, self.state) return [self.state.objs[k].obj for k in objkeys] diff --git a/infinigen_examples/generate_indoors.py b/infinigen_examples/generate_indoors.py index de2d94b45..8788fc537 100644 --- a/infinigen_examples/generate_indoors.py +++ b/infinigen_examples/generate_indoors.py @@ -30,12 +30,12 @@ from infinigen.core.constraints import constraint_language as cl from infinigen.core.constraints import reasoning as r from infinigen.core.constraints.example_solver import ( - Solver, greedy, populate, state_def, ) from infinigen.core.constraints.example_solver.room import decorate as room_dec +from infinigen.core.constraints.example_solver.solve import Solver from infinigen.core.placement import camera as cam_util from infinigen.core.util import blender as butil from infinigen.core.util import pipeline From 6034f69b887ac27f72509c116a42ffb9588b8e73 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 19 Dec 2024 11:54:34 -0500 Subject: [PATCH 08/11] added copyright notice on 2 files --- infinigen/core/nodes/utils.py | 5 +++++ infinigen/datagen/manage_jobs.py | 2 +- tests/conftest.py | 5 +++++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/infinigen/core/nodes/utils.py b/infinigen/core/nodes/utils.py index f1a32c62e..f91bab364 100644 --- a/infinigen/core/nodes/utils.py +++ b/infinigen/core/nodes/utils.py @@ -1,3 +1,8 @@ +# Copyright (C) 2024, Princeton University. +# This source code is licensed under the BSD 3-Clause license found in the LICENSE file in the root directory of this source tree. + +# Authors: Lingjie Mei + import logging import bpy diff --git a/infinigen/datagen/manage_jobs.py b/infinigen/datagen/manage_jobs.py index 74605116e..5363b9bf7 100644 --- a/infinigen/datagen/manage_jobs.py +++ b/infinigen/datagen/manage_jobs.py @@ -661,7 +661,7 @@ def manage_datagen_jobs( all_scenes: list[dict], elapsed: float, num_concurrent: int, - disk_sleep_threshold=0.95, + disk_sleep_threshold=0.99, ): if LocalScheduleHandler._inst is not None: sys.path = ORIG_SYS_PATH # hacky workaround because bpy module breaks with multiprocessing diff --git a/tests/conftest.py b/tests/conftest.py index b7bde09cd..5cb0cd57e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,3 +1,8 @@ +# Copyright (C) 2024, Princeton University. +# This source code is licensed under the BSD 3-Clause license found in the LICENSE file in the root directory of this source tree. + +# Authors: Vineet Bansal + import bpy import gin import pytest From cfc28e37fe736a5eeeaf07f5e53c6bf3c46231fc Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Thu, 19 Dec 2024 12:10:05 -0500 Subject: [PATCH 09/11] bumped build version --- infinigen/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infinigen/__init__.py b/infinigen/__init__.py index 82213320b..82944fc94 100644 --- a/infinigen/__init__.py +++ b/infinigen/__init__.py @@ -6,7 +6,7 @@ import logging from pathlib import Path -__version__ = "1.11.3" +__version__ = "1.11.4" def repo_root(): From e30e922a3119b75c43c3bbf7a83fd30674c77799 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Mon, 9 Dec 2024 16:36:07 -0500 Subject: [PATCH 10/11] workflow for pypi release --- .github/workflows/release.yml | 120 +++++++++++++++++++++++++++++ pyproject.toml | 1 - scripts/install/compile_terrain.sh | 40 +++++----- setup.py | 3 +- 4 files changed, 141 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..27c933957 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,120 @@ +name: Build & Release + +on: + push: + branches: + - main + tags: + - '*' + pull_request: + branches: + - main + +jobs: + build_sdist_purepy: + name: Build source/pure python wheels + runs-on: ubuntu-latest + strategy: + fail-fast: false + steps: + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Build source/pure python wheels + run: | + python -m pip install build + python -m build --outdir=wheelhouse + env: + INFINIGEN_MINIMAL_INSTALL: "True" + + - name: Upload to github + uses: actions/upload-artifact@v4 + with: + path: wheelhouse/* + if-no-files-found: error + + build_wheels: + name: Build wheel on ${{ matrix.os }} for ${{ matrix.cibw_archs }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + # we skip Windows since that results in pure Python wheels + # anyway; (compile_terrain.sh is not supported on Windows) + - os: ubuntu-latest + cibw_archs: "x86_64" + - os: ubuntu-latest + cibw_archs: "aarch64" + - os: macos-13 + cibw_archs: "x86_64" + - os: macos-latest + cibw_archs: "arm64" + + steps: + - name: Install macOS dependencies + if: runner.os == 'macOS' + run: | + brew install libomp + echo "MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion | cut -d '.' -f 1-2)" >> $GITHUB_ENV + + - name: Set CXXFLAGS and LDFLAGS for macOS + if: matrix.os == 'macos-13' + run: | + echo "CXX=$(brew --prefix llvm@15)/bin/clang++" >> $GITHUB_ENV + echo "CXXFLAGS=-I/usr/local/opt/libomp/include" >> $GITHUB_ENV + echo "LDFLAGS=-Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp" >> $GITHUB_ENV + + - name: Set CXXFLAGS and LDFLAGS for macOS + if: matrix.os == 'macos-latest' + run: | + echo "CXX=$(brew --prefix llvm@15)/bin/clang++" >> $GITHUB_ENV + echo "CXXFLAGS=-I/opt/homebrew/opt/libomp/include" >> $GITHUB_ENV + echo "LDFLAGS=-Wl,-rpath,/opt/homebrew/opt/libomp/lib -L/opt/homebrew/opt/libomp/lib -lomp" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + with: + submodules: 'recursive' + + - name: Build wheels + uses: pypa/cibuildwheel@v2.21 + with: + output-dir: wheelhouse + env: + CIBW_BUILD: "cp311-*" + # `bpy` is not easily pip-installable on manylinux (no sdists either), + # so we skip tests + CIBW_TEST_COMMAND: "" + CIBW_BUILD_VERBOSITY: 1 + CIBW_ENVIRONMENT: + CXXFLAGS="${{ env.CXXFLAGS }}" + LDFLAGS="${{ env.LDFLAGS }}" + + - name: Upload artifacts to github + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ runner.os }}-${{ matrix.cibw_archs }} + path: ./wheelhouse/*.whl + if-no-files-found: error + + publish_to_pypi: + name: Publish wheels to PyPi + if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') + needs: [build_sdist_purepy, build_wheels] + runs-on: ubuntu-latest + steps: + - name: Download packages + uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + + - name: Upload wheels to pypi + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} + TWINE_REPOSITORY: testpypi + run: | + python -m pip install --upgrade twine + twine upload dist/* diff --git a/pyproject.toml b/pyproject.toml index bb2b857be..785bff097 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,6 @@ keywords = [ "procedural" ] classifiers = [ - "Framework :: Blender", "Programming Language :: Python :: 3", ] diff --git a/scripts/install/compile_terrain.sh b/scripts/install/compile_terrain.sh index 5f74dc267..d8bab46f4 100644 --- a/scripts/install/compile_terrain.sh +++ b/scripts/install/compile_terrain.sh @@ -71,23 +71,26 @@ fi OS=$(uname -s) ARCH=$(uname -m) -if [ "${OS}" = "Linux" ]; then - alias gx1="g++ -O3 -c -fpic -fopenmp " - alias gx2="g++ -O3 -shared -fopenmp " -elif [ "${OS}" = "Darwin" ]; then - if [ "${ARCH}" = "arm64" ]; then - compiler="/opt/homebrew/opt/llvm/bin/clang++" +if [ -n "$CXX" ]; then + compiler="$CXX" +else + if [ "${OS}" = "Linux" ]; then + compiler="g++" + elif [ "${OS}" = "Darwin" ]; then + if [ "${ARCH}" = "arm64" ]; then + compiler="/opt/homebrew/opt/llvm/bin/clang++" + else + compiler="/usr/local/opt/llvm/bin/clang++" + fi else - compiler="/usr/local/opt/llvm/bin/clang++" + echo "Unsupported OS" + exit -1 fi - alias gx1="${compiler} -O3 -c -fpic -fopenmp " - alias gx2="${compiler} -O3 -shared -fopenmp " - alias gx="${compiler} -O3 -fpic -shared -fopenmp " -else - echo "Unsupported OS" - exit -1 fi +alias gx1="${compiler} \$CXXFLAGS -O3 -c -fpic -fopenmp " +alias gx2="${compiler} \$LDFLAGS -O3 -shared -fopenmp " + mkdir -p lib/cpu/utils gx1 -o lib/cpu/utils/FastNoiseLite.o source/cpu/utils/FastNoiseLite.cpp gx2 -o lib/cpu/utils/FastNoiseLite.so lib/cpu/utils/FastNoiseLite.o @@ -121,16 +124,11 @@ gx2 -o lib/cpu/meshing/utils.so lib/cpu/meshing/utils.o echo "compiled lib/cpu/meshing/utils.so" if [ "${OS}" = "Darwin" ]; then - if [ "${ARCH}" = "arm64" ]; then - alias gx1="CPATH=/opt/homebrew/include:${CPATH} g++ -O3 -c -fpic -std=c++17" - alias gx2="CPATH=/opt/homebrew/include:${CPATH} g++ -O3 -shared -std=c++17" - else - alias gx1="CPATH=/usr/local/include:${CPATH} g++ -O3 -c -fpic -std=c++17" - alias gx2="CPATH=/usr/local/include:${CPATH} g++ -O3 -shared -std=c++17" - fi + alias gx1="CPATH=/opt/homebrew/include:${CPATH} ${compiler} -O3 -c -fpic -std=c++17" + alias gx2="CPATH=/opt/homebrew/include:${CPATH} ${compiler} -O3 -shared -std=c++17" fi mkdir -p lib/cpu/soil_machine -gx1 -o lib/cpu/soil_machine/SoilMachine.o source/cpu/soil_machine/SoilMachine.cpp +gx1 -I../datagen/customgt/dependencies/glm -o lib/cpu/soil_machine/SoilMachine.o source/cpu/soil_machine/SoilMachine.cpp gx2 -o lib/cpu/soil_machine/SoilMachine.so lib/cpu/soil_machine/SoilMachine.o echo "compiled lib/cpu/soil_machine/SoilMachine.so" diff --git a/setup.py b/setup.py index d7c7fb75e..332cb96e3 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,8 @@ def ensure_submodules(): ) -ensure_submodules() +if not MINIMAL_INSTALL: + ensure_submodules() # inspired by https://github.com/pytorch/pytorch/blob/161ea463e690dcb91a30faacbf7d100b98524b6b/setup.py#L290 # theirs seems to not exclude dist_info but this causes duplicate compiling in my tests From be0b14358b32c126f9892a13550fc345df75be68 Mon Sep 17 00:00:00 2001 From: Vineet Bansal Date: Mon, 9 Dec 2024 16:37:52 -0500 Subject: [PATCH 11/11] using pypi instead of testpypi --- .github/workflows/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 27c933957..4a1b5b468 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -114,7 +114,6 @@ jobs: env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} - TWINE_REPOSITORY: testpypi run: | python -m pip install --upgrade twine twine upload dist/*