diff --git a/README.md b/README.md index a63146f5..3a1e41af 100644 --- a/README.md +++ b/README.md @@ -304,4 +304,22 @@ python -m carps.utils.check_missing rundir # Gather logs from files python -m carps.analysis.gather_data rundir +``` + +# Commands using Flags +```bash +# Fill database +carps --create_cluster_configs --optimizer DUMMY/config --problem DUMMY/config + +# Run from database (with local env) +carps --run_from_db + +# Run local with filelogging +carps --run --optimizer DUMMY/config --problem DUMMY/config --seed 0 + +# Check missing runs (creates runcommands_missing.sh) +carps --check_missing + +# Gather logs from files +carps --gather_data ``` \ No newline at end of file diff --git a/carps/cli.py b/carps/cli.py new file mode 100644 index 00000000..4d9b4828 --- /dev/null +++ b/carps/cli.py @@ -0,0 +1,113 @@ +""" +# CLI. + +This module defines command-line options using flags. + +This includes the entry point for the programs execution. +""" + +import subprocess +from typing import Any + +from absl import app, flags + +FLAGS = flags.FLAGS +flags.DEFINE_boolean( + "create_cluster_configs", + short_name="c", + default=False, + help="Create cluster configs in the database. Passing paths to the " + "optimizer and problem config to be used is required.", +) +flags.DEFINE_boolean( + "run_from_db", + short_name="db", + default=False, + help="Start runs for all configs in the database.", +) +flags.DEFINE_boolean( + "run", + short_name="r", + default=False, + help="Start run for provided optimizer, problem, and seed. Passing paths to " + "the optimizer and problem config to be used is required, as well as " + "providing a seed.", +) +flags.DEFINE_boolean( + "reset_experiments", + short_name="re", + default=False, + help="Reset status of experiments in the database.", +) +flags.DEFINE_boolean( + "check_missing", short_name="cm", default=False, help="Check for missing runs." +) +flags.DEFINE_boolean( + "gather_data", + short_name="gd", + default=False, + help="Gather logs from the run files.", +) +flags.DEFINE_string( + "optimizer", short_name="o", default=None, help="Path to optimizer config." +) +flags.DEFINE_string( + "problem", short_name="p", default=None, help="Path to problem config." +) +flags.DEFINE_string( + "seed", + short_name="s", + default=None, + help="Seed to be used when running the experiment.", +) +flags.DEFINE_string( + "rundir", + short_name="rd", + default="runs", + help="Path to the run directory where results and logs are stored.", +) + + +def main(argv: Any) -> None: + """Call the function to execute.""" + if FLAGS.create_cluster_configs: + if FLAGS.optimizer is None or FLAGS.problem is None: + print("Please provide optimizer and problem.") + return + subprocess.call( + [ + "python", + "-m", + "carps.container.create_cluster_configs", + f"+optimizer={FLAGS.optimizer}", + f"+problem={FLAGS.problem}", + ] + ) + if FLAGS.run_from_db: + subprocess.call(["python", "-m", "carps.run_from_db"]) + if FLAGS.run: + if FLAGS.optimizer is None or FLAGS.problem is None or FLAGS.seed is None: + print("Please provide optimizer, problem and seed.") + return + subprocess.call( + [ + "python", + "-m", + "carps.run", + f"+optimizer={FLAGS.optimizer}", + f"+problem={FLAGS.problem}", + f"seed={FLAGS.seed}", + ] + ) + if FLAGS.reset_experiments: + subprocess.call(["python", "-m", "carps.utils.database.reset_experiments"]) + if FLAGS.check_missing: + subprocess.call(["python", "-m", "carps.utils.check_missing", FLAGS.rundir]) + if FLAGS.gather_data: + subprocess.call(["python", "-m", "carps.analysis.gather_data", FLAGS.rundir]) + + +if __name__ == "__main__": + pass + +app.run(main) diff --git a/carps/container/create_cluster_configs.py b/carps/container/create_cluster_configs.py index 115b287d..a10f0976 100644 --- a/carps/container/create_cluster_configs.py +++ b/carps/container/create_cluster_configs.py @@ -25,22 +25,33 @@ def main(cfg: DictConfig) -> None: """ cfg_dict = OmegaConf.to_container(cfg=cfg, resolve=True) - experiment_configuration_file_path = cfg.pyexperimenter_configuration_file_path or Path(__file__).parent / "py_experimenter.yaml" - - database_credential_file_path = cfg.database_credential_file_path or Path(__file__).parent / "credentials.yaml" - if database_credential_file_path is not None and not database_credential_file_path.exists(): + experiment_configuration_file_path = ( + cfg.pyexperimenter_configuration_file_path + or Path(__file__).parent / "py_experimenter.yaml" + ) + + database_credential_file_path = ( + cfg.database_credential_file_path or Path(__file__).parent / "credentials.yaml" + ) + if ( + database_credential_file_path is not None + and not database_credential_file_path.exists() + ): database_credential_file_path = None - experimenter = PyExperimenter(experiment_configuration_file_path=experiment_configuration_file_path, - name="carps", - database_credential_file_path=database_credential_file_path, - log_level=logging.INFO, - use_ssh_tunnel=OmegaConf.load(experiment_configuration_file_path).PY_EXPERIMENTER.Database.use_ssh_tunnel - ) + experimenter = PyExperimenter( + experiment_configuration_file_path=experiment_configuration_file_path, + name="carps", + database_credential_file_path=database_credential_file_path, + log_level=logging.INFO, + use_ssh_tunnel=OmegaConf.load( + experiment_configuration_file_path + ).PY_EXPERIMENTER.Database.use_ssh_tunnel, + ) cfg_json = OmegaConf.to_container(cfg, resolve=True) - # This value will always be unique so it + # This value will always be unique so it # disables duplicate checking when adding entries to the database. # Py_experimenter will add a creation date so the information # is not lost. @@ -54,19 +65,23 @@ def main(cfg: DictConfig) -> None: cfg_str = json.dumps(cfg_json) cfg_hash = hashlib.sha256(cfg_str.encode()).hexdigest() - rows = [{ - "config": cfg_str, - "config_hash": cfg_hash, - "benchmark_id": cfg_dict["benchmark_id"], - "problem_id": cfg_dict["problem_id"], - "optimizer_id": cfg_dict["optimizer_id"], - "optimizer_container_id": cfg_dict["optimizer_container_id"], - "seed": cfg_dict["seed"], - "n_trials": cfg_dict["task"]["n_trials"], - "time_budget": cfg_dict["task"]["time_budget"], - }] - - column_names = list(experimenter.db_connector.database_configuration.keyfields.keys()) + rows = [ + { + "config": cfg_str, + "config_hash": cfg_hash, + "benchmark_id": cfg_dict["benchmark_id"], + "problem_id": cfg_dict["problem_id"], + "optimizer_id": cfg_dict["optimizer_id"], + "optimizer_container_id": cfg_dict["optimizer_container_id"], + "seed": cfg_dict["seed"], + "n_trials": cfg_dict["task"]["n_trials"], + "time_budget": cfg_dict["task"]["time_budget"], + } + ] + + column_names = list( + experimenter.db_connector.database_configuration.keyfields.keys() + ) exists = False @@ -78,16 +93,22 @@ def main(cfg: DictConfig) -> None: for e in existing_rows: if e["config_hash"] == cfg_hash: exists = True - logger.info("Experiment not added to the database because config hash already exists!") + logger.info( + "Experiment not added to the database because config hash already exists!" + ) except DatabaseConnectionError as e: if "1146" in e.args[0] or "no such table" in e.args[0]: - logger.info("Database empty, will fill.:)") + logger.info("Database does not exist and will be created.") else: raise e if not exists: experimenter.fill_table_with_rows(rows) - + logger.info("Experiment was added to the database.") + logger.info( + "Next, add more experiments or run all experiments in the database " + "by calling 'carps --run_from_db'." + ) return None diff --git a/pyproject.toml b/pyproject.toml index 0c4ed374..9864edc8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,8 +3,8 @@ name = "CARP-S" version = "0.1.0" dependencies = [ "typing_extensions", # Better typing - "tomli", - "pre-commit", + "tomli", + "pre-commit", "pytest==6.2.4", "coverage==4.5.4", "ruff", @@ -45,6 +45,9 @@ classifiers = [ ] license = { file = "LICENSE" } +[project.scripts] +carps = "carps.cli:main" + [project.optional-dependencies] dev = ["carps[doc, tooling, test, examples]"] tooling = ["commitizen", "pre-commit", "ruff"] diff --git a/setup.py b/setup.py index a69b2793..88d7360a 100644 --- a/setup.py +++ b/setup.py @@ -58,6 +58,9 @@ def read_file(filepath: str) -> str: "numpy" ], extras_require=extras_require, + entry_points={ + "console_scripts": ["carps = carps.cli:main"], + }, test_suite="pytest", platforms=["Linux"], classifiers=[