diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbbc910..6d2878d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,10 @@ jobs: steps: - uses: actions/checkout@v3 with: - path: times-excel-reader + path: xl2times - name: Install tool and dependencies - working-directory: times-excel-reader + working-directory: xl2times run: | python -m venv .venv source .venv/bin/activate @@ -28,7 +28,7 @@ jobs: python -m pip install -e .[dev] - name: Check code formatting - working-directory: times-excel-reader + working-directory: xl2times run: | source .venv/bin/activate pre-commit install @@ -44,12 +44,12 @@ jobs: - uses: actions/checkout@v3 with: repository: olejandro/demos-dd - path: times-excel-reader/benchmarks/dd + path: xl2times/benchmarks/dd - uses: actions/checkout@v3 with: repository: olejandro/demos-xlsx - path: times-excel-reader/benchmarks/xlsx + path: xl2times/benchmarks/xlsx token: ${{ secrets.GH_PAT_DEMOS_XLSX }} # ---------- Prepare TIMES Ireland Model @@ -59,12 +59,12 @@ jobs: - uses: actions/checkout@v3 with: repository: samwebster/times-ireland-model - path: times-excel-reader/benchmarks/xlsx/Ireland + path: xl2times/benchmarks/xlsx/Ireland - uses: actions/checkout@v3 with: repository: olejandro/times-ireland-model_gams - path: times-excel-reader/benchmarks/dd/Ireland + path: xl2times/benchmarks/dd/Ireland # ---------- Install GAMS @@ -87,7 +87,7 @@ jobs: # ---------- Run tool, check for regressions - name: Run tool on all benchmarks - working-directory: times-excel-reader + working-directory: xl2times # Use tee to also save the output to out.txt so that the summary table can be # printed again in the next step. # Save the return code to retcode.txt so that the next step can fail the action @@ -102,7 +102,7 @@ jobs: echo ${PIPESTATUS[0]} > retcode.txt) - name: Print summary - working-directory: times-excel-reader + working-directory: xl2times run: | sed -n '/Benchmark *Time.*Accuracy/h;//!H;$!d;x;//p' out.txt exit $(cat retcode.txt) diff --git a/MANIFEST.in b/MANIFEST.in index 80e2533..dce8b6e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,2 @@ -include times_reader/config/* -include times_reader/gams_scaffold/* +include xl2times/config/* +include xl2times/gams_scaffold/* diff --git a/README.md b/README.md index cb60e08..72262ec 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Project +# xl2times **Note: this tool is a work in progress and not yet in a useful state** -This project is an open source tool to convert TIMES model Excel input files to DD format ready for processing by [GAMS](https://www.gams.com/). The intention is to make it easier for people to reproduce research results on TIMES models. +`xl2times` is an open source tool to convert TIMES model Excel input files to DD format ready for processing by [GAMS](https://www.gams.com/). The intention is to make it easier for people to reproduce research results on TIMES models. [TIMES](https://iea-etsap.org/index.php/etsap-tools/model-generators/times) is an energy systems model generator from the International Energy Agency that is used around the world to inform energy policy. It is fully explained in the [TIMES Model Documentation](https://iea-etsap.org/index.php/documentation). @@ -11,14 +11,19 @@ The Excel input format accepted by this tool is documented in the [TIMES Model D ## Installation and Basic Usage -The tool is not (yet) on PyPI, but you can still install it using pip (preferably in a virtual environment) by cloning the repository and running the following command in the root directory: +You can install the latest published version of the tool from PyPI using pip (preferably in a virtual environment): +```bash +pip install xl2times +``` + +You can also install the latest development version by cloning this repository and running the following command in the root directory: ```bash pip install . ``` After installation, run the following command to see the basic usage and available options: ```bash -times-excel-reader --help +xl2times --help ``` ## Development Setup @@ -48,15 +53,26 @@ If you want to skip these pre-commit steps for a particular commit, if for insta git commit --no-verify ``` +### Publishing the Tool + +To publish a new version of the tool on PyPI, update the version number in `pyproject.toml`, and then run: +```bash +python -m pip install --upgrade build +python -m pip install --upgrade twine +rm -rf dist +python -m build +python -m twine upload dist/* +``` + ## Debugging Regressions If your change is causing regressions on one of the benchmarks, a useful way to debug and find the difference is to run the tool in verbose mode and compare the intermediate tables. For example, if your branch has regressions on Demo 1: ```bash # First, on the `main` branch: -times-excel-reader benchmarks/xlsx/DemoS_001 --output_dir benchmarks/out/DemoS_001-all --ground_truth_dir benchmarks/csv/DemoS_001-all --verbose > before 2>&1 +xl2times benchmarks/xlsx/DemoS_001 --output_dir benchmarks/out/DemoS_001-all --ground_truth_dir benchmarks/csv/DemoS_001-all --verbose > before 2>&1 # Then, on your branch: git checkout my-branch-name -times-excel-reader benchmarks/xlsx/DemoS_001 --output_dir benchmarks/out/DemoS_001-all --ground_truth_dir benchmarks/csv/DemoS_001-all --verbose > after 2>&1 +xl2times benchmarks/xlsx/DemoS_001 --output_dir benchmarks/out/DemoS_001-all --ground_truth_dir benchmarks/csv/DemoS_001-all --verbose > after 2>&1 # And then compare the files `before` and `after` code -d before after ``` diff --git a/pyproject.toml b/pyproject.toml index 93f60dd..e2fd43c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,14 +3,14 @@ requires = ["setuptools>=61.0.0", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools] -packages = ["times_reader"] +packages = ["xl2times"] [project] -name = "times-excel-reader" +name = "xl2times" version = "0.1.0" -description = 'An open source tool to convert TIMES model Excel input files to DD format ready for processing by GAMS' +description = 'An open source tool to convert Excel input files for TIMES models to the DD format accepted by GAMS' readme = "README.md" -requires-python = ">=3.8" +requires-python = ">=3.9" license = { file = "LICENSE" } keywords = [] classifiers = [ @@ -23,7 +23,7 @@ dependencies = [ "GitPython >= 3.1.31, < 3.2", "more-itertools", "openpyxl >= 3.0, < 3.1", - "pandas >= 1.5", + "pandas >= 2.1", ] [project.optional-dependencies] @@ -35,4 +35,4 @@ Issues = "https://github.com/etsap-TIMES/times-excel-reader/issues" Source = "https://github.com/etsap-TIMES/times-excel-reader" [project.scripts] -times-excel-reader = "times_reader.__main__:main" +xl2times = "xl2times.__main__:main" diff --git a/pyrightconfig.json b/pyrightconfig.json index 2ad499a..9e24664 100644 --- a/pyrightconfig.json +++ b/pyrightconfig.json @@ -1,6 +1,6 @@ { "include": [ - "times_reader", + "xl2times", "utils", "times_excel_reader.py" ], diff --git a/requirements.txt b/requirements.txt index 8961526..395357a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ openpyxl >= 3.0, < 3.1 -pandas >= 1.5 +pandas >= 2.1 more-itertools pre-commit black diff --git a/utils/dd_to_csv.py b/utils/dd_to_csv.py index 4c35ca6..5028cc4 100644 --- a/utils/dd_to_csv.py +++ b/utils/dd_to_csv.py @@ -181,7 +181,7 @@ def convert_dd_to_tabular(basedir: str, output_dir: str) -> None: os.makedirs(set_path, exist_ok=True) # Extract headers with key=param_name and value=List[attributes] - lines = list(open("times_reader/config/times_mapping.txt", "r")) + lines = list(open("xl2times/config/times_mapping.txt", "r")) headers_data = dict() for line in lines: line = line.strip() diff --git a/utils/run_benchmarks.py b/utils/run_benchmarks.py index 1db8ad2..5a702d5 100644 --- a/utils/run_benchmarks.py +++ b/utils/run_benchmarks.py @@ -39,7 +39,7 @@ def run_gams_gdxdiff( # Copy GAMS scaffolding scaffolding_folder = path.join( - path.dirname(path.realpath(__file__)), "..", "times_reader", "gams_scaffold" + path.dirname(path.realpath(__file__)), "..", "xl2times", "gams_scaffold" ) shutil.copytree(scaffolding_folder, out_folder, dirs_exist_ok=True) # Create link to TIMES source @@ -168,7 +168,7 @@ def run_benchmark( args.append(xl_folder) start = time.time() res = subprocess.run( - ["times-excel-reader"] + args, + ["xl2times"] + args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, diff --git a/times_reader/__init__.py b/xl2times/__init__.py similarity index 100% rename from times_reader/__init__.py rename to xl2times/__init__.py diff --git a/times_reader/__main__.py b/xl2times/__main__.py similarity index 100% rename from times_reader/__main__.py rename to xl2times/__main__.py diff --git a/times_reader/config/__init__.py b/xl2times/config/__init__.py similarity index 100% rename from times_reader/config/__init__.py rename to xl2times/config/__init__.py diff --git a/times_reader/config/times-info.json b/xl2times/config/times-info.json similarity index 100% rename from times_reader/config/times-info.json rename to xl2times/config/times-info.json diff --git a/times_reader/config/times_mapping.txt b/xl2times/config/times_mapping.txt similarity index 100% rename from times_reader/config/times_mapping.txt rename to xl2times/config/times_mapping.txt diff --git a/times_reader/config/veda-attr-defaults.json b/xl2times/config/veda-attr-defaults.json similarity index 100% rename from times_reader/config/veda-attr-defaults.json rename to xl2times/config/veda-attr-defaults.json diff --git a/times_reader/config/veda-tags.json b/xl2times/config/veda-tags.json similarity index 100% rename from times_reader/config/veda-tags.json rename to xl2times/config/veda-tags.json diff --git a/times_reader/datatypes.py b/xl2times/datatypes.py similarity index 97% rename from times_reader/datatypes.py rename to xl2times/datatypes.py index d5659db..e0967f0 100644 --- a/times_reader/datatypes.py +++ b/xl2times/datatypes.py @@ -179,7 +179,7 @@ def __init__( def _process_times_info(times_info_file: str) -> Tuple[Iterable[str], Set[str]]: # Read times_info_file and compute dd_table_order: # We output tables in order by categories: set, subset, subsubset, md-set, and parameter - with resources.open_text("times_reader.config", times_info_file) as f: + with resources.open_text("xl2times.config", times_info_file) as f: table_info = json.load(f) categories = ["set", "subset", "subsubset", "md-set", "parameter"] cat_to_tables = defaultdict(list) @@ -223,7 +223,7 @@ def _read_mappings(filename: str) -> List[TimesXlMap]: """ mappings = [] dropped = [] - with resources.open_text("times_reader.config", filename) as file: + with resources.open_text("xl2times.config", filename) as file: while True: line = file.readline().rstrip() if line == "": @@ -280,7 +280,7 @@ def to_tag(s: str) -> Tag: return Tag("~" + s.upper()) # Read veda_tags_file - with resources.open_text("times_reader.config", veda_tags_file) as f: + with resources.open_text("xl2times.config", veda_tags_file) as f: veda_tags_info = json.load(f) # Check that all the tags we use are present in veda_tags_file @@ -336,7 +336,7 @@ def _read_veda_attr_defaults( veda_attr_defaults_file: str, ) -> Tuple[Dict[str, Dict[str, list]], Set[str]]: # Read veda_tags_file - with resources.open_text("times_reader.config", veda_attr_defaults_file) as f: + with resources.open_text("xl2times.config", veda_attr_defaults_file) as f: defaults = json.load(f) veda_attr_defaults = { diff --git a/times_reader/excel.py b/xl2times/excel.py similarity index 100% rename from times_reader/excel.py rename to xl2times/excel.py diff --git a/times_reader/gams_scaffold/__init__.py b/xl2times/gams_scaffold/__init__.py similarity index 100% rename from times_reader/gams_scaffold/__init__.py rename to xl2times/gams_scaffold/__init__.py diff --git a/times_reader/gams_scaffold/gams.opt b/xl2times/gams_scaffold/gams.opt similarity index 100% rename from times_reader/gams_scaffold/gams.opt rename to xl2times/gams_scaffold/gams.opt diff --git a/times_reader/gams_scaffold/runmodel.gms b/xl2times/gams_scaffold/runmodel.gms similarity index 100% rename from times_reader/gams_scaffold/runmodel.gms rename to xl2times/gams_scaffold/runmodel.gms diff --git a/times_reader/gams_scaffold/scenario.run b/xl2times/gams_scaffold/scenario.run similarity index 100% rename from times_reader/gams_scaffold/scenario.run rename to xl2times/gams_scaffold/scenario.run diff --git a/times_reader/transforms.py b/xl2times/transforms.py similarity index 100% rename from times_reader/transforms.py rename to xl2times/transforms.py diff --git a/times_reader/utils.py b/xl2times/utils.py similarity index 100% rename from times_reader/utils.py rename to xl2times/utils.py