diff --git a/qgreenland/cli/run.py b/qgreenland/cli/run.py index 1a4b6679..f248a129 100644 --- a/qgreenland/cli/run.py +++ b/qgreenland/cli/run.py @@ -73,9 +73,8 @@ def run( # https://github.com/click-contrib/sphinx-click/issues/86#issuecomment-991196764 from qgreenland.util.config.config import get_config, init_config from qgreenland.util.luigi.tasks.pipeline import ( - LayerPipelines, - QGreenlandAll, - QGreenlandNoZip, + QGreenlandPackages, + QGreenlandPackagesNoZip, ) if force_package_zip and force_no_package_zip: @@ -92,11 +91,15 @@ def run( if fetch_only: # Don't do anything except fetch the input asset for each layer. - tasks = [LayerPipelines(fetch_only=fetch_only)] + # TODO: How to keep "fetch all" functionality? Bring back LayerPipelines solely + # for this? Or make the user pass a specific package, or specific a magic word + # like "__all__"? ¯\_(ツ)_/¯ + # tasks = [LayerPipelines(fetch_only=fetch_only)] + ... elif skip_zip: - tasks = [QGreenlandNoZip()] + tasks = [QGreenlandPackagesNoZip()] else: - tasks = [QGreenlandAll()] + tasks = [QGreenlandPackages()] print(f"Running tasks: {str(tasks)}") print() diff --git a/qgreenland/config/helpers/layers/continental_shelf.py b/qgreenland/config/helpers/layers/continental_shelf.py index b3c11a84..23d61a8c 100644 --- a/qgreenland/config/helpers/layers/continental_shelf.py +++ b/qgreenland/config/helpers/layers/continental_shelf.py @@ -77,7 +77,7 @@ def make_layers() -> list[Layer]: id=f"continental_shelf_{key}", title=params["title"], description=params["description"], - tags=[], + packaging_tags=["core"], input=LayerInput( dataset=dataset, asset=dataset.assets[key], diff --git a/qgreenland/config/helpers/layers/esa_cci_surface_elev.py b/qgreenland/config/helpers/layers/esa_cci_surface_elev.py index 545f4f25..c72b9561 100644 --- a/qgreenland/config/helpers/layers/esa_cci_surface_elev.py +++ b/qgreenland/config/helpers/layers/esa_cci_surface_elev.py @@ -43,7 +43,7 @@ def surface_elevation_layer( id=f"surface_elevation_change_{variable.lower()}_{start_year}_{end_year}", title=f"Surface elevation change {start_year}-{end_year}", description=description, - tags=[], + packaging_tags=["core"], style=style, input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/geological_map.py b/qgreenland/config/helpers/layers/geological_map.py index d7974846..9be58328 100644 --- a/qgreenland/config/helpers/layers/geological_map.py +++ b/qgreenland/config/helpers/layers/geological_map.py @@ -39,7 +39,7 @@ def make_layer(*, layer_id: str, layer_params: dict) -> Layer: id=layer_id, title=layer_params["title"], description=layer_params["description"], - tags=[], + packaging_tags=["core"], style=layer_params["style"], input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/gravity_anomalies_geoid.py b/qgreenland/config/helpers/layers/gravity_anomalies_geoid.py index 321c1dcb..be182b73 100644 --- a/qgreenland/config/helpers/layers/gravity_anomalies_geoid.py +++ b/qgreenland/config/helpers/layers/gravity_anomalies_geoid.py @@ -23,7 +23,7 @@ def _make_layer( id=layer_id, title=title, description=description, - tags=[], + packaging_tags=["core"], style=style, input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/lonlat.py b/qgreenland/config/helpers/layers/lonlat.py index 8c6f52e1..99d54c67 100644 --- a/qgreenland/config/helpers/layers/lonlat.py +++ b/qgreenland/config/helpers/layers/lonlat.py @@ -45,7 +45,7 @@ def _make_lonlat_layer( id=asset.id, title=f"{title_prefix} lines ({deg} degree)", description=(f"Lines of {title_prefix.lower()} in {deg}-degree resolution."), - tags=["reference"], + packaging_tags=["core"], style="lonlat", input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/nunagis_protected_areas.py b/qgreenland/config/helpers/layers/nunagis_protected_areas.py index cadc8452..8749a45d 100644 --- a/qgreenland/config/helpers/layers/nunagis_protected_areas.py +++ b/qgreenland/config/helpers/layers/nunagis_protected_areas.py @@ -152,7 +152,7 @@ def _make_layer( id=layer_id, title=title, description=description, - tags=[], + packaging_tags=["core"], style=style, input=LayerInput( dataset=nunagis_protected_areas, diff --git a/qgreenland/config/helpers/layers/racmo.py b/qgreenland/config/helpers/layers/racmo.py index 4aea2e1b..68937d14 100644 --- a/qgreenland/config/helpers/layers/racmo.py +++ b/qgreenland/config/helpers/layers/racmo.py @@ -19,7 +19,7 @@ def _make_racmo_wind_vectors() -> Layer: """Averaged annual mean wind direction in meters per second from RACMO2.3p2 for the period 1958-2019.""" ), - tags=[], + packaging_tags=["core"], style="racmo_wind_vectors", input=LayerInput( dataset=dataset, @@ -46,7 +46,7 @@ def _make_racmo_wind_speed() -> Layer: """Averaged annual mean wind speed in meters per second from RACMO2.3p2 for the period 1958-2019.""" ), - tags=[], + packaging_tags=["core"], style="racmo_wind_speed", input=LayerInput( dataset=dataset, @@ -154,7 +154,7 @@ def _make_masked_racmo_layer( id=layer_id, title=title, description=description, - tags=[], + packaging_tags=["core"], style=style, input=LayerInput( dataset=dataset, @@ -267,7 +267,7 @@ def make_racmo_supplemental_layers() -> list[Layer]: id=layer_id, title=params["title"], description=params["description"], - tags=[], + packaging_tags=["core"], style=params["style"], input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/sea_ice_age.py b/qgreenland/config/helpers/layers/sea_ice_age.py index c279371a..0007ea3f 100644 --- a/qgreenland/config/helpers/layers/sea_ice_age.py +++ b/qgreenland/config/helpers/layers/sea_ice_age.py @@ -38,7 +38,7 @@ def sea_ice_age_layer(year: int, age_type: AgeType) -> Layer: of {age_type} extent chosen based on NSDIC's Sea Ice Index 5-day average.""" ), - tags=[], + packaging_tags=["core"], style="sea_ice_age", input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/streams_outlets_basins.py b/qgreenland/config/helpers/layers/streams_outlets_basins.py index e6993dd2..e565d01b 100644 --- a/qgreenland/config/helpers/layers/streams_outlets_basins.py +++ b/qgreenland/config/helpers/layers/streams_outlets_basins.py @@ -72,7 +72,7 @@ id=layer_id, title=layer_id.replace("_", " ").capitalize(), description=params["description"], - tags=[], + packaging_tags=["core"], style=layer_id.replace("_filled", ""), input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/helpers/layers/territorial_waters.py b/qgreenland/config/helpers/layers/territorial_waters.py index 6582d752..20276295 100644 --- a/qgreenland/config/helpers/layers/territorial_waters.py +++ b/qgreenland/config/helpers/layers/territorial_waters.py @@ -73,7 +73,7 @@ def make_layers() -> list[Layer]: id=key, title=params["title"], description=params["description"], - tags=[], + packaging_tags=["core"], input=LayerInput( dataset=dataset, asset=dataset.assets["only"], diff --git a/qgreenland/config/helpers/layers/wmm.py b/qgreenland/config/helpers/layers/wmm.py index 1cf5c7cb..6fada465 100644 --- a/qgreenland/config/helpers/layers/wmm.py +++ b/qgreenland/config/helpers/layers/wmm.py @@ -24,8 +24,7 @@ def make_boz_layer(*, year: int) -> Layer: BoZ, where caution must be exercised while using a compass. Compass accuracy may be degraded in this region. """, - tags=["wmm"], - in_package=True, + packaging_tags=["core"], show=False, style="blackout_zones", input=LayerInput( @@ -256,7 +255,7 @@ def make_wmm_variable_layer( description=variable_config["description"], # We keep the main field declination layers (`d`) in the core # package. All other variables will only be available from the plugin. - in_package=True if variable == "d" else False, + packaging_tags=["core"] if variable == "d" else [], style="wmm_contours", input=LayerInput( dataset=wmm.wmm, diff --git a/qgreenland/config/helpers/layers/woa.py b/qgreenland/config/helpers/layers/woa.py index 5aa4d850..91ee343d 100644 --- a/qgreenland/config/helpers/layers/woa.py +++ b/qgreenland/config/helpers/layers/woa.py @@ -78,7 +78,7 @@ def make_layer(*, dataset, depth, season, variable, units) -> Layer: remotely sensed observations, also require a more recent climatology.""" ), - tags=[], + packaging_tags=["core"], style=f"seawater_{variable}", input=LayerInput( dataset=dataset, diff --git a/qgreenland/config/layers/Basemaps/land_ocean_shape.py b/qgreenland/config/layers/Basemaps/land_ocean_shape.py index f4437d3c..ec6893b1 100644 --- a/qgreenland/config/layers/Basemaps/land_ocean_shape.py +++ b/qgreenland/config/layers/Basemaps/land_ocean_shape.py @@ -14,7 +14,7 @@ def make_land_ocean_layer(layer_id: str) -> Layer: id=layer_id, title=layer_id.capitalize(), description=(f"""Polygons representing the {layer_id}."""), - tags=[], + packaging_tags=["foo"] if layer_id == "land" else ["core", "foo"], style=layer_id, input=LayerInput( dataset=layer_params[layer_id], diff --git a/qgreenland/config/layers/Reference/arctic_circle.py b/qgreenland/config/layers/Reference/arctic_circle.py index 4da5cf6c..9ef45a23 100644 --- a/qgreenland/config/layers/Reference/arctic_circle.py +++ b/qgreenland/config/layers/Reference/arctic_circle.py @@ -11,7 +11,7 @@ not set on the summer solstice, and does not rise on the winter solstice.""" ), - tags=[], + packaging_tags=["core"], style="arctic_circle", input=LayerInput( dataset=dataset, diff --git a/qgreenland/models/config/layer.py b/qgreenland/models/config/layer.py index 9218a87f..a0d5c485 100644 --- a/qgreenland/models/config/layer.py +++ b/qgreenland/models/config/layer.py @@ -36,11 +36,17 @@ class Layer(QgrBaseModel): description: str = Field(..., min_length=1) """Descriptive text shown as hover-text in the QGIS Layer Panel.""" + # Temporary keeping these around for testing before config is fully migrated + in_package: bool = True tags: list[str] = [] - """Additional categories that describe this data.""" - in_package: bool = True - """Is this layer in the final QGreenland zip file?""" + packaging_tags: list[str] = ["core"] + """Which packages does this layer belong in? Layer will be omitted if empty. + + Temporarily defaults to ["core"] to help us test more quickly. + + TODO: Validate len > 0. + """ show: bool = False """Is this layer initially "checked" or visible in QGIS?""" diff --git a/qgreenland/test/conftest.py b/qgreenland/test/conftest.py index a67c3003..7c8e273a 100644 --- a/qgreenland/test/conftest.py +++ b/qgreenland/test/conftest.py @@ -27,8 +27,7 @@ "id": "example_online", "title": "Example online", "description": "Example layer description.", - "tags": ["foo", "bar", "baz"], - "in_package": True, + "packaging_tags": ["foo", "bar"], "input": { "dataset": { "id": "baz", @@ -48,8 +47,7 @@ "id": "example_raster", "title": "Example raster", "description": "Example layer description.", - "tags": ["foo", "bar", "baz"], - "in_package": True, + "packaging_tags": ["bar", "baz"], "input": { "dataset": { "id": "example_dataset", diff --git a/qgreenland/test/constants.py b/qgreenland/test/constants.py index face8b93..f3d1dbf8 100644 --- a/qgreenland/test/constants.py +++ b/qgreenland/test/constants.py @@ -9,5 +9,6 @@ TEST_CONFIG_W_MISSING_SETTINGS_DIR = TEST_DATA_DIR / "config_with_missing_settings" TEST_CONFIG_W_STRING_ORDER_VALUES = TEST_DATA_DIR / "config_with_string_order_values" +# TODO: Handle new package-specific compile dirs MOCK_COMPILE_PACKAGE_DIR = TEST_DATA_DIR / "compile" MOCK_RELEASE_LAYERS_DIR = TEST_DATA_DIR / "release" / "layers" diff --git a/qgreenland/test/util/config/test_config_export.py b/qgreenland/test/util/config/test_config_export.py index a9b5f6f4..71de1262 100644 --- a/qgreenland/test/util/config/test_config_export.py +++ b/qgreenland/test/util/config/test_config_export.py @@ -1,96 +1,14 @@ import csv -import json import tempfile from pathlib import Path from unittest.mock import patch -from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR, MOCK_RELEASE_LAYERS_DIR -from qgreenland.util.config.export import export_config_csv, export_config_manifest +from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR +from qgreenland.util.config.export import export_config_csv @patch( - "qgreenland.util.layer.RELEASE_LAYERS_DIR", - new=MOCK_RELEASE_LAYERS_DIR, -) -def test_export_config_manifest(full_cfg): - common = { - "description": "Example layer description.", - # TODO: Generate this with imported function? This should be tested - # by itself elsewhere, so there's no need to test the expected output - # here too. - "layer_details": """Example layer description. - -=== Original Data Source === -Example Dataset - -Example abstract. - -Citation: -NSIDC 2020 - -Citation URL: -https://nsidc.org""", - "tags": ["foo", "bar", "baz"], - "hierarchy": ["Group", "Subgroup"], - } - with tempfile.NamedTemporaryFile("r") as tf: - export_config_manifest( - full_cfg, - output_path=Path(tf.name), - ) - - actual = json.load(tf) - - assert type(actual["qgr_version"]) is str - assert len(actual["qgr_version"]) >= 6 - del actual["qgr_version"] - - # For now, do not include online layers in the layer manifest. The - # `QGreenland Custom` QGIS Plugin does not currently support online - # layers. Once online layers are supported in the plugin, this commented out - # `online_asset` can be re-added: - # online_asset = { - # 'type': 'online', - # **full_cfg.layers['example_online'].input.asset.dict( - # include={'provider', 'url'}, - # ), - # } - expected = { - "version": "v0.1.0", - "layers": [ - # { - # 'id': 'example_online', - # 'title': 'Example online', - # 'assets': [online_asset], - # **common, - # }, - { - "id": "example_raster", - "title": "Example raster", - "assets": [ - { - "checksum": "a9a103f208179726038fa7178747a0a1", - "file": "example.tif", - "size_bytes": 287, - "type": "data", - }, - { - "checksum": "22b427acc6e4ebf57052115fdd5ac450", - "file": "example.tif.aux.xml", - "size_bytes": 332, - "type": "ancillary", - }, - ], - **common, - }, - ], - } - - assert actual == expected - - -@patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + "qgreenland.util.misc.COMPILE_PACKAGE_DIR", new=MOCK_COMPILE_PACKAGE_DIR, ) def test_export_config_csv(full_cfg): @@ -109,6 +27,7 @@ def test_export_config_csv(full_cfg): export_config_csv( full_cfg, output_path=Path(tf.name), + package_name="bar", ) actual = list(csv.DictReader(tf)) diff --git a/qgreenland/test/util/test_misc.py b/qgreenland/test/util/test_misc.py index ab6e6d78..b69c74aa 100644 --- a/qgreenland/test/util/test_misc.py +++ b/qgreenland/test/util/test_misc.py @@ -3,8 +3,8 @@ def test_layer_compile_dir(raster_layer_node): - expected = COMPILE_PACKAGE_DIR / "Group" / "Subgroup" / "Example raster" - actual = layer_util.get_layer_compile_dir(raster_layer_node) + expected = COMPILE_PACKAGE_DIR / "foo" / "Group" / "Subgroup" / "Example raster" + actual = layer_util.get_layer_compile_dir(raster_layer_node, package_name="foo") assert expected == actual diff --git a/qgreenland/test/util/test_qgis.py b/qgreenland/test/util/test_qgis.py index 6b573e71..6a190298 100644 --- a/qgreenland/test/util/test_qgis.py +++ b/qgreenland/test/util/test_qgis.py @@ -9,22 +9,24 @@ import qgreenland.util.qgis.project as prj from qgreenland.test.constants import MOCK_COMPILE_PACKAGE_DIR +mock_package_name = "bar" + @pytest.mark.usefixtures("setup_teardown_qgis_app") class TestQgisApp: def test_make_map_layer_online(self, online_layer_node): - result = qgl.make_map_layer(online_layer_node) + result = qgl.make_map_layer(online_layer_node, package_name=mock_package_name) assert "https://demo.mapserver.org" in result.source() assert result.dataProvider().name() == "wms" assert result.name() == online_layer_node.layer_cfg.title @patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + "qgreenland.util.misc.COMPILE_PACKAGE_DIR", new=MOCK_COMPILE_PACKAGE_DIR, ) def test_make_map_layer_raster(self, raster_layer_node): - result = qgl.make_map_layer(raster_layer_node) + result = qgl.make_map_layer(raster_layer_node, package_name=mock_package_name) # The result is a a raster layer assert isinstance(result, qgc.QgsRasterLayer) @@ -48,11 +50,14 @@ def test_make_map_layer_raster(self, raster_layer_node): assert result.name() == raster_layer_node.layer_cfg.title @patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + "qgreenland.util.misc.COMPILE_PACKAGE_DIR", new=MOCK_COMPILE_PACKAGE_DIR, ) def test_add_layer_metadata(self, raster_layer_node): - mock_raster_layer = qgl.make_map_layer(raster_layer_node) + mock_raster_layer = qgl.make_map_layer( + raster_layer_node, + package_name=mock_package_name, + ) qgl.add_layer_metadata(mock_raster_layer, raster_layer_node.layer_cfg) @@ -74,13 +79,17 @@ def test_add_layer_metadata(self, raster_layer_node): assert expected_extent == meta_extent.bounds.toRectangle() @patch( - "qgreenland.util.layer.COMPILE_PACKAGE_DIR", + "qgreenland.util.misc.COMPILE_PACKAGE_DIR", new=MOCK_COMPILE_PACKAGE_DIR, ) def test__add_layers_and_groups(self, raster_layer_node): # Test that _add_layers_and_groups works without error project = qgc.QgsProject.instance() - prj._add_layers_and_groups(project, raster_layer_node.root) + prj._add_layers_and_groups( + project, + raster_layer_node.root, + package_name=mock_package_name, + ) added_layers = list(project.mapLayers().values()) assert len(added_layers) == 1 assert added_layers[0].name() == "Example raster" @@ -91,5 +100,9 @@ def test__add_layers_and_groups(self, raster_layer_node): # Test that an exception is raised when parent groups of a layer are not # created first with pytest.raises(exc.QgrQgsLayerTreeGroupError): - prj._add_layers_and_groups(project, raster_layer_node) + prj._add_layers_and_groups( + project, + raster_layer_node, + package_name=mock_package_name, + ) project.clear() diff --git a/qgreenland/util/config/export.py b/qgreenland/util/config/export.py index 6912fcf1..67015ccd 100644 --- a/qgreenland/util/config/export.py +++ b/qgreenland/util/config/export.py @@ -22,61 +22,27 @@ get_layer_release_filepath, vector_or_raster, ) -from qgreenland.util.metadata import build_layer_metadata -from qgreenland.util.tree import LayerNode -from qgreenland.util.version import get_build_version - -DEFAULT_LAYER_MANIFEST_PATH = Path("./layers.csv") - - -def export_config_manifest( - cfg: Config, - output_path: Path = DEFAULT_LAYER_MANIFEST_PATH, -) -> None: - """Write a machine-readable manifest to disk describing available layers. - - This includes layers for which `in_package is False`. - - This must be run after the layers are in their release location, because we - need to calculate their size on disk. - """ - manifest_spec_version = "v0.1.0" - manifest = { - "version": manifest_spec_version, - "qgr_version": get_build_version(), - "layers": [ - { - # ID first for readability - "id": layer_node.layer_cfg.id, - **layer_node.layer_cfg.dict(include={"title", "description", "tags"}), - "hierarchy": layer_node.group_name_path, - "layer_details": build_layer_metadata(layer_node.layer_cfg), - "assets": _layer_manifest_final_assets(layer_node), - } - for layer_node in cfg.layer_tree.leaves - # For now, do not include online layers in the layer manifest. The - # `QGreenland Custom` QGIS Plugin does not currently support online - # layers. Once online layers are supported in the plugin, this `if` - # statement can be removed. - if not isinstance(layer_node.layer_cfg.input.asset, OnlineAsset) - ], - } - - with open(output_path, "w") as ofile: - json.dump(manifest, ofile) +from qgreenland.util.tree import LayerNode, prune_layers_not_in_package def export_config_csv( cfg: Config, - output_path: Path = DEFAULT_LAYER_MANIFEST_PATH, + *, + output_path: Path, + package_name: str, ) -> None: """Write a report to disk summarizing layers in the zip package. This must be run after the layers are in their location, because we need to calculate their size on disk. """ + layer_tree = prune_layers_not_in_package( + cfg.layer_tree, + package_name=package_name, + ) + report = [] - for layer_node in cfg.layer_tree.leaves: + for layer_node in layer_tree.leaves: layer_cfg = layer_node.layer_cfg if not layer_cfg.in_package: @@ -92,7 +58,10 @@ def export_config_csv( layer_size_bytes = 0 internet_required = False else: - layer_fp = get_layer_compile_filepath(layer_node) + layer_fp = get_layer_compile_filepath( + layer_node, + package_name=package_name, + ) layer_dir = layer_fp.parent layer_size_bytes = directory_size_bytes(layer_dir) internet_required = True diff --git a/qgreenland/util/layer.py b/qgreenland/util/layer.py index d7677b71..8a086d10 100644 --- a/qgreenland/util/layer.py +++ b/qgreenland/util/layer.py @@ -3,29 +3,34 @@ import qgreenland.exceptions as exc from qgreenland._typing import VectorOrRaster from qgreenland.constants.misc import PROVIDER_VECTOR_OR_RASTER_MAPPING -from qgreenland.constants.paths import COMPILE_PACKAGE_DIR, RELEASE_LAYERS_DIR +from qgreenland.constants.paths import RELEASE_LAYERS_DIR from qgreenland.models.config.asset import OnlineAsset from qgreenland.models.config.layer import Layer from qgreenland.util.fs import get_layer_fp +from qgreenland.util.misc import compile_package_dir from qgreenland.util.tree import LayerNode -def vector_or_raster(layer_node: LayerNode) -> VectorOrRaster: +def vector_or_raster( + layer_node: LayerNode, +) -> VectorOrRaster: layer_cfg = layer_node.layer_cfg if type(layer_cfg.input.asset) is OnlineAsset: return PROVIDER_VECTOR_OR_RASTER_MAPPING[layer_cfg.input.asset.provider] else: - layer_path = get_layer_compile_filepath(layer_node) + layer_path = get_layer_release_filepath(layer_node) return _vector_or_raster_from_fp(layer_path) def get_layer_compile_dir( layer_node: LayerNode, + *, + package_name: str, ) -> Path: """Get the layer directory in package compilation location.""" layer_group_path_str = "/".join(layer_node.group_name_path) return ( - COMPILE_PACKAGE_DIR + compile_package_dir(package_name) / layer_group_path_str / _layer_dirname_from_cfg(layer_node.layer_cfg) ) @@ -41,10 +46,13 @@ def datasource_dirname(*, dataset_id: str, asset_id: str) -> str: return f"{dataset_id}.{asset_id}" -def get_layer_compile_filepath( - layer_node: LayerNode, -) -> Path: - return get_layer_fp(get_layer_compile_dir(layer_node)) +def get_layer_compile_filepath(layer_node: LayerNode, *, package_name: str) -> Path: + return get_layer_fp( + get_layer_compile_dir( + layer_node, + package_name=package_name, + ) + ) def get_layer_release_filepath( diff --git a/qgreenland/util/luigi/__init__.py b/qgreenland/util/luigi/__init__.py index 1bad7582..7df0b4e8 100644 --- a/qgreenland/util/luigi/__init__.py +++ b/qgreenland/util/luigi/__init__.py @@ -69,6 +69,7 @@ def fetch_tasks_from_dataset( @cache def generate_layer_pipelines( *, + package_name: str, fetch_only: bool = False, ) -> list[luigi.Task]: """Generate a list of pre-configured tasks based on layer configuration. @@ -87,6 +88,10 @@ def generate_layer_pipelines( if isinstance(layer_cfg.input.asset, OnlineAsset): continue + # Check if the layer is meant to be in `package_name`. + if package_name not in layer_cfg.packaging_tags: + continue + # Create tasks, making each task dependent on the previous task. task = fetch_task_from_layer(layer_cfg) if fetch_only: diff --git a/qgreenland/util/luigi/tasks/ancillary.py b/qgreenland/util/luigi/tasks/ancillary.py index 0f6f7723..26f6c2b2 100644 --- a/qgreenland/util/luigi/tasks/ancillary.py +++ b/qgreenland/util/luigi/tasks/ancillary.py @@ -6,8 +6,8 @@ import luigi import markdown -from qgreenland.constants.paths import COMPILE_PACKAGE_DIR from qgreenland.util.command import run_cmd +from qgreenland.util.misc import compile_package_dir logger = logging.getLogger("luigi-interface") @@ -15,14 +15,15 @@ class AncillaryFile(luigi.Task): """Copy an ancillary file in to the final QGreenland package.""" + package_name = luigi.Parameter() # Absolute path src_filepath = luigi.Parameter() - # Relative to the root of QGreenland + # Relative to the root of the package dest_relative_filepath = luigi.Parameter() def output(self): return luigi.LocalTarget( - COMPILE_PACKAGE_DIR / self.dest_relative_filepath, + compile_package_dir(self.package_name) / self.dest_relative_filepath, ) def run(self): diff --git a/qgreenland/util/luigi/tasks/main.py b/qgreenland/util/luigi/tasks/main.py index cc747f9d..b6f30a8b 100644 --- a/qgreenland/util/luigi/tasks/main.py +++ b/qgreenland/util/luigi/tasks/main.py @@ -111,9 +111,14 @@ def run(self): class LinkLayer(QgrLayerTask): + package_name = luigi.Parameter() + def output(self): return luigi.LocalTarget( - get_layer_compile_dir(self.node), + get_layer_compile_dir( + self.node, + package_name=self.package_name, + ), ) def run(self): @@ -157,8 +162,8 @@ def run(self): with temporary_path_dir(self.output()) as temp_path: shutil.copy2(input_fp, temp_path / final_fn) - # Create layer provenance and metadata files. These are not - # "AncillaryFile" jobs because we need one file per layer. + # Create layer provenance and metadata files. These are not "AncillaryFile" + # jobs because we need one file per layer instead of one file per package. write_provenance_file( layer_cfg=self.layer_cfg, filepath=temp_path / "provenance.txt", diff --git a/qgreenland/util/luigi/tasks/pipeline.py b/qgreenland/util/luigi/tasks/pipeline.py index 1037fc8e..780f2068 100644 --- a/qgreenland/util/luigi/tasks/pipeline.py +++ b/qgreenland/util/luigi/tasks/pipeline.py @@ -6,16 +6,15 @@ from qgreenland.constants.paths import ( ANCILLARY_DIR, - COMPILE_PACKAGE_DIR, PROJECT_DIR, - RELEASE_LAYERS_DIR, VERSIONED_PACKAGE_DIR, WIP_PACKAGE_DIR, ) from qgreenland.constants.project import ENVIRONMENT, PROJECT +from qgreenland.models.config import Config from qgreenland.util.cleanup import cleanup_intermediate_dirs from qgreenland.util.config.config import get_config -from qgreenland.util.config.export import export_config_csv, export_config_manifest +from qgreenland.util.config.export import export_config_csv from qgreenland.util.luigi import generate_layer_pipelines from qgreenland.util.luigi.tasks.ancillary import ( AncillaryFile, @@ -23,27 +22,28 @@ AncillarySphinxPdfFile, ) from qgreenland.util.luigi.tasks.main import LinkLayer +from qgreenland.util.misc import compile_package_dir from qgreenland.util.qgis.project import QgsApplicationContext, make_qgis_project_file from qgreenland.util.version import get_build_version logger = logging.getLogger("luigi-interface") +# TODO: Can a wrapper task have Parameters? class LayersInPackage(luigi.WrapperTask): - """Hard link layers in to the packaging location. + """Hard link layers in to the packaging location.""" - Only layers with the `in_package` attribute set to `True` will be hard linked. - """ + package_name = luigi.Parameter() def requires(self): - tasks = generate_layer_pipelines() + tasks = generate_layer_pipelines(package_name=self.package_name) for task in tasks: - if task.layer_cfg.in_package: - yield LinkLayer( - requires_task=task, - layer_id=task.layer_cfg.id, - ) + yield LinkLayer( + requires_task=task, + layer_id=task.layer_cfg.id, + package_name=self.package_name, + ) class PackageLayerList(AncillaryFile): @@ -56,76 +56,53 @@ class PackageLayerList(AncillaryFile): dest_relative_filepath = "layer_list.csv" def requires(self): - yield LayersInPackage() + yield LayersInPackage(package_name=self.package_name) def run(self): config = get_config() with self.output().temporary_path() as temp_path: - export_config_csv(config, output_path=temp_path) - - -class LayerPipelines(luigi.WrapperTask): - """Build all the layers.""" - - fetch_only = luigi.BoolParameter(default=False) - - def requires(self): - """All layers that will be added to the project.""" - tasks = generate_layer_pipelines( - fetch_only=self.fetch_only, - ) - - yield from tasks - - -class LayerManifest(luigi.Task): - """A JSON manifest of layers available for access. - - Intended to be processed by machine, e.g. QGIS plugin. - """ - - def output(self): - return luigi.LocalTarget( - RELEASE_LAYERS_DIR / "manifest.json", - ) - - def requires(self): - yield LayerPipelines() - - def run(self): - config = get_config() - with self.output().temporary_path() as temp_path: - export_config_manifest(config, output_path=temp_path) + export_config_csv( + config, + output_path=temp_path, + package_name=self.package_name, + ) class CreateQgisProjectFile(luigi.Task): """Create .qgz/.qgs project file.""" + package_name = luigi.Parameter() + def requires(self): - yield LayersInPackage() + yield LayersInPackage(package_name=self.package_name) yield AncillaryFile( + package_name=self.package_name, src_filepath=ANCILLARY_DIR / "images" / "qgreenland.png", dest_relative_filepath="qgreenland.png", ) # TODO: Nothing below this line is really _required_ for the project # file. Only required for the Zip file. Extract. yield AncillaryMarkdownFileToHtml( + package_name=self.package_name, src_filepath=PROJECT_DIR / "README.md", dest_relative_filepath="README.html", ) yield AncillaryFile( + package_name=self.package_name, src_filepath=PROJECT_DIR / "doc" / "_pdf" / "QuickStartGuide.pdf", dest_relative_filepath="QuickStartGuide.pdf", ) yield AncillaryMarkdownFileToHtml( + package_name=self.package_name, src_filepath=PROJECT_DIR / "CHANGELOG.md", dest_relative_filepath="CHANGELOG.html", ) yield AncillarySphinxPdfFile( + package_name=self.package_name, src_filepath=PROJECT_DIR / "doc" / "Makefile", dest_relative_filepath="UserGuide.pdf", ) - yield PackageLayerList() + yield PackageLayerList(package_name=self.package_name) def output(self): versioned_package_name = f"{PROJECT}_{get_build_version()}" @@ -137,13 +114,16 @@ def run(self): # writing shapefiles, except this time we want to put them inside a # pre-existing directory. with QgsApplicationContext(): - make_qgis_project_file(COMPILE_PACKAGE_DIR / "qgreenland.qgz") + make_qgis_project_file( + compile_package_dir(self.package_name) / "qgreenland.qgz", + package_name=self.package_name, + ) # Create symbolic link to zip with the final versioned filename # We don't _need_ a symbolic link here, but this also serves to trigger # the next job. Path(self.output().path).symlink_to( - COMPILE_PACKAGE_DIR, + compile_package_dir(self.package_name), target_is_directory=True, ) @@ -151,8 +131,10 @@ def run(self): class ZipQGreenland(luigi.Task): """Zip entire QGreenland package for distribution.""" + package_name = luigi.Parameter() + def requires(self): - return CreateQgisProjectFile() + return CreateQgisProjectFile(package_name=self.package_name) def output(self): VERSIONED_PACKAGE_DIR.mkdir(parents=True, exist_ok=True) @@ -196,19 +178,23 @@ def run(self): ) -class HostedLayers(luigi.WrapperTask): - def requires(self): - yield LayerPipelines() - yield LayerManifest() +def _unique_packages_in_config(config: Config) -> set[str]: + packages = set() + for layer in config.layers.values(): + packages.update(layer.packaging_tags) + return packages -class QGreenlandAll(luigi.WrapperTask): + +class QGreenlandPackagesNoZip(luigi.WrapperTask): def requires(self): - yield ZipQGreenland() - yield HostedLayers() + config = get_config() + for package_name in _unique_packages_in_config(config): + yield CreateQgisProjectFile(package_name=package_name) -class QGreenlandNoZip(luigi.WrapperTask): +class QGreenlandPackages(luigi.WrapperTask): def requires(self): - yield CreateQgisProjectFile() - yield HostedLayers() + config = get_config() + for package_name in _unique_packages_in_config(config): + yield ZipQGreenland(package_name=package_name) diff --git a/qgreenland/util/misc.py b/qgreenland/util/misc.py index ca3410b2..259c31a6 100644 --- a/qgreenland/util/misc.py +++ b/qgreenland/util/misc.py @@ -1,6 +1,13 @@ from collections import Counter from collections.abc import Iterator +from pathlib import Path + +from qgreenland.constants.paths import COMPILE_PACKAGE_DIR def find_duplicates(items: Iterator[str]) -> list[str]: return [i for i, count in Counter(items).items() if count > 1] + + +def compile_package_dir(package_name: str) -> Path: + return COMPILE_PACKAGE_DIR / package_name diff --git a/qgreenland/util/qgis/layer.py b/qgreenland/util/qgis/layer.py index be8b5fba..e1e88e4c 100644 --- a/qgreenland/util/qgis/layer.py +++ b/qgreenland/util/qgis/layer.py @@ -76,9 +76,14 @@ def add_layer_metadata(map_layer: qgc.QgsMapLayer, layer_cfg: Layer) -> None: map_layer.loadNamedMetadata(temp_file.name) -def make_map_layer(layer_node: LayerNode) -> qgc.QgsMapLayer: +def make_map_layer( + layer_node: LayerNode, + *, + package_name: str, +) -> qgc.QgsMapLayer: layer_path = _layer_path( layer_node=layer_node, + package_name=package_name, ) layer_type = vector_or_raster(layer_node) if layer_type == "Vector": @@ -102,6 +107,7 @@ def make_map_layer(layer_node: LayerNode) -> qgc.QgsMapLayer: map_layer = _create_layer_with_side_effects( creator, layer_node=layer_node, + package_name=package_name, ) if not map_layer.isValid(): @@ -119,6 +125,8 @@ def make_map_layer(layer_node: LayerNode) -> qgc.QgsMapLayer: def _layer_path( layer_node: LayerNode, + *, + package_name: str, ) -> Union[Path, str]: layer_cfg = layer_node.layer_cfg if type(layer_cfg.input.asset) is OnlineAsset: @@ -128,16 +136,17 @@ def _layer_path( # automatically generates the correct relative paths. Using a relative # path causes statistics (nodata value, min/max) to not be generated, # resulting in rendering a gray rectangle. - return get_layer_compile_filepath(layer_node) + return get_layer_compile_filepath(layer_node, package_name=package_name) def _offline_raster_side_effects( creator: Callable[..., qgc.QgsRasterLayer], *, layer_node: LayerNode, + package_name: str, ) -> qgc.QgsRasterLayer: """Generate raster statistics on disk and in the layer object.""" - layer_path = _layer_path(layer_node) + layer_path = _layer_path(layer_node, package_name=package_name) # Create .aux.xml metadatafile with raster band statistics; useful # for styling and accurate min/max/stdev/mean in QGIS layer info @@ -159,6 +168,7 @@ def _create_layer_with_side_effects( creator: Callable[..., qgc.QgsMapLayer], *, layer_node: LayerNode, + package_name: str, ) -> qgc.QgsMapLayer: """Apply special steps before/after creating a layer.""" layer_cfg = layer_node.layer_cfg @@ -169,7 +179,11 @@ def _create_layer_with_side_effects( ) if offline_raster: - return _offline_raster_side_effects(creator, layer_node=layer_node) + return _offline_raster_side_effects( + creator, + layer_node=layer_node, + package_name=package_name, + ) else: return creator() diff --git a/qgreenland/util/qgis/project.py b/qgreenland/util/qgis/project.py index eb12e391..33e086de 100644 --- a/qgreenland/util/qgis/project.py +++ b/qgreenland/util/qgis/project.py @@ -38,7 +38,7 @@ def __exit__(self, _exc_type, _exc_value, _traceback): self.qgs.exitQgis() -def make_qgis_project_file(path: Path) -> None: +def make_qgis_project_file(path: Path, *, package_name: str) -> None: """Create a QGIS project file with the correct stuff in it. path: the desired path to .qgs/.qgz project file, e.g.: @@ -77,8 +77,15 @@ def make_qgis_project_file(path: Path) -> None: _add_decorations(project) - package_layer_tree = prune_layers_not_in_package(config.layer_tree) - _add_layers_and_groups(project, package_layer_tree) + package_layer_tree = prune_layers_not_in_package( + config.layer_tree, + package_name=package_name, + ) + _add_layers_and_groups( + project, + package_layer_tree, + package_name=package_name, + ) _add_project_metadata(project) @@ -117,7 +124,12 @@ def _apply_group_settings( group.setExpanded(settings.expand) -def _add_layers_and_groups(project: qgc.QgsProject, layer_tree: LayerGroupNode) -> None: +def _add_layers_and_groups( + project: qgc.QgsProject, + layer_tree: LayerGroupNode, + *, + package_name: str, +) -> None: """Iterate through the layer tree and create the relevant Qgs objects.""" # `anytree.PreOrderIter` is necessary here so that the # `_create_and_add_layer` function receives the correct group. @@ -141,6 +153,7 @@ def _add_layers_and_groups(project: qgc.QgsProject, layer_tree: LayerGroupNode) node=node, project=project, group=parent_group, + package_name=package_name, ) else: raise TypeError(f"Unexpected `node` type: {type(node)}") @@ -196,12 +209,13 @@ def _create_and_add_layer( node: LayerNode, project: qgc.QgsProject, group: qgc.QgsLayerTreeGroup, + package_name: str, ) -> None: layer_id = node.name logger.debug(f"Adding {layer_id}...") layer_cfg = node.layer_cfg - map_layer = make_map_layer(node) + map_layer = make_map_layer(node, package_name=package_name) # Assign to a different name because this changes the type. We need to add # `map_layer` to the project as the last step. diff --git a/qgreenland/util/tree.py b/qgreenland/util/tree.py index c8346c13..e5457299 100644 --- a/qgreenland/util/tree.py +++ b/qgreenland/util/tree.py @@ -489,8 +489,9 @@ def _delete_node( def prune_layers_not_in_package( tree: LayerGroupNode, + package_name: str, ) -> LayerGroupNode: - """Remove any leaf nodes that are `not in_package`. + """Remove any leaf nodes that are not to be packaged for `package_name`. If there are any empty groups after this, remove them too. """ @@ -499,7 +500,10 @@ def prune_layers_not_in_package( # We can iterate using tree.leaves here because we know all LayerNodes are # leaves. Removing these leaves shouldn't create new LayerNode leaves. for node in lt.leaves: - if type(node) is LayerNode and not node.layer_cfg.in_package: + if ( + type(node) is LayerNode + and package_name not in node.layer_cfg.packaging_tags + ): _delete_node(node, msg="Removing layer not in package") lt = prune_empty_groups(lt) diff --git a/scripts/export_config_csv.py b/scripts/export_config_csv.py index f4673462..93b59fd8 100644 --- a/scripts/export_config_csv.py +++ b/scripts/export_config_csv.py @@ -3,6 +3,7 @@ # Hack to import from qgreenland import os import sys +from pathlib import Path THIS_DIR = os.path.dirname(os.path.abspath(__file__)) PARENT_DIR = os.path.dirname(THIS_DIR) @@ -13,4 +14,9 @@ from qgreenland.util.config.export import export_config_csv init_config() - export_config_csv(get_config()) + export_config_csv( + get_config(), + output_path=Path("./layers.csv"), + # TODO: Come back to me + package_name="core", + )