Skip to content

Commit

Permalink
Also include raw raster layer files in zipped raster layer uploads (#52)
Browse files Browse the repository at this point in the history 
* Refactor to allow multiple files to be included in zip per exported
layer

* Also include raw raster layer files in zipped raster layer uploads
  • Loading branch information
@nyalldawson
nyalldawson authored Nov 21, 2023
1 parent 7b853ab commit d5220c0
Show file tree
Hide file tree
Showing 3 changed files with 145 additions and 49 deletions.
2 changes: 1 addition & 1 deletion felt/core/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from .map_uploader import MapUploaderTask # noqa
from .layer_exporter import ( # noqa
LayerExporter,
ExportResult
ZippedExportResult
)
from .multi_step_feedback import MultiStepFeedback # noqa
from .meta import PLUGIN_METADATA_PARSER # noqa
Expand Down
164 changes: 118 additions & 46 deletions felt/core/layer_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@
import tempfile
import uuid
import zipfile
import math
from dataclasses import dataclass
from pathlib import Path
from typing import Optional
from typing import (
Optional,
List,
Tuple
)

from qgis.PyQt.QtCore import (
QVariant,
Expand Down Expand Up @@ -49,7 +54,10 @@
QgsSimpleLineSymbolLayer,
QgsSimpleMarkerSymbolLayer,
QgsEllipseSymbolLayer,
QgsReadWriteContext
QgsReadWriteContext,
QgsRasterPipe,
QgsRasterNuller,
QgsRasterRange
)

from .enums import LayerExportResult
Expand All @@ -59,10 +67,23 @@


@dataclass
class ExportResult:
class LayerExportDetails:
"""
Export results
"""
representative_filename: str
filenames: List[str]
result: LayerExportResult
error_message: str
qgis_style_xml: str
style: Optional[LayerStyle] = None


@dataclass
class ZippedExportResult:
"""
A zipped export results
"""
filename: str
result: LayerExportResult
error_message: str
Expand Down Expand Up @@ -174,7 +195,7 @@ def export_layer_for_felt(
self,
layer: QgsMapLayer,
feedback: Optional[QgsFeedback] = None
) -> ExportResult:
) -> ZippedExportResult:
"""
Exports a layer into a format acceptable for Felt
:raises LayerPackagingException
Expand All @@ -189,15 +210,21 @@ def export_layer_for_felt(
# package into zip
zip_file_path = (
(Path(str(self.temp_dir.name)) /
(Path(res.filename).stem + '.zip')).as_posix())
(Path(res.representative_filename).stem + '.zip')).as_posix())
with zipfile.ZipFile(zip_file_path, "w", zipfile.ZIP_DEFLATED) as zipf:
zipf.write(res.filename, Path(res.filename).name)
for filename in res.filenames:
zipf.write(filename, Path(filename).name)

# add QGIS layer style xml also
zipf.writestr("qgis_style.xml", res.qgis_style_xml)

res.filename = zip_file_path
return res
return ZippedExportResult(
filename=zip_file_path,
result=res.result,
error_message=res.error_message,
qgis_style_xml=res.qgis_style_xml,
style=res.style
)

@staticmethod
def _get_original_style_xml(layer: QgsMapLayer) -> str:
Expand All @@ -212,7 +239,7 @@ def _get_original_style_xml(layer: QgsMapLayer) -> str:
def export_vector_layer(
self,
layer: QgsVectorLayer,
feedback: Optional[QgsFeedback] = None) -> ExportResult:
feedback: Optional[QgsFeedback] = None) -> LayerExportDetails:
"""
Exports a vector layer into a format acceptable for Felt
"""
Expand Down Expand Up @@ -305,25 +332,25 @@ def export_vector_layer(
res_layer.featureCount(), layer.featureCount())
)

return ExportResult(
filename=dest_file,
return LayerExportDetails(
representative_filename=dest_file,
filenames=[dest_file],
result=layer_export_result,
error_message=error_message,
qgis_style_xml=self._get_original_style_xml(layer),
style=self.representative_layer_style(layer)
)

def export_raster_layer(
self,
layer: QgsRasterLayer,
feedback: Optional[QgsFeedback] = None) -> ExportResult:
def run_raster_writer(self,
layer: QgsRasterLayer,
file_name: str,
use_style: bool,
feedback: Optional[QgsFeedback] = None) \
-> Tuple[LayerExportResult, Optional[str]]:
"""
Exports a raster layer into a format acceptable for Felt
Runs a raster write operation for the layer
"""

dest_file = self.generate_file_name('.tif')

writer = QgsRasterFileWriter(dest_file)
writer = QgsRasterFileWriter(file_name)
writer.setOutputFormat('GTiff')
writer.setOutputProviderKey('gdal')
writer.setTiledMode(False)
Expand All @@ -336,26 +363,29 @@ def export_raster_layer(
])

extent = layer.extent()
raster_pipe = layer.pipe()
projector = raster_pipe.projector()
if use_style:
raster_pipe = layer.pipe()
else:
raster_pipe = QgsRasterPipe()
raster_pipe.set(layer.dataProvider().clone())
nuller = QgsRasterNuller()
for band in range(1, layer.dataProvider().bandCount() + 1):
additional_no_data_values = (
layer.dataProvider().userNoDataValues(
band))
source_no_data = layer.dataProvider().sourceNoDataValue(band)
if not math.isnan(source_no_data):
additional_no_data_values.append(
QgsRasterRange(
layer.dataProvider().sourceNoDataValue(band),
layer.dataProvider().sourceNoDataValue(band)
)
)
if additional_no_data_values:
nuller.setNoData(band, additional_no_data_values)
raster_pipe.insert(1, nuller)

dest_crs = layer.crs()
# disable local reprojection for now - see #14
if False: # pylint: disable=using-constant-test
dest_crs = QgsCoordinateReferenceSystem('EPSG:3857')
projector.setCrs(
layer.crs(),
dest_crs,
self.transform_context
)

to_3857_transform = QgsCoordinateTransform(
layer.crs(),
dest_crs,
self.transform_context
)
extent = to_3857_transform.transformBoundingBox(extent)

width = layer.width()
if feedback:
block_feedback = QgsRasterBlockFeedback()
Expand Down Expand Up @@ -388,6 +418,32 @@ def export_raster_layer(
QgsRasterFileWriter.WriterError.WriteCanceled:
None,
}[res]

layer_export_result = {
QgsRasterFileWriter.WriterError.NoError:
LayerExportResult.Success,
QgsRasterFileWriter.WriterError.WriteCanceled:
LayerExportResult.Canceled,
}[res]

return layer_export_result, error_message

def export_raster_layer(
self,
layer: QgsRasterLayer,
feedback: Optional[QgsFeedback] = None) -> LayerExportDetails:
"""
Exports a raster layer into a format acceptable for Felt
"""
raw_dest_file = self.generate_file_name('.tif')
styled_dest_file = raw_dest_file.replace('.tif', '_styled.tif')

layer_export_result, error_message = self.run_raster_writer(
layer,
file_name=styled_dest_file,
use_style=True,
feedback=feedback)

if error_message:
Logger.instance().log_error_json(
{
Expand All @@ -397,15 +453,31 @@ def export_raster_layer(
)
raise LayerPackagingException(error_message)

layer_export_result = {
QgsRasterFileWriter.WriterError.NoError:
LayerExportResult.Success,
QgsRasterFileWriter.WriterError.WriteCanceled:
LayerExportResult.Canceled,
}[res]
filenames = [styled_dest_file]
if layer_export_result != LayerExportResult.Canceled:
# also write raw raster

layer_export_result, error_message = self.run_raster_writer(
layer,
file_name=raw_dest_file,
use_style=False,
feedback=feedback)

if error_message:
Logger.instance().log_error_json(
{
'type': Logger.PACKAGING_RASTER,
'error': 'Error packaging layer: {}'.format(
error_message)
}
)
raise LayerPackagingException(error_message)

filenames.append(raw_dest_file)

return ExportResult(
filename=dest_file,
return LayerExportDetails(
representative_filename=raw_dest_file,
filenames=filenames,
result=layer_export_result,
error_message=error_message,
qgis_style_xml=self._get_original_style_xml(layer)
Expand Down
28 changes: 26 additions & 2 deletions felt/test/test_layer_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -172,14 +172,17 @@ def test_raster_conversion(self):
self.assertEqual(result.filename[-4:], '.zip')
with zipfile.ZipFile(result.filename) as z:
tif_files = [f for f in z.namelist() if f.endswith('tif')]
self.assertEqual(len(tif_files), 1)
self.assertEqual(len(tif_files), 2)

styled_tif = [f for f in tif_files if '_styled' in f][0]

self.assertEqual(
result.qgis_style_xml[:58],
"<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>"
)

out_layer = QgsRasterLayer(
'/vsizip/{}/{}'.format(result.filename, tif_files[0]),
'/vsizip/{}/{}'.format(result.filename, styled_tif),
'test')
self.assertTrue(out_layer.isValid())
self.assertEqual(out_layer.width(), 373)
Expand All @@ -204,6 +207,27 @@ def test_raster_conversion(self):
self.assertAlmostEqual(out_layer.extent().yMaximum(),
45.8117014376, 3)

raw_tif = [f for f in tif_files if '_styled' not in f][0]
out_layer = QgsRasterLayer(
'/vsizip/{}/{}'.format(result.filename, raw_tif),
'test')
self.assertTrue(out_layer.isValid())
self.assertEqual(out_layer.width(), 373)
self.assertEqual(out_layer.height(), 350)
self.assertEqual(out_layer.bandCount(), 1)
self.assertEqual(out_layer.dataProvider().dataType(1),
Qgis.DataType.Float32)
self.assertEqual(out_layer.crs(),
QgsCoordinateReferenceSystem('EPSG:4326'))
self.assertAlmostEqual(out_layer.extent().xMinimum(),
18.6662979442, 3)
self.assertAlmostEqual(out_layer.extent().xMaximum(),
18.7035979442, 3)
self.assertAlmostEqual(out_layer.extent().yMinimum(),
45.7767014376, 3)
self.assertAlmostEqual(out_layer.extent().yMaximum(),
45.8117014376, 3)


if __name__ == "__main__":
suite = unittest.makeSuite(LayerExporterTest)
Expand Down

0 comments on commit d5220c0

Please sign in to comment.