Skip to content

Commit

Permalink
Rename existing fid columns to old_fid when export to gpkg fails
Browse files Browse the repository at this point in the history
  • Loading branch information
@nyalldawson
nyalldawson committed Jun 5, 2024
1 parent d8ec0b2 commit 7ef7e3b
Show file tree
Hide file tree
Showing 2 changed files with 116 additions and 12 deletions.
31 changes: 21 additions & 10 deletions felt/core/layer_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,9 @@ def export_vector_layer(
self,
layer: QgsVectorLayer,
conversion_context: ConversionContext,
feedback: Optional[QgsFeedback] = None) -> LayerExportDetails:
feedback: Optional[QgsFeedback] = None,
force_rewrite_fid: bool = False
) -> LayerExportDetails:
"""
Exports a vector layer into a format acceptable for Felt
"""
Expand All @@ -349,7 +351,9 @@ def export_vector_layer(
self.transform_context
)
writer_options.feedback = feedback
writer_options.forceMulti = True
writer_options.forceMulti = QgsWkbTypes.geometryType(
layer.wkbType()) in (QgsWkbTypes.LineGeometry,
QgsWkbTypes.PolygonGeometry)
writer_options.overrideGeometryType = QgsWkbTypes.dropM(
QgsWkbTypes.dropZ(layer.wkbType())
)
Expand All @@ -366,13 +370,13 @@ def export_vector_layer(
writer_options.attributes = fields.allAttributesList()
if fid_index >= 0:
fid_type = fields.field(fid_index).type()
if fid_type not in (QVariant.Int,
QVariant.UInt,
QVariant.LongLong,
QVariant.ULongLong):
writer_options.attributes = [a for a in
writer_options.attributes if
a != fid_index]
if force_rewrite_fid or fid_type not in (QVariant.Int,
QVariant.UInt,
QVariant.LongLong,
QVariant.ULongLong):
writer_options.attributesExportNames = [
f.name() if f.name().lower() != 'fid' else 'old_fid'
for f in fields]

# pylint: disable=unused-variable
res, error_message, new_filename, new_layer_name = \
Expand All @@ -384,7 +388,14 @@ def export_vector_layer(
)
# pylint: enable=unused-variable

if res not in (QgsVectorFileWriter.WriterError.NoError,
if (not force_rewrite_fid and
res == QgsVectorFileWriter.WriterError.ErrFeatureWriteFailed):
# could not write attributes -- possibly eg due to duplicate
# FIDs. Let's try with renaming FID
return self.export_vector_layer(
layer, conversion_context, feedback, True
)
elif res not in (QgsVectorFileWriter.WriterError.NoError,
QgsVectorFileWriter.WriterError.Canceled):
Logger.instance().log_error_json(
{
Expand Down
97 changes: 95 additions & 2 deletions felt/test/test_layer_exporter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,10 @@
QgsCoordinateReferenceSystem,
QgsWkbTypes,
QgsPalettedRasterRenderer,
QgsRasterContourRenderer
QgsRasterContourRenderer,
QgsFeature,
QgsGeometry,
QgsPointXY
)

from .utilities import get_qgis_app
Expand Down Expand Up @@ -189,7 +192,10 @@ def test_vector_conversion(self):
'test')
self.assertTrue(out_layer.isValid())
self.assertEqual(out_layer.featureCount(), layer.featureCount())
self.assertEqual(out_layer.wkbType(), QgsWkbTypes.MultiPoint)
self.assertEqual(out_layer.wkbType(), QgsWkbTypes.Point)
self.assertEqual([f.name() for f in out_layer.fields()],
['fid', 'Class', 'Heading', 'Importance',
'Pilots', 'Cabin Crew', 'Staff'])

def test_gml_conversion(self):
"""
Expand Down Expand Up @@ -217,6 +223,93 @@ def test_gml_conversion(self):
self.assertTrue(out_layer.isValid())
self.assertEqual(out_layer.featureCount(), layer.featureCount())
self.assertEqual(out_layer.wkbType(), QgsWkbTypes.MultiPolygon)
self.assertEqual([f.name() for f in out_layer.fields()],
['fid', 'old_fid', 'name', 'intval', 'floatval'])

def test_layer_conversion_string_fid(self):
"""
Test vector layer conversion where FID columns are not compatible
with geopackge
"""
layer = QgsVectorLayer('Point?crs=EPSG:4326&'
'field=fid:string(255,0)&'
'field=label:string(255,0)', 'test', 'memory')
self.assertTrue(layer.isValid())

f = QgsFeature(layer.fields())
f.setAttributes(['abc', 'def'])
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(1, 2)))
self.assertTrue(layer.dataProvider().addFeature(f))
f.setAttributes(['15', 'ghi'])
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(3, 4)))
self.assertTrue(layer.dataProvider().addFeature(f))

exporter = LayerExporter(
QgsCoordinateTransformContext()
)
conversion_context = ConversionContext()
result = exporter.export_layer_for_felt(layer, conversion_context)
self.assertEqual(result.result, LayerExportResult.Success)
self.assertTrue(result.filename)
self.assertEqual(result.filename[-4:], '.zip')
with zipfile.ZipFile(result.filename) as z:
gpkg_files = [f for f in z.namelist() if f.endswith('gpkg')]
self.assertEqual(len(gpkg_files), 1)

out_layer = QgsVectorLayer(
'/vsizip/{}/{}'.format(result.filename, gpkg_files[0]),
'test')
self.assertTrue(out_layer.isValid())
self.assertEqual(out_layer.featureCount(), layer.featureCount())
self.assertEqual(out_layer.wkbType(), QgsWkbTypes.Point)
self.assertEqual([f.name() for f in out_layer.fields()],
['fid', 'old_fid', 'label'])
features = list(out_layer.getFeatures())
self.assertEqual([feature.attributes() for feature in features],
[[1, 'abc', 'def'],
[2, '15', 'ghi']])

def test_layer_conversion_duplicate_fid(self):
"""
Test vector layer conversion where FID columns are not compatible
with geopackge
"""
layer = QgsVectorLayer('Point?crs=EPSG:4326&'
'field=fid:integer&'
'field=label:string(255,0)', 'test', 'memory')
self.assertTrue(layer.isValid())

f = QgsFeature(layer.fields())
f.setAttributes([15, 'abc'])
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(1, 2)))
self.assertTrue(layer.dataProvider().addFeature(f))
f.setAttributes([15, 'def'])
f.setGeometry(QgsGeometry.fromPointXY(QgsPointXY(3, 4)))
self.assertTrue(layer.dataProvider().addFeature(f))

exporter = LayerExporter(
QgsCoordinateTransformContext()
)
conversion_context = ConversionContext()
result = exporter.export_layer_for_felt(layer, conversion_context)
self.assertEqual(result.result, LayerExportResult.Success)
self.assertTrue(result.filename)
self.assertEqual(result.filename[-4:], '.zip')
with zipfile.ZipFile(result.filename) as z:
gpkg_files = [f for f in z.namelist() if f.endswith('gpkg')]
self.assertEqual(len(gpkg_files), 1)

out_layer = QgsVectorLayer(
'/vsizip/{}/{}'.format(result.filename, gpkg_files[0]),
'test')
self.assertTrue(out_layer.isValid())
self.assertEqual(out_layer.featureCount(), layer.featureCount())
self.assertEqual(out_layer.wkbType(), QgsWkbTypes.Point)
self.assertEqual([f.name() for f in out_layer.fields()],
['fid', 'old_fid', 'label'])
features = list(out_layer.getFeatures())
self.assertEqual([feature.attributes() for feature in features],
[[1, 15, 'abc'], [2, 15, 'def']])

def test_raster_conversion_raw(self):
"""
Expand Down

0 comments on commit 7ef7e3b

Please sign in to comment.