From 36dcbc248aa4dbbaba729159cb41ded0740fbe31 Mon Sep 17 00:00:00 2001 From: Samuele Santi Date: Thu, 19 Dec 2024 19:42:50 -0500 Subject: [PATCH] Add support for geography columns --- martin/src/pg/query_tables.rs | 6 +-- .../src/pg/scripts/query_available_tables.sql | 43 +++++++++++++++-- martin/tests/pg_server_test.rs | 2 + martin/tests/pg_table_source_test.rs | 18 ++++++++ tests/expected/auto/catalog_auto.json | 3 ++ tests/expected/auto/save_config.yaml | 13 ++++++ .../martin-cp/flat-with-hash_save_config.yaml | 13 ++++++ .../expected/martin-cp/flat_save_config.yaml | 13 ++++++ .../martin-cp/normalized_save_config.yaml | 13 ++++++ tests/fixtures/tables/table_source_geog.sql | 46 +++++++++++++++++++ tests/test.sh | 2 + 11 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/tables/table_source_geog.sql diff --git a/martin/src/pg/query_tables.rs b/martin/src/pg/query_tables.rs index 9aea4d575..6879f4800 100644 --- a/martin/src/pg/query_tables.rs +++ b/martin/src/pg/query_tables.rs @@ -192,7 +192,7 @@ SELECT FROM ( SELECT ST_AsMVTGeom( - ST_Transform(ST_CurveToLine({geometry_column}), 3857), + ST_Transform(ST_CurveToLine({geometry_column}::geometry), 3857), ST_TileEnvelope($1::integer, $2::integer, $3::integer), {extent}, {buffer}, {clip_geom} ) AS geom @@ -223,11 +223,11 @@ async fn calc_bounds( .await? .query_one(&format!( r#" -WITH real_bounds AS (SELECT ST_SetSRID(ST_Extent({geometry_column}), {srid}) AS rb FROM {schema}.{table}) +WITH real_bounds AS (SELECT ST_SetSRID(ST_Extent({geometry_column}::geometry), {srid}) AS rb FROM {schema}.{table}) SELECT ST_Transform( CASE WHEN (SELECT ST_GeometryType(rb) FROM real_bounds LIMIT 1) = 'ST_Point' - THEN ST_SetSRID(ST_Extent(ST_Expand({geometry_column}, 1)), {srid}) + THEN ST_SetSRID(ST_Extent(ST_Expand({geometry_column}::geometry, 1)), {srid}) ELSE (SELECT * FROM real_bounds) END, 4326 diff --git a/martin/src/pg/scripts/query_available_tables.sql b/martin/src/pg/scripts/query_available_tables.sql index 783a0d30e..cbef43c9b 100755 --- a/martin/src/pg/scripts/query_available_tables.sql +++ b/martin/src/pg/scripts/query_available_tables.sql @@ -28,7 +28,8 @@ WITH JOIN pg_opclass op ON op.oid = ix.indclass[0] AND op.opcname IN ('gist_geometry_ops_2d', 'spgist_geometry_ops_2d', - 'brin_geometry_inclusion_ops_2d') + 'brin_geometry_inclusion_ops_2d', + 'gist_geography_ops') GROUP BY 1, 2, 3), -- annotated_geometry_columns AS ( @@ -38,6 +39,7 @@ WITH f_geometry_column AS geom, srid, type, + -- 'geometry' AS column_type COALESCE(class.relkind = 'v', false) AS is_view, bool_or(sic.column_name is not null) as geom_idx FROM geometry_columns @@ -50,6 +52,33 @@ WITH geometry_columns.f_table_name = sic.table_name AND geometry_columns.f_geometry_column = sic.column_name GROUP BY 1, 2, 3, 4, 5, 6), + -- + annotated_geography_columns AS ( + -- list of geography columns with additional metadata + SELECT f_table_schema AS schema, + f_table_name AS name, + f_geography_column AS geom, + srid, + type, + -- 'geography' AS column_type + COALESCE(class.relkind = 'v', false) AS is_view, + bool_or(sic.column_name is not null) as geom_idx + FROM geography_columns + JOIN pg_catalog.pg_class AS class + ON class.relname = geography_columns.f_table_name + JOIN pg_catalog.pg_namespace AS ns + ON ns.nspname = geography_columns.f_table_schema + LEFT JOIN spatially_indexed_columns AS sic ON + geography_columns.f_table_schema = sic.table_schema AND + geography_columns.f_table_name = sic.table_name AND + geography_columns.f_geography_column = sic.column_name + GROUP BY 1, 2, 3, 4, 5, 6), + -- + annotated_geo_columns AS ( + SELECT * FROM annotated_geometry_columns + UNION SELECT * FROM annotated_geography_columns + ), + -- descriptions AS ( -- comments on table/views SELECT @@ -69,12 +98,16 @@ SELECT schema, is_view, geom_idx, COALESCE( - jsonb_object_agg(columns.column_name, columns.type_name) - FILTER (WHERE columns.column_name IS NOT NULL AND columns.type_name != 'geometry'), - '{}'::jsonb + jsonb_object_agg(columns.column_name, columns.type_name) + FILTER ( + WHERE columns.column_name IS NOT NULL + AND columns.type_name != 'geometry' + AND columns.type_name != 'geography' + ), + '{}'::jsonb ) as properties, dc.description -FROM annotated_geometry_columns AS gc +FROM annotated_geo_columns AS gc LEFT JOIN columns ON gc.schema = columns.table_schema AND gc.name = columns.table_name AND diff --git a/martin/tests/pg_server_test.rs b/martin/tests/pg_server_test.rs index 34ca2dc0d..d826f8ff2 100644 --- a/martin/tests/pg_server_test.rs +++ b/martin/tests/pg_server_test.rs @@ -121,6 +121,8 @@ postgres: description: public.points3857.geom table_source: content_type: application/x-protobuf + table_source_geog: + content_type: application/x-protobuf table_source_multiple_geom: content_type: application/x-protobuf description: public.table_source_multiple_geom.geom1 diff --git a/martin/tests/pg_table_source_test.rs b/martin/tests/pg_table_source_test.rs index 817dcc017..5db4b01c2 100644 --- a/martin/tests/pg_table_source_test.rs +++ b/martin/tests/pg_table_source_test.rs @@ -82,6 +82,8 @@ async fn table_source() { description: public.points3857.geom table_source: content_type: application/x-protobuf + table_source_geog: + content_type: application/x-protobuf table_source_multiple_geom: content_type: application/x-protobuf description: public.table_source_multiple_geom.geom1 @@ -106,6 +108,22 @@ async fn table_source() { properties: gid: int4 "); + + let source2 = table(&mock, "table_source_geog"); + assert_yaml_snapshot!(source2, @r" + schema: public + table: table_source_geog + srid: 4326 + geometry_column: geog + bounds: + - -2 + - 0 + - 142.84131509869133 + - 45 + geometry_type: Geometry + properties: + gid: int4 + "); } #[actix_rt::test] diff --git a/tests/expected/auto/catalog_auto.json b/tests/expected/auto/catalog_auto.json index eb755495d..45a230f73 100644 --- a/tests/expected/auto/catalog_auto.json +++ b/tests/expected/auto/catalog_auto.json @@ -156,6 +156,9 @@ "table_source": { "content_type": "application/x-protobuf" }, + "table_source_geog": { + "content_type": "application/x-protobuf" + }, "table_source_multiple_geom": { "content_type": "application/x-protobuf", "description": "public.table_source_multiple_geom.geom1" diff --git a/tests/expected/auto/save_config.yaml b/tests/expected/auto/save_config.yaml index a6fc8d64c..f578a9aab 100644 --- a/tests/expected/auto/save_config.yaml +++ b/tests/expected/auto/save_config.yaml @@ -146,6 +146,19 @@ postgres: geometry_type: GEOMETRY properties: gid: int4 + table_source_geog: + schema: public + table: table_source_geog + srid: 4326 + geometry_column: geog + bounds: + - -2.0 + - 0.0 + - 142.84131509869133 + - 45.0 + geometry_type: Geometry + properties: + gid: int4 table_source_multiple_geom: schema: public table: table_source_multiple_geom diff --git a/tests/expected/martin-cp/flat-with-hash_save_config.yaml b/tests/expected/martin-cp/flat-with-hash_save_config.yaml index e3b1109ea..ea1fb2bf0 100644 --- a/tests/expected/martin-cp/flat-with-hash_save_config.yaml +++ b/tests/expected/martin-cp/flat-with-hash_save_config.yaml @@ -145,6 +145,19 @@ postgres: geometry_type: GEOMETRY properties: gid: int4 + table_source_geog: + schema: public + table: table_source_geog + srid: 4326 + geometry_column: geog + bounds: + - -2.0 + - 0.0 + - 142.84131509869133 + - 45.0 + geometry_type: Geometry + properties: + gid: int4 table_source_multiple_geom: schema: public table: table_source_multiple_geom diff --git a/tests/expected/martin-cp/flat_save_config.yaml b/tests/expected/martin-cp/flat_save_config.yaml index e3b1109ea..ea1fb2bf0 100644 --- a/tests/expected/martin-cp/flat_save_config.yaml +++ b/tests/expected/martin-cp/flat_save_config.yaml @@ -145,6 +145,19 @@ postgres: geometry_type: GEOMETRY properties: gid: int4 + table_source_geog: + schema: public + table: table_source_geog + srid: 4326 + geometry_column: geog + bounds: + - -2.0 + - 0.0 + - 142.84131509869133 + - 45.0 + geometry_type: Geometry + properties: + gid: int4 table_source_multiple_geom: schema: public table: table_source_multiple_geom diff --git a/tests/expected/martin-cp/normalized_save_config.yaml b/tests/expected/martin-cp/normalized_save_config.yaml index e3b1109ea..ea1fb2bf0 100644 --- a/tests/expected/martin-cp/normalized_save_config.yaml +++ b/tests/expected/martin-cp/normalized_save_config.yaml @@ -145,6 +145,19 @@ postgres: geometry_type: GEOMETRY properties: gid: int4 + table_source_geog: + schema: public + table: table_source_geog + srid: 4326 + geometry_column: geog + bounds: + - -2.0 + - 0.0 + - 142.84131509869133 + - 45.0 + geometry_type: Geometry + properties: + gid: int4 table_source_multiple_geom: schema: public table: table_source_multiple_geom diff --git a/tests/fixtures/tables/table_source_geog.sql b/tests/fixtures/tables/table_source_geog.sql new file mode 100644 index 000000000..aacef52bb --- /dev/null +++ b/tests/fixtures/tables/table_source_geog.sql @@ -0,0 +1,46 @@ +DROP TABLE IF EXISTS table_source_geog; +CREATE TABLE table_source_geog(gid serial PRIMARY KEY, geog geography(GEOMETRY, 4326)); + +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(0 0)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(-2 2)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;LINESTRING(0 0, 1 1)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;LINESTRING(2 2, 3 3)'::geography); + +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(30 10)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;LINESTRING(30 10, 10 30, 40 40)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POLYGON((30 10, 40 40, 20 40, 10 20, 30 10))'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POLYGON((35 10, 45 45, 15 40, 10 20, 35 10),(20 30, 35 35, 30 20, 20 30))'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;MULTIPOINT((10 40), (40 30), (20 20), (30 10))'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;MULTIPOINT(10 40, 40 30, 20 20, 30 10)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;MULTILINESTRING((10 10, 20 20, 10 40),(40 40, 30 30, 40 20, 30 10))'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;MULTIPOLYGON(((30 20, 45 40, 10 40, 30 20)),((15 5, 40 10, 10 20, 5 10, 15 5)))'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;MULTIPOLYGON(((40 40, 20 45, 45 30, 40 40)),((20 35, 10 30, 10 10, 30 5, 45 20, 20 35),(30 20, 20 15, 20 25, 30 20)))'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;GEOMETRYCOLLECTION(POINT(4 6),LINESTRING(4 6,7 10))'::geography); + +-- Curves are not supported in geography columns +-- INSERT INTO table_source_geog(geog) values ('SRID=4326;CIRCULARSTRING(1 5, 6 2, 7 3)'::geography); +-- INSERT INTO table_source_geog(geog) values ('SRID=4326;COMPOUNDCURVE(CIRCULARSTRING(0 0,1 1,1 0),(1 0,0 1))'::geography); +-- INSERT INTO table_source_geog(geog) values ('SRID=4326;CURVEPOLYGON(CIRCULARSTRING(-2 0,-1 -1,0 0,1 -1,2 0,0 2,-2 0),(-1 0,0 0.5,1 0,0 1,-1 0))'::geography); +-- INSERT INTO table_source_geog(geog) values ('SRID=4326;MULTICURVE((5 5,3 5,3 3,0 3),CIRCULARSTRING(0 0,2 1,2 2))'::geography); + +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84124343269863 11.927545216212339)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84022627741408 11.926919775099435)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84116724279622 11.926986082398354)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84129834730146 11.926483025982757)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84086326293937 11.92741281580712)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84083973422645 11.927188724740008)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.8407405154705 11.92659842381238)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84029057105903 11.92711170365923)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.8403402985401 11.927568375227375)'::geography); +INSERT INTO table_source_geog(geog) values ('SRID=4326;POINT(142.84131509869133 11.92781306544329)'::geography); + +-- DO NOT CREATE INDEX ON GEOGRAPHY COLUMN -- this table is used in a test case + +DO $do$ BEGIN + EXECUTE 'COMMENT ON TABLE table_source_geog IS $tj$' || $$ + { + "description": null, + "foo": {"bar": "foo"} + } + $$::json || '$tj$'; +END $do$; diff --git a/tests/test.sh b/tests/test.sh index 2bf4812bd..ca94d434f 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -291,6 +291,7 @@ test_jsn fnc_comment function_Mixed_Name kill_process "$MARTIN_PROC_ID" Martin test_log_has_str "$LOG_FILE" 'WARN martin::pg::query_tables] Table public.table_source has no spatial index on column geom' +test_log_has_str "$LOG_FILE" 'WARN martin::pg::query_tables] Table public.table_source_geog has no spatial index on column geog' test_log_has_str "$LOG_FILE" 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests' validate_log "$LOG_FILE" remove_line "${TEST_OUT_DIR}/save_config.yaml" " connection_string: " @@ -387,6 +388,7 @@ test_jsn fnc_comment_cfg fnc_Mixed_Name kill_process "$MARTIN_PROC_ID" Martin test_log_has_str "$LOG_FILE" 'WARN martin::pg::query_tables] Table public.table_source has no spatial index on column geom' +test_log_has_str "$LOG_FILE" 'WARN martin::pg::query_tables] Table public.table_source_geog has no spatial index on column geog' test_log_has_str "$LOG_FILE" 'WARN martin::fonts] Ignoring duplicate font Overpass Mono Regular from tests' validate_log "$LOG_FILE" remove_line "${TEST_OUT_DIR}/save_config.yaml" " connection_string: "