From 422ca29ca230aebc13a4c7def4e4cf367411dd49 Mon Sep 17 00:00:00 2001 From: falke-design Date: Sun, 19 Sep 2021 15:59:00 +0200 Subject: [PATCH 1/5] Add helpText to requires --- openmaptiles/sql.py | 29 +++++++++++++++++++++++++-- openmaptiles/tileset.py | 8 +++++++- tests/python/test_sql.py | 43 ++++++++++++++++++++++++++++++++++++++-- 3 files changed, 75 insertions(+), 5 deletions(-) diff --git a/openmaptiles/sql.py b/openmaptiles/sql.py index ad2d232b..4fefc775 100644 --- a/openmaptiles/sql.py +++ b/openmaptiles/sql.py @@ -74,10 +74,35 @@ def collect_sql(tileset_filename, parallel=False, nodata=False def layer_to_sql(layer: Layer, nodata: bool): sql = f"DO $$ BEGIN RAISE NOTICE 'Processing layer {layer.id}'; END$$;\n\n" + for table in layer.requires_tables: - sql += f"-- Assert {table} exists\nSELECT '{table}'::regclass;\n\n" + tableErrorText = f"The required table '{table}' is not existing for the layer '{layer.id}'" + if layer.requires_helpText: + tableErrorText = layer.requires_helpText + tableErrorText = tableErrorText.replace("'", '"') + + sql += f'-- Assert {table} exists\n' + \ + 'do $$\nbegin\n' + \ + f" PERFORM '{table}'::regclass;\n" + \ + 'exception when undefined_table then\n' + \ + f" RAISE EXCEPTION '%! {tableErrorText}', SQLERRM;\n" + \ + "end;\n$$ language 'plpgsql';\n\n" + for func in layer.requires_functions: - sql += f"-- Assert {func} exists\nSELECT '{func}'::regprocedure;\n\n" + functionErrorText = f"The required function '{func}' is not existing for the layer '{layer.id}'" + if layer.requires_helpText: + functionErrorText = layer.requires_helpText + functionErrorText = functionErrorText.replace("'", '"') + + sql += f'-- Assert {func} exists\n' + \ + 'do $$\nbegin\n' + \ + f" PERFORM '{func}'::regprocedure;\n" + \ + 'exception when undefined_function then\n' + \ + f" RAISE EXCEPTION '%! {functionErrorText}', SQLERRM;\n" + \ + 'when invalid_text_representation then\n' + \ + f" RAISE EXCEPTION '%! The arguments of the required function \"{func}\" of the layer \"{layer.id}\" are missing. Example: \"{func}(TEXT, TEXT)\"', SQLERRM;\n" + \ + "end;\n$$ language 'plpgsql';\n\n" + for schema in layer.schemas: sql += to_sql(schema, layer, nodata) + '\n\n' sql += f"DO $$ BEGIN RAISE NOTICE 'Finished layer {layer.id}'; END$$;\n" diff --git a/openmaptiles/tileset.py b/openmaptiles/tileset.py index bf7594b9..a6d0028a 100644 --- a/openmaptiles/tileset.py +++ b/openmaptiles/tileset.py @@ -115,7 +115,8 @@ def __init__(self, else: requires = requires.copy() # dict will be modified to detect unrecognized properties - err = 'If set, "requires" parameter must be a map with optional "layers", "tables", and "functions" sub-elements. Each sub-element must be a string or a list of strings. If "requires" is a list or a string itself, it is treated as a list of layers. ' + err = 'If set, "requires" parameter must be a map with optional "layers", "tables", and "functions" sub-elements. Each sub-element must be a string or a list of strings. If "requires" is a list or a string itself, it is treated as a list of layers. ' + \ + 'Additionally a sub-element "helpText" can be defined, which is thrown as error message if one of the "requires" values are not existing.' self.requires_layers = get_requires_prop( requires, 'layers', err + '"requires.layers" must be an ID of another layer, or a list of layer IDs.') @@ -126,6 +127,11 @@ def __init__(self, requires, 'functions', err + '"requires.functions" must be a PostgreSQL function name with parameters or a list of functions. Example: "sql_func(TEXT, TEXT)"') + self.requires_helpText = None + if requires.get('helpText'): + self.requires_helpText = requires.get('helpText') + requires.pop('helpText', []) + if requires: # get_requires_prop will delete the key it handled. Remaining keys are errors. raise ValueError(f'Unrecognized sub-elements in the \"requires\" parameter: {str(list(requires.keys()))}') diff --git a/tests/python/test_sql.py b/tests/python/test_sql.py index f55053b9..4b1e9d4b 100644 --- a/tests/python/test_sql.py +++ b/tests/python/test_sql.py @@ -18,10 +18,36 @@ class Case: def expected_sql(case: Case): result = f"DO $$ BEGIN RAISE NOTICE 'Processing layer {case.id}'; END$$;\n\n" if isinstance(case.reqs, dict): + for table in case.reqs.get('tables', []): - result += f"-- Assert {table} exists\nSELECT '{table}'::regclass;\n\n" + tableErrorText = f"The required table '{table}' is not existing for the layer '{case.id}'" + if case.reqs.get('helpText'): + tableErrorText = case.reqs.get('helpText') + + tableErrorText = tableErrorText.replace("'",'"') + + result += f"-- Assert {table} exists\n" + \ + "do $$\nbegin\n" + \ + f" PERFORM '{table}'::regclass;\n" + \ + "exception when undefined_table then\n" + \ + f" RAISE EXCEPTION '%! {tableErrorText}', SQLERRM;" + \ + "end;\n$$ language 'plpgsql';\n\n" + for func in case.reqs.get('functions', []): - result += f"-- Assert {func} exists\nSELECT '{func}'::regprocedure;\n\n" + functionErrorText = f"The required function '{func}' is not existing for the layer '{case.id}'" + if case.reqs.get('helpText'): + functionErrorText = case.reqs.get('helpText') + + functionErrorText = functionErrorText.replace("'",'"') + + result += f"-- Assert {func} exists\n" + \ + "do $$\nbegin\n" + \ + f" PERFORM '{func}'::regprocedure;\n" + \ + "exception when undefined_function then\n" + \ + f" RAISE EXCEPTION '%! {functionErrorText}', SQLERRM;\n" + \ + "when invalid_text_representation then\n" + \ + f" RAISE EXCEPTION '%! The arguments of the required function \"{func}\" of the layer \"{case.id}\" are missing. Example: \"{func}(text)\"', SQLERRM;\n" + \ + "end;\n$$ language 'plpgsql';\n\n" result += f"""\ -- Layer {case.id} - {case.id}_s.yaml @@ -93,6 +119,14 @@ def _test(self, name, layers: List[Case], msg=f"{name} - parallel") def test_require(self): + + + c12 = Case("c12", "SELECT 12;", reqs=dict(functions=["fnc1", "fnc2"])) + self._test("a18", [c12], dict(c12=[c12])) + + return + + c1 = Case("c1", "SELECT 1;") c2 = Case("c2", "SELECT 2;") c3r2 = Case("c3", "SELECT 3;", reqs="c2") @@ -105,7 +139,10 @@ def test_require(self): c10 = Case("c10", "SELECT 10;", reqs=dict(tables=["tbl1", "tbl2"])) c11 = Case("c11", "SELECT 11;", reqs=dict(functions=["fnc1"])) c12 = Case("c12", "SELECT 12;", reqs=dict(functions=["fnc1", "fnc2"])) + c13 = Case("c13", "SELECT 13;", reqs=dict(functions=["fnc1", "fnc2"], helpText="Custom 'ERROR MESSAGE' for missing function - single quote")) + c14 = Case("c14", "SELECT 14;", reqs=dict(tables=["tbl1"], helpText='Custom "ERROR MESSAGE" for missing table - double quote')) + self._test("a18", [c12], dict(c12=[c12])) self._test("a01", [], {}) self._test("a02", [c1], dict(c1=c1)) self._test("a03", [c1, c2], dict(c1=c1, c2=c2)) @@ -127,6 +164,8 @@ def test_require(self): self._test("a16", [c10], dict(c10=[c10])) self._test("a17", [c11], dict(c11=[c11])) self._test("a18", [c12], dict(c12=[c12])) + self._test("a19", [c13], dict(c13=[c13])) + self._test("a20", [c14], dict(c14=[c14])) def _ts_parse(self, reqs, expected_layers, expected_tables, expected_funcs, extra_cases=None): cases = [] if not extra_cases else list(extra_cases) From e83b514041cdda93f2bc37e743d159ddf5d9c904 Mon Sep 17 00:00:00 2001 From: falke-design Date: Sun, 19 Sep 2021 16:17:32 +0200 Subject: [PATCH 2/5] Fix test --- tests/python/test_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/python/test_sql.py b/tests/python/test_sql.py index 4b1e9d4b..72165caa 100644 --- a/tests/python/test_sql.py +++ b/tests/python/test_sql.py @@ -46,7 +46,7 @@ def expected_sql(case: Case): "exception when undefined_function then\n" + \ f" RAISE EXCEPTION '%! {functionErrorText}', SQLERRM;\n" + \ "when invalid_text_representation then\n" + \ - f" RAISE EXCEPTION '%! The arguments of the required function \"{func}\" of the layer \"{case.id}\" are missing. Example: \"{func}(text)\"', SQLERRM;\n" + \ + f" RAISE EXCEPTION '%! The arguments of the required function \"{func}\" of the layer \"{case.id}\" are missing. Example: \"{func}(TEXT, TEXT)\"', SQLERRM;\n" + \ "end;\n$$ language 'plpgsql';\n\n" result += f"""\ -- Layer {case.id} - {case.id}_s.yaml From 1b9c3d29e142db3d96d0ad7f6c8dcab654d6722d Mon Sep 17 00:00:00 2001 From: falke-design Date: Sun, 19 Sep 2021 16:46:40 +0200 Subject: [PATCH 3/5] Add expected --- .../parallel_sql/parallel/mountain_peak.sql | 18 ++++++++++++++++-- .../parallel_sql2/parallel/mountain_peak.sql | 18 ++++++++++++++++-- tests/expected/sql.sql | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/tests/expected/parallel_sql/parallel/mountain_peak.sql b/tests/expected/parallel_sql/parallel/mountain_peak.sql index 02c42c22..328e5418 100644 --- a/tests/expected/parallel_sql/parallel/mountain_peak.sql +++ b/tests/expected/parallel_sql/parallel/mountain_peak.sql @@ -1,9 +1,23 @@ DO $$ BEGIN RAISE NOTICE 'Processing layer mountain_peak'; END$$; -- Assert my_magic_table exists -SELECT 'my_magic_table'::regclass; +do $$ +begin + PERFORM 'my_magic_table'::regclass; +exception when undefined_table then + RAISE EXCEPTION '%! The required table "my_magic_table" is not existing for the layer "mountain_peak"', SQLERRM; +end; +$$ language 'plpgsql'; -- Assert my_magic_func(TEXT, TEXT) exists -SELECT 'my_magic_func(TEXT, TEXT)'::regprocedure; +do $$ +begin + PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; +exception when undefined_function then + RAISE EXCEPTION '%! The required function "my_magic_func(TEXT, TEXT)" is not existing for the layer "mountain_peak"', SQLERRM; +when invalid_text_representation then + RAISE EXCEPTION '%! The arguments of the required function "my_magic_func(TEXT, TEXT)" of the layer "mountain_peak" are missing. Example: "my_magic_func(TEXT, TEXT)(TEXT, TEXT)"', SQLERRM; +end; +$$ language 'plpgsql'; DO $$ BEGIN RAISE NOTICE 'Finished layer mountain_peak'; END$$; diff --git a/tests/expected/parallel_sql2/parallel/mountain_peak.sql b/tests/expected/parallel_sql2/parallel/mountain_peak.sql index 02c42c22..328e5418 100644 --- a/tests/expected/parallel_sql2/parallel/mountain_peak.sql +++ b/tests/expected/parallel_sql2/parallel/mountain_peak.sql @@ -1,9 +1,23 @@ DO $$ BEGIN RAISE NOTICE 'Processing layer mountain_peak'; END$$; -- Assert my_magic_table exists -SELECT 'my_magic_table'::regclass; +do $$ +begin + PERFORM 'my_magic_table'::regclass; +exception when undefined_table then + RAISE EXCEPTION '%! The required table "my_magic_table" is not existing for the layer "mountain_peak"', SQLERRM; +end; +$$ language 'plpgsql'; -- Assert my_magic_func(TEXT, TEXT) exists -SELECT 'my_magic_func(TEXT, TEXT)'::regprocedure; +do $$ +begin + PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; +exception when undefined_function then + RAISE EXCEPTION '%! The required function "my_magic_func(TEXT, TEXT)" is not existing for the layer "mountain_peak"', SQLERRM; +when invalid_text_representation then + RAISE EXCEPTION '%! The arguments of the required function "my_magic_func(TEXT, TEXT)" of the layer "mountain_peak" are missing. Example: "my_magic_func(TEXT, TEXT)(TEXT, TEXT)"', SQLERRM; +end; +$$ language 'plpgsql'; DO $$ BEGIN RAISE NOTICE 'Finished layer mountain_peak'; END$$; diff --git a/tests/expected/sql.sql b/tests/expected/sql.sql index c8b2e62a..e2fa22d5 100644 --- a/tests/expected/sql.sql +++ b/tests/expected/sql.sql @@ -71,10 +71,24 @@ DO $$ BEGIN RAISE NOTICE 'Finished layer enumfield'; END$$; DO $$ BEGIN RAISE NOTICE 'Processing layer mountain_peak'; END$$; -- Assert my_magic_table exists -SELECT 'my_magic_table'::regclass; +do $$ +begin + PERFORM 'my_magic_table'::regclass; +exception when undefined_table then + RAISE EXCEPTION '%! The required table "my_magic_table" is not existing for the layer "mountain_peak"', SQLERRM; +end; +$$ language 'plpgsql'; -- Assert my_magic_func(TEXT, TEXT) exists -SELECT 'my_magic_func(TEXT, TEXT)'::regprocedure; +do $$ +begin + PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; +exception when undefined_function then + RAISE EXCEPTION '%! The required function "my_magic_func(TEXT, TEXT)" is not existing for the layer "mountain_peak"', SQLERRM; +when invalid_text_representation then + RAISE EXCEPTION '%! The arguments of the required function "my_magic_func(TEXT, TEXT)" of the layer "mountain_peak" are missing. Example: "my_magic_func(TEXT, TEXT)(TEXT, TEXT)"', SQLERRM; +end; +$$ language 'plpgsql'; DO $$ BEGIN RAISE NOTICE 'Finished layer mountain_peak'; END$$; From 8bd9c91722a140c25afb7e95c6e7292c8495b97e Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 21 Sep 2021 12:24:12 -0400 Subject: [PATCH 4/5] a few simplifications and code cleanup --- openmaptiles/pgutils.py | 5 ++ openmaptiles/sql.py | 63 ++++++++++++------- openmaptiles/tileset.py | 2 +- .../parallel_sql/parallel/mountain_peak.sql | 37 +++++------ .../parallel_sql2/parallel/mountain_peak.sql | 37 +++++------ tests/expected/sql.sql | 39 ++++++------ tests/python/test_sql.py | 48 +++----------- 7 files changed, 112 insertions(+), 119 deletions(-) diff --git a/openmaptiles/pgutils.py b/openmaptiles/pgutils.py index 8188ec4c..1e73ae2a 100644 --- a/openmaptiles/pgutils.py +++ b/openmaptiles/pgutils.py @@ -164,3 +164,8 @@ def print_query_error(error_msg, err, pg_warnings, verbose, query, layer_sql=Non query_msg += f'\n\n== MVT SQL\n{layer_sql}' print(query_msg) print(f'{line}\n') + + +def quote_literal(string): + """Adapted from asyncpg.utils""" + return "'{}'".format(string.replace("'", "''")) diff --git a/openmaptiles/sql.py b/openmaptiles/sql.py index 4fefc775..3048f2e8 100644 --- a/openmaptiles/sql.py +++ b/openmaptiles/sql.py @@ -3,6 +3,7 @@ from sys import stderr +from openmaptiles.pgutils import quote_literal from openmaptiles.tileset import Tileset, Layer @@ -76,32 +77,10 @@ def layer_to_sql(layer: Layer, nodata: bool): sql = f"DO $$ BEGIN RAISE NOTICE 'Processing layer {layer.id}'; END$$;\n\n" for table in layer.requires_tables: - tableErrorText = f"The required table '{table}' is not existing for the layer '{layer.id}'" - if layer.requires_helpText: - tableErrorText = layer.requires_helpText - tableErrorText = tableErrorText.replace("'", '"') - - sql += f'-- Assert {table} exists\n' + \ - 'do $$\nbegin\n' + \ - f" PERFORM '{table}'::regclass;\n" + \ - 'exception when undefined_table then\n' + \ - f" RAISE EXCEPTION '%! {tableErrorText}', SQLERRM;\n" + \ - "end;\n$$ language 'plpgsql';\n\n" + sql += sql_assert_table(table, layer.requires_helpText, layer.id) for func in layer.requires_functions: - functionErrorText = f"The required function '{func}' is not existing for the layer '{layer.id}'" - if layer.requires_helpText: - functionErrorText = layer.requires_helpText - functionErrorText = functionErrorText.replace("'", '"') - - sql += f'-- Assert {func} exists\n' + \ - 'do $$\nbegin\n' + \ - f" PERFORM '{func}'::regprocedure;\n" + \ - 'exception when undefined_function then\n' + \ - f" RAISE EXCEPTION '%! {functionErrorText}', SQLERRM;\n" + \ - 'when invalid_text_representation then\n' + \ - f" RAISE EXCEPTION '%! The arguments of the required function \"{func}\" of the layer \"{layer.id}\" are missing. Example: \"{func}(TEXT, TEXT)\"', SQLERRM;\n" + \ - "end;\n$$ language 'plpgsql';\n\n" + sql += sql_assert_func(func, layer.requires_helpText, layer.id) for schema in layer.schemas: sql += to_sql(schema, layer, nodata) + '\n\n' @@ -110,6 +89,42 @@ def layer_to_sql(layer: Layer, nodata: bool): return sql +def _sql_hint_clause(hint): + if hint: + return f',\n HINT = {quote_literal(hint)}' + else: + return '' + + +def sql_assert_table(table, hint, layer_id): + return f"""\ +DO $$ BEGIN + PERFORM {quote_literal(table)}::regclass; +EXCEPTION + WHEN undefined_table THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this table or view is required for layer "{layer_id}"'{_sql_hint_clause(hint)}; +END; +$$ LANGUAGE 'plpgsql';\n +""" + + +def sql_assert_func(func, hint, layer_id): + return f"""\ +DO $$ BEGIN + PERFORM {quote_literal(func)}::regprocedure; +EXCEPTION + WHEN undefined_function THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this function is required for layer "{layer_id}"'{_sql_hint_clause(hint)}; + WHEN invalid_text_representation THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'Required function "{func}" in layer "{layer_id}" is incorrectly declared. Use full function signature with parameter types, e.g. "my_magic_func(TEXT, TEXT)"'; +END; +$$ LANGUAGE 'plpgsql';\n +""" + + def get_slice_language_tags(tileset): include_tags = list(map(lambda l: 'name:' + l, tileset.languages)) include_tags.append('int_name') diff --git a/openmaptiles/tileset.py b/openmaptiles/tileset.py index a6d0028a..eea16e4f 100644 --- a/openmaptiles/tileset.py +++ b/openmaptiles/tileset.py @@ -116,7 +116,7 @@ def __init__(self, requires = requires.copy() # dict will be modified to detect unrecognized properties err = 'If set, "requires" parameter must be a map with optional "layers", "tables", and "functions" sub-elements. Each sub-element must be a string or a list of strings. If "requires" is a list or a string itself, it is treated as a list of layers. ' + \ - 'Additionally a sub-element "helpText" can be defined, which is thrown as error message if one of the "requires" values are not existing.' + 'Optionally add "helpText" sub-element string to help the user with generating missing tables and functions.' self.requires_layers = get_requires_prop( requires, 'layers', err + '"requires.layers" must be an ID of another layer, or a list of layer IDs.') diff --git a/tests/expected/parallel_sql/parallel/mountain_peak.sql b/tests/expected/parallel_sql/parallel/mountain_peak.sql index 328e5418..c0c94e63 100644 --- a/tests/expected/parallel_sql/parallel/mountain_peak.sql +++ b/tests/expected/parallel_sql/parallel/mountain_peak.sql @@ -1,23 +1,24 @@ DO $$ BEGIN RAISE NOTICE 'Processing layer mountain_peak'; END$$; --- Assert my_magic_table exists -do $$ -begin - PERFORM 'my_magic_table'::regclass; -exception when undefined_table then - RAISE EXCEPTION '%! The required table "my_magic_table" is not existing for the layer "mountain_peak"', SQLERRM; -end; -$$ language 'plpgsql'; +DO $$ BEGIN + PERFORM 'my_magic_table'::regclass; +EXCEPTION + WHEN undefined_table THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this table or view is required for layer "mountain_peak"'; +END; +$$ LANGUAGE 'plpgsql'; --- Assert my_magic_func(TEXT, TEXT) exists -do $$ -begin - PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; -exception when undefined_function then - RAISE EXCEPTION '%! The required function "my_magic_func(TEXT, TEXT)" is not existing for the layer "mountain_peak"', SQLERRM; -when invalid_text_representation then - RAISE EXCEPTION '%! The arguments of the required function "my_magic_func(TEXT, TEXT)" of the layer "mountain_peak" are missing. Example: "my_magic_func(TEXT, TEXT)(TEXT, TEXT)"', SQLERRM; -end; -$$ language 'plpgsql'; +DO $$ BEGIN + PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; +EXCEPTION + WHEN undefined_function THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this function is required for layer "mountain_peak"'; + WHEN invalid_text_representation THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'Required function "my_magic_func(TEXT, TEXT)" in layer "mountain_peak" is incorrectly declared. Use full function signature with parameter types, e.g. "my_magic_func(TEXT, TEXT)"'; +END; +$$ LANGUAGE 'plpgsql'; DO $$ BEGIN RAISE NOTICE 'Finished layer mountain_peak'; END$$; diff --git a/tests/expected/parallel_sql2/parallel/mountain_peak.sql b/tests/expected/parallel_sql2/parallel/mountain_peak.sql index 328e5418..c0c94e63 100644 --- a/tests/expected/parallel_sql2/parallel/mountain_peak.sql +++ b/tests/expected/parallel_sql2/parallel/mountain_peak.sql @@ -1,23 +1,24 @@ DO $$ BEGIN RAISE NOTICE 'Processing layer mountain_peak'; END$$; --- Assert my_magic_table exists -do $$ -begin - PERFORM 'my_magic_table'::regclass; -exception when undefined_table then - RAISE EXCEPTION '%! The required table "my_magic_table" is not existing for the layer "mountain_peak"', SQLERRM; -end; -$$ language 'plpgsql'; +DO $$ BEGIN + PERFORM 'my_magic_table'::regclass; +EXCEPTION + WHEN undefined_table THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this table or view is required for layer "mountain_peak"'; +END; +$$ LANGUAGE 'plpgsql'; --- Assert my_magic_func(TEXT, TEXT) exists -do $$ -begin - PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; -exception when undefined_function then - RAISE EXCEPTION '%! The required function "my_magic_func(TEXT, TEXT)" is not existing for the layer "mountain_peak"', SQLERRM; -when invalid_text_representation then - RAISE EXCEPTION '%! The arguments of the required function "my_magic_func(TEXT, TEXT)" of the layer "mountain_peak" are missing. Example: "my_magic_func(TEXT, TEXT)(TEXT, TEXT)"', SQLERRM; -end; -$$ language 'plpgsql'; +DO $$ BEGIN + PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; +EXCEPTION + WHEN undefined_function THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this function is required for layer "mountain_peak"'; + WHEN invalid_text_representation THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'Required function "my_magic_func(TEXT, TEXT)" in layer "mountain_peak" is incorrectly declared. Use full function signature with parameter types, e.g. "my_magic_func(TEXT, TEXT)"'; +END; +$$ LANGUAGE 'plpgsql'; DO $$ BEGIN RAISE NOTICE 'Finished layer mountain_peak'; END$$; diff --git a/tests/expected/sql.sql b/tests/expected/sql.sql index e2fa22d5..6e8c8f41 100644 --- a/tests/expected/sql.sql +++ b/tests/expected/sql.sql @@ -70,25 +70,26 @@ DO $$ BEGIN RAISE NOTICE 'Finished layer enumfield'; END$$; DO $$ BEGIN RAISE NOTICE 'Processing layer mountain_peak'; END$$; --- Assert my_magic_table exists -do $$ -begin - PERFORM 'my_magic_table'::regclass; -exception when undefined_table then - RAISE EXCEPTION '%! The required table "my_magic_table" is not existing for the layer "mountain_peak"', SQLERRM; -end; -$$ language 'plpgsql'; - --- Assert my_magic_func(TEXT, TEXT) exists -do $$ -begin - PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; -exception when undefined_function then - RAISE EXCEPTION '%! The required function "my_magic_func(TEXT, TEXT)" is not existing for the layer "mountain_peak"', SQLERRM; -when invalid_text_representation then - RAISE EXCEPTION '%! The arguments of the required function "my_magic_func(TEXT, TEXT)" of the layer "mountain_peak" are missing. Example: "my_magic_func(TEXT, TEXT)(TEXT, TEXT)"', SQLERRM; -end; -$$ language 'plpgsql'; +DO $$ BEGIN + PERFORM 'my_magic_table'::regclass; +EXCEPTION + WHEN undefined_table THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this table or view is required for layer "mountain_peak"'; +END; +$$ LANGUAGE 'plpgsql'; + +DO $$ BEGIN + PERFORM 'my_magic_func(TEXT, TEXT)'::regprocedure; +EXCEPTION + WHEN undefined_function THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'this function is required for layer "mountain_peak"'; + WHEN invalid_text_representation THEN + RAISE EXCEPTION '%', SQLERRM + USING DETAIL = 'Required function "my_magic_func(TEXT, TEXT)" in layer "mountain_peak" is incorrectly declared. Use full function signature with parameter types, e.g. "my_magic_func(TEXT, TEXT)"'; +END; +$$ LANGUAGE 'plpgsql'; DO $$ BEGIN RAISE NOTICE 'Finished layer mountain_peak'; END$$; diff --git a/tests/python/test_sql.py b/tests/python/test_sql.py index 72165caa..ee9a5b76 100644 --- a/tests/python/test_sql.py +++ b/tests/python/test_sql.py @@ -4,7 +4,7 @@ from typing import List, Union, Dict from unittest import main, TestCase -from openmaptiles.sql import collect_sql +from openmaptiles.sql import collect_sql, sql_assert_table, sql_assert_func from openmaptiles.tileset import ParsedData, Tileset @@ -18,36 +18,12 @@ class Case: def expected_sql(case: Case): result = f"DO $$ BEGIN RAISE NOTICE 'Processing layer {case.id}'; END$$;\n\n" if isinstance(case.reqs, dict): - + # Use helper functions for SQL generation. Actual SQL is tested by integration tests for table in case.reqs.get('tables', []): - tableErrorText = f"The required table '{table}' is not existing for the layer '{case.id}'" - if case.reqs.get('helpText'): - tableErrorText = case.reqs.get('helpText') - - tableErrorText = tableErrorText.replace("'",'"') - - result += f"-- Assert {table} exists\n" + \ - "do $$\nbegin\n" + \ - f" PERFORM '{table}'::regclass;\n" + \ - "exception when undefined_table then\n" + \ - f" RAISE EXCEPTION '%! {tableErrorText}', SQLERRM;" + \ - "end;\n$$ language 'plpgsql';\n\n" - + result += sql_assert_table(table, case.reqs.get('helpText'), case.id) for func in case.reqs.get('functions', []): - functionErrorText = f"The required function '{func}' is not existing for the layer '{case.id}'" - if case.reqs.get('helpText'): - functionErrorText = case.reqs.get('helpText') - - functionErrorText = functionErrorText.replace("'",'"') - - result += f"-- Assert {func} exists\n" + \ - "do $$\nbegin\n" + \ - f" PERFORM '{func}'::regprocedure;\n" + \ - "exception when undefined_function then\n" + \ - f" RAISE EXCEPTION '%! {functionErrorText}', SQLERRM;\n" + \ - "when invalid_text_representation then\n" + \ - f" RAISE EXCEPTION '%! The arguments of the required function \"{func}\" of the layer \"{case.id}\" are missing. Example: \"{func}(TEXT, TEXT)\"', SQLERRM;\n" + \ - "end;\n$$ language 'plpgsql';\n\n" + result += sql_assert_func(func, case.reqs.get('helpText'), case.id) + result += f"""\ -- Layer {case.id} - {case.id}_s.yaml @@ -119,14 +95,6 @@ def _test(self, name, layers: List[Case], msg=f"{name} - parallel") def test_require(self): - - - c12 = Case("c12", "SELECT 12;", reqs=dict(functions=["fnc1", "fnc2"])) - self._test("a18", [c12], dict(c12=[c12])) - - return - - c1 = Case("c1", "SELECT 1;") c2 = Case("c2", "SELECT 2;") c3r2 = Case("c3", "SELECT 3;", reqs="c2") @@ -139,8 +107,10 @@ def test_require(self): c10 = Case("c10", "SELECT 10;", reqs=dict(tables=["tbl1", "tbl2"])) c11 = Case("c11", "SELECT 11;", reqs=dict(functions=["fnc1"])) c12 = Case("c12", "SELECT 12;", reqs=dict(functions=["fnc1", "fnc2"])) - c13 = Case("c13", "SELECT 13;", reqs=dict(functions=["fnc1", "fnc2"], helpText="Custom 'ERROR MESSAGE' for missing function - single quote")) - c14 = Case("c14", "SELECT 14;", reqs=dict(tables=["tbl1"], helpText='Custom "ERROR MESSAGE" for missing table - double quote')) + c13 = Case("c13", "SELECT 13;", reqs=dict(functions=["fnc1", "fnc2"], + helpText="Custom 'ERROR MESSAGE' for missing function - single quote")) + c14 = Case("c14", "SELECT 14;", + reqs=dict(tables=["tbl1"], helpText='Custom "ERROR MESSAGE" for missing table - double quote')) self._test("a18", [c12], dict(c12=[c12])) self._test("a01", [], {}) From 7d633293d45d07a2f916f374135b7249969acebf Mon Sep 17 00:00:00 2001 From: Yuri Astrakhan Date: Tue, 21 Sep 2021 18:29:19 -0400 Subject: [PATCH 5/5] fix lint issues --- tests/python/test_sql.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/python/test_sql.py b/tests/python/test_sql.py index 69dee349..37ffe103 100644 --- a/tests/python/test_sql.py +++ b/tests/python/test_sql.py @@ -107,12 +107,12 @@ def test_require(self): c10 = Case('c10', 'SELECT 10;', reqs=dict(tables=['tbl1', 'tbl2'])) c11 = Case('c11', 'SELECT 11;', reqs=dict(functions=['fnc1'])) c12 = Case('c12', 'SELECT 12;', reqs=dict(functions=['fnc1', 'fnc2'])) - c13 = Case("c13", "SELECT 13;", reqs=dict(functions=["fnc1", "fnc2"], + c13 = Case('c13', 'SELECT 13;', reqs=dict(functions=['fnc1', 'fnc2'], helpText="Custom 'ERROR MESSAGE' for missing function - single quote")) - c14 = Case("c14", "SELECT 14;", - reqs=dict(tables=["tbl1"], helpText='Custom "ERROR MESSAGE" for missing table - double quote')) + c14 = Case('c14', 'SELECT 14;', + reqs=dict(tables=['tbl1'], helpText='Custom "ERROR MESSAGE" for missing table - double quote')) - self._test("a18", [c12], dict(c12=[c12])) + self._test('a18', [c12], dict(c12=[c12])) self._test('a01', [], {}) self._test('a02', [c1], dict(c1=c1)) self._test('a03', [c1, c2], dict(c1=c1, c2=c2)) @@ -134,8 +134,8 @@ def test_require(self): self._test('a16', [c10], dict(c10=[c10])) self._test('a17', [c11], dict(c11=[c11])) self._test('a18', [c12], dict(c12=[c12])) - self._test("a19", [c13], dict(c13=[c13])) - self._test("a20", [c14], dict(c14=[c14])) + self._test('a19', [c13], dict(c13=[c13])) + self._test('a20', [c14], dict(c14=[c14])) def _ts_parse(self, reqs, expected_layers, expected_tables, expected_funcs, extra_cases=None): cases = [] if not extra_cases else list(extra_cases)