Merge pull request #10 from joshtemple/release/0.1.1

0.1.1

lkml/__init__.py

@@ -47,7 +47,7 @@ def cli():
     handler = logging.StreamHandler()
     handler.setLevel(logging.DEBUG)
 
-    formatter = logging.Formatter("%(name)s . %(message)s")
+    formatter = logging.Formatter("%(name)s %(levelname)s: %(message)s")
 
     handler.setFormatter(formatter)
     logger.addHandler(handler)

lkml/parser.py

@@ -14,7 +14,7 @@
 
 list = key "[" csv? "]"
 
-csv = (literal / quoted_literal) ("," (literal / quoted_literal))*
+csv = (literal / quoted_literal) ("," (literal / quoted_literal))* ","?
 
 value = literal / quoted_literal / expression_block
 
@@ -97,53 +97,67 @@ def check(self, *token_types: Type[tokens.Token]) -> bool:
     def parse(self) -> List:
         return self.parse_expression()
 
-    @staticmethod
-    def update_tree(target, update):
+    def update_tree(self, target, update):
+        plural_keys = frozenset(
+            (
+                "view",
+                "measure",
+                "dimension",
+                "dimension_group",
+                "filter",
+                "access_filter",
+                "bind_filters",
+                "map_layer",
+                "parameter",
+                "set",
+                "column",
+                "derived_column",
+                "include",
+                "explore",
+                "link",
+                "when",
+                "allowed_value",
+                "named_value_format",
+                "join",
+                "datagroup",
+                "access_grant",
+                "sql_step",
+                "sql_where",
+            )
+        )
+
         keys = tuple(update.keys())
         if len(keys) > 1:
             raise KeyError("Dictionary to update with cannot have multiple keys.")
         key = keys[0]
-        if key.rstrip("s") in [
-            "view",
-            "measure",
-            "dimension",
-            "dimension_group",
-            "filter",
-            "access_filter",
-            "bind_filters",
-            "map_layer",
-            "parameter",
-            "set",
-            "column",
-            "derived_column",
-            "include",
-            "explore",
-            "link",
-            "when",
-            "allowed_value",
-            "named_value_format",
-            "join",
-            "datagroup",
-            "access_grant",
-        ]:
-            plural_key = key.rstrip("s") + "s"
+        stripped_key = key.rstrip("s")
+        if stripped_key in plural_keys:
+            plural_key = stripped_key + "s"
             if plural_key in target.keys():
                 target[plural_key].append(update[key])
             else:
                 target[plural_key] = [update[key]]
         elif key in target.keys():
-            raise KeyError(
-                f'Key "{key}" already exists in tree '
-                "and would overwrite the existing value."
-            )
+            if self.depth == 0:
+                self.logger.warning(
+                    'Multiple declarations of top-level key "%s" found. '
+                    "Using the last-declared value.",
+                    key,
+                )
+                target[key] = update[key]
+            else:
+                raise KeyError(
+                    f'Key "{key}" already exists in tree '
+                    "and would overwrite the existing value."
+                )
         else:
             target[key] = update[key]
 
     @backtrack_if_none
     def parse_expression(self) -> dict:
         if self.log_debug:
             grammar = "[expression] = (block / pair / list)*"
-            self.logger.debug(self.depth * DELIMITER + f"Try to parse {grammar}")
+            self.logger.debug("%sTry to parse %s", self.depth * DELIMITER, grammar)
         expression: dict = {}
         if self.check(tokens.StreamStartToken):
             self.advance()
@@ -166,7 +180,7 @@ def parse_expression(self) -> dict:
             token = self.tokens[self.progress]
             raise SyntaxError(
                 f"Unable to find a matching expression for '{token.id}' "
-                f"on line {token.line_number}, position "
+                f"on line {token.line_number}"
             )
 
         if self.log_debug:
@@ -307,9 +321,7 @@ def parse_list(self) -> Optional[dict]:
     @backtrack_if_none
     def parse_csv(self) -> Optional[list]:
         if self.log_debug:
-            grammar = (
-                "[csv] = (literal / quoted_literal) (" " (literal / quoted_literal))*"
-            )
+            grammar = '[csv] = (literal / quoted_literal) ("," (literal / quoted_literal))* ","?'
             self.logger.debug("%sTry to parse %s", self.depth * DELIMITER, grammar)
         values = []
 
@@ -330,6 +342,8 @@ def parse_csv(self) -> Optional[list]:
                 values.append(self.consume_token_value())
             elif self.check(tokens.QuotedLiteralToken):
                 values.append(self.consume_token_value())
+            elif self.check(tokens.ListEndToken):
+                break
             else:
                 return None
 

lkml/tokens.py

@@ -10,7 +10,7 @@ def __eq__(self, other):
 
     def __repr__(self):
         value = getattr(self, "value", "").strip()
-        value = (value[:25].rstrip() + " ...") if len(value) > 25 else value
+        value = (value[:25].rstrip() + " ... ") if len(value) > 25 else value
         return f"{self.__class__.__name__}({value})"
 
 

setup.py

@@ -2,7 +2,7 @@
 from setuptools import setup, find_packages
 from codecs import open
 
-__version__ = "0.1.0"
+__version__ = "0.1.1"
 
 here = Path(__file__).parent.resolve()
 
@@ -18,17 +18,16 @@
     long_description_content_type="text/markdown",
     url="https://github.com/joshtemple/lkml",
     download_url="https://github.com/joshtemple/lkml/tarball/" + __version__,
-    license="BSD",
     classifiers=[
         "Development Status :: 3 - Alpha",
         "Intended Audience :: Developers",
         "License :: OSI Approved :: MIT License",
         "Programming Language :: Python :: 3.7",
         "Topic :: Software Development",
     ],
-    keywords="",
+    keywords="lookml looker parser",
     entry_points={"console_scripts": ["lkml = lkml.__init__:cli"]},
-    packages=find_packages(exclude=["docs", "tests*"]),
+    packages=find_packages(exclude=["docs", "tests*", "scripts"]),
     include_package_data=True,
     author="Josh Temple",
     tests_require=["pytest"],

tests/resources/duplicate_non_top_level_keys.view.lkml

@@ -0,0 +1,4 @@
+view: view_name {
+  label: "View Label"
+  label: "Another View Label"
+}

tests/resources/duplicate_top_level_keys.view.lkml

@@ -0,0 +1,10 @@
+connection: "prod-data-playground-std-bq"
+connection: "prod-data-playground-std-bq"
+label: "CIE - Memberhip Facts"
+label: "CIE - Memberhip Facts"
+week_start_day: sunday
+week_start_day: sunday
+include: "*.view.lkml"                       # include all views in this project
+include: "*.view.lkml"                       # include all views in this project
+include: "attendance_base.explore.lkml"
+include: "attendance_base.explore.lkml"

tests/test_functional.py

@@ -1,4 +1,5 @@
 from pathlib import Path
+import pytest
 import lkml
 
 
@@ -32,3 +33,13 @@ def test_view_with_all_fields():
 def test_model_with_all_fields():
     lookml = load("model_with_all_fields.model.lkml")
     assert lookml is not None
+
+
+def test_duplicate_top_level_keys():
+    lookml = load("duplicate_top_level_keys.view.lkml")
+    assert lookml is not None
+
+
+def test_duplicate_non_top_level_keys():
+    with pytest.raises(KeyError):
+        lookml = load("duplicate_non_top_level_keys.view.lkml")

tests/test_parser.py

@@ -365,7 +365,7 @@ def test_parse_list_with_trailing_comma():
     )
     parser = lkml.parser.Parser(stream)
     result = parser.parse_list()
-    assert result is None
+    assert result == {"drill_fields": ["view_name.field_one"]}
 
 
 def test_parse_list_with_missing_comma():
@@ -412,6 +412,40 @@ def test_parse_list_with_no_opening_bracket():
     assert result is None
 
 
+def test_parse_list_with_bad_token():
+    stream = (
+        tokens.LiteralToken("drill_fields", 1),
+        tokens.ValueToken(1),
+        tokens.ListStartToken(1),
+        tokens.LiteralToken("view_name.field_one", 1),
+        tokens.CommaToken(1),
+        tokens.LiteralToken("view_name.field_two", 1),
+        tokens.CommaToken(1),
+        tokens.ValueToken(1),
+        tokens.ListEndToken(1),
+        tokens.StreamEndToken(1),
+    )
+    parser = lkml.parser.Parser(stream)
+    result = parser.parse_list()
+    assert result is None
+
+
+def test_parse_list_with_only_commas():
+    stream = (
+        tokens.LiteralToken("drill_fields", 1),
+        tokens.ValueToken(1),
+        tokens.ListStartToken(1),
+        tokens.CommaToken(1),
+        tokens.CommaToken(1),
+        tokens.CommaToken(1),
+        tokens.ListEndToken(1),
+        tokens.StreamEndToken(1),
+    )
+    parser = lkml.parser.Parser(stream)
+    result = parser.parse_list()
+    assert result is None
+
+
 def test_parse_block_with_no_expression():
     stream = (
         tokens.LiteralToken("dimension", 1),