Merge pull request #13 from joshtemple/release/0.1.2

0.1.2

lkml/__init__.py

@@ -2,7 +2,6 @@
 import sys
 import argparse
 import json
-from pathlib import Path
 from lkml.lexer import Lexer
 from lkml.parser import Parser
 

lkml/lexer.py

@@ -1,12 +1,33 @@
 from typing import Tuple, List
 import lkml.tokens as tokens
 
+EXPR_BLOCK_KEYS = (
+    "expression_custom_filter",
+    "html",
+    "sql_trigger_value",
+    "sql_table_name",
+    "sql_distinct_key",
+    "sql_start",
+    "sql_always_having",
+    "sql_always_where",
+    "sql_trigger",
+    "sql_foreign_key",
+    "sql_where",
+    "sql_end",
+    "sql_create",
+    "sql_latitude",
+    "sql_longitude",
+    "sql_step",
+    "sql_on",
+    "sql",
+)
+
 
 class Lexer:
-    def __init__(self, text):
+    def __init__(self, text: str):
         self.text = text + "\0"
         self.index = 0
-        self.tokens = []
+        self.tokens: List[tokens.Token] = []
         self.line_number = 1
 
     def peek(self) -> str:
@@ -68,11 +89,7 @@ def scan(self) -> Tuple[tokens.Token, ...]:
             elif ch == '"':
                 self.advance()
                 self.tokens.append(self.scan_quoted_literal())
-            elif (
-                self.peek_multiple(3) == "sql"
-                or self.peek_multiple(4) == "html"
-                or self.peek_multiple(24) == "expression_custom_filter"
-            ):
+            elif self.check_for_expression_block(self.peek_multiple(25)):
                 self.tokens.append(self.scan_literal())
                 self.scan_until_token()
                 self.advance()
@@ -86,6 +103,10 @@ def scan(self) -> Tuple[tokens.Token, ...]:
 
         return tuple(self.tokens)
 
+    @staticmethod
+    def check_for_expression_block(string: str):
+        return any(string.startswith(key + ":") for key in EXPR_BLOCK_KEYS)
+
     def scan_expression_block(self) -> tokens.ExpressionBlockToken:
         chars = ""
         while self.peek_multiple(2) != ";;":
@@ -104,7 +125,6 @@ def scan_literal(self) -> tokens.LiteralToken:
         return tokens.LiteralToken(chars, self.line_number)
 
     def scan_quoted_literal(self) -> tokens.QuotedLiteralToken:
-        # TODO: Check and see if literals can be single-quoted
         chars = ""
         while True:
             ch = self.peek()

lkml/parser.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List, Sequence, Type, Optional, Any, Union
+from typing import List, Sequence, Type, Optional, Any
 import lkml.tokens as tokens
 
 """
@@ -31,6 +31,39 @@
 # Delimiter during logging to show parse tree depth
 DELIMITER = ". "
 
+PLURAL_KEYS = frozenset(
+    (
+        "view",
+        "measure",
+        "dimension",
+        "dimension_group",
+        "filter",
+        "access_filter",
+        "bind_filters",
+        "map_layer",
+        "parameter",
+        "set",
+        "column",
+        "derived_column",
+        "include",
+        "explore",
+        "link",
+        "when",
+        "allowed_value",
+        "named_value_format",
+        "join",
+        "datagroup",
+        "access_grant",
+        "sql_step",
+        "sql_where",
+        "action",
+        "param",
+        "form_param",
+        "option",
+        "user_attribute_param",
+    )
+)
+
 
 class Parser:
     def __init__(self, stream: Sequence[tokens.Token]):
@@ -97,41 +130,13 @@ def check(self, *token_types: Type[tokens.Token]) -> bool:
     def parse(self) -> List:
         return self.parse_expression()
 
-    def update_tree(self, target, update):
-        plural_keys = frozenset(
-            (
-                "view",
-                "measure",
-                "dimension",
-                "dimension_group",
-                "filter",
-                "access_filter",
-                "bind_filters",
-                "map_layer",
-                "parameter",
-                "set",
-                "column",
-                "derived_column",
-                "include",
-                "explore",
-                "link",
-                "when",
-                "allowed_value",
-                "named_value_format",
-                "join",
-                "datagroup",
-                "access_grant",
-                "sql_step",
-                "sql_where",
-            )
-        )
-
+    def update_tree(self, target: dict, update: dict):
         keys = tuple(update.keys())
         if len(keys) > 1:
             raise KeyError("Dictionary to update with cannot have multiple keys.")
         key = keys[0]
         stripped_key = key.rstrip("s")
-        if stripped_key in plural_keys:
+        if stripped_key in PLURAL_KEYS:
             plural_key = stripped_key + "s"
             if plural_key in target.keys():
                 target[plural_key].append(update[key])

setup.py

@@ -2,7 +2,7 @@
 from setuptools import setup, find_packages
 from codecs import open
 
-__version__ = "0.1.1"
+__version__ = "0.1.2"
 
 here = Path(__file__).parent.resolve()
 
@@ -13,7 +13,7 @@
 setup(
     name="lkml",
     version=__version__,
-    description="A fast LookML parser.",
+    description="A speedy LookML parser implemented in pure Python.",
     long_description=long_description,
     long_description_content_type="text/markdown",
     url="https://github.com/joshtemple/lkml",
@@ -26,6 +26,7 @@
         "Topic :: Software Development",
     ],
     keywords="lookml looker parser",
+    license="MIT",
     entry_points={"console_scripts": ["lkml = lkml.__init__:cli"]},
     packages=find_packages(exclude=["docs", "tests*", "scripts"]),
     include_package_data=True,

tests/resources/block_with_reserved_dimension_names.view.lkml

@@ -0,0 +1,10 @@
+view: view_name {
+  sql: select ${orders.items}, "Some SQL" as thing, 'Some Quoted Literal' from TABLE ;;
+  dimension: sql {}
+  dimension: sql_table_name {}
+  dimension: sql_dimension_name {}
+  dimension: html {}
+  dimension: html_dimension_name {}
+  dimension: expression_custom_filter {}
+  dimension: expression_custom_filter {}
+}

tests/resources/view_with_all_fields.view.lkml

@@ -3,9 +3,7 @@ view: view_name {
   suggestions: no
   extends: [view_name, view_name]
   extension: required
-
-required_access_grants: [access_grant_name, access_grant_name]
-
+  required_access_grants: [access_grant_name, access_grant_name]
   derived_table: {
     explore_source: explore_name {
       bind_filters: {
@@ -55,12 +53,83 @@ required_access_grants: [access_grant_name, access_grant_name]
     }
     sql_create: SQL query ;;
   }
-
   set: set_name {
     fields: [field_or_set, field_or_set]
   }
-
   dimension: field_name {
+    action: {
+      label: "Label to Appear in Action Menu"
+      url: "url"
+      icon_url: "url"
+      form_url: "url"
+      user_attribute_param: {
+        user_attribute: user_attribute_name
+        name: "name_for_json_payload"
+      }
+      user_attribute_param: {
+        user_attribute: user_attribute_name
+        name: "name_for_json_payload"
+      }
+      param: {
+        name: "name string"
+        value: "value string"
+      }
+      param: {
+        name: "name string"
+        value: "value string"
+      }
+      form_param: {
+        name: "form_param_name_1"
+        type: select
+        label: "desired label name"
+        required: yes
+        default: "value string"
+        option: {
+          name: "name string"
+          value: "value string"
+        }
+        option: {
+          name: "name string"
+          value: "value string"
+        }
+      }
+      form_param: {
+        name: "title"
+        type: string
+        label: "desired label name"
+        required: no
+        default: "value string"
+        option: {
+          name: "name string"
+          value: "value string"
+        }
+        option: {
+          name: "name string"
+          value: "value string"
+        }
+      }
+    }
+    action: {
+      label: "Another Label to Appear in Action Menu"
+      url: "url"
+      icon_url: "url"
+      form_url: "url"
+      param: {
+        name: "name string"
+        value: "value string"
+      }
+      form_param: {
+        name: "form_param_name_1"
+        type: select
+        label: "desired label name"
+        required: yes
+        default: "value string"
+        option: {
+          name: "name string"
+          value: "value string"
+        }
+      }
+    }
     label: "desired label name"
     label_from_parameter: parameter_name
     view_label: "desired label name"
@@ -85,7 +154,10 @@ required_access_grants: [access_grant_name, access_grant_name]
         sql: SQL condition ;;
         label: "value"
       }
-      # Possibly more when statements
+      when: {
+        sql: SQL condition ;;
+        label: "value"
+      }
     }
     alpha_sort: no
     tiers: [N, N]
@@ -101,7 +173,10 @@ required_access_grants: [access_grant_name, access_grant_name]
       label: "desired label name"
       value: "looker filter expression"
     }
-    # Possibly more allowed_value definitions
+    allowed_value: {
+      label: "desired label name"
+      value: "looker filter expression"
+    }
     required_access_grants: [access_grant_name, access_grant_name]
     bypass_suggest_restrictions: no
     full_suggestions: no
@@ -114,8 +189,11 @@ required_access_grants: [access_grant_name, access_grant_name]
       url: "desired_url"
       icon_url: "url_of_an_ico_file"
     }
-    # Possibly more links
-
+    link: {
+      label: "desired label name;"
+      url: "desired_url"
+      icon_url: "url_of_an_ico_file"
+    }
     timeframes: [timeframe, timeframe]
     convert_tz: no
     datatype: timestamp
@@ -129,14 +207,14 @@ required_access_grants: [access_grant_name, access_grant_name]
     list_field: dimension_name
     percentile: 90
     precision: N
-
     filters: {
       field: dimension_name
       value: "looker filter expression"
     }
-    # Possibly more filters statements
-
+    filters: {
+      field: dimension_name
+      value: "looker filter expression"
+    }
     default_value: "desired default value"
   }
-  # Possibly more dimension or measure declarations
 }

tests/test_functional.py

@@ -43,3 +43,8 @@ def test_duplicate_top_level_keys():
 def test_duplicate_non_top_level_keys():
     with pytest.raises(KeyError):
         lookml = load("duplicate_non_top_level_keys.view.lkml")
+
+
+def test_reserved_dimension_names():
+    lookml = load("block_with_reserved_dimension_names.view.lkml")
+    assert lookml is not None

tests/test_lexer.py

@@ -151,3 +151,27 @@ def test_scan_with_complex_sql_block():
         tokens.ExpressionBlockEndToken(1),
         tokens.StreamEndToken(1),
     )
+
+
+def test_scan_with_non_expression_block_starting_with_sql():
+    text = "sql_not_reserved_field: yes"
+    output = lkml.Lexer(text).scan()
+    assert output == (
+        tokens.StreamStartToken(1),
+        tokens.LiteralToken("sql_not_reserved_field", 1),
+        tokens.ValueToken(1),
+        tokens.LiteralToken("yes", 1),
+        tokens.StreamEndToken(1),
+    )
+
+
+def test_scan_with_non_expression_block_starting_with_html():
+    text = "html_not_reserved_field: yes"
+    output = lkml.Lexer(text).scan()
+    assert output == (
+        tokens.StreamStartToken(1),
+        tokens.LiteralToken("html_not_reserved_field", 1),
+        tokens.ValueToken(1),
+        tokens.LiteralToken("yes", 1),
+        tokens.StreamEndToken(1),
+    )