final 3.8 updates and dtype support

.github/workflows/lint.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.8"]
 
     steps:
       - name: Check out the code

.github/workflows/test.yaml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.10"]
+        python-version: ["3.8"]
     steps:
       - name: Check out the code
         uses: actions/checkout@v4

README.md

@@ -49,13 +49,13 @@ python
 ```
 3. Generate a semantic model
 ```python
-from semantic_model_generator.generate_model import generate_base_semantic_context_from_snowflake
+from semantic_model_generator.generate_model import generate_base_semantic_model_from_snowflake
 
 PHYSICAL_TABLES = ['<your-database-name-1>.<your-schema-name-1>.<your-physical-table-or-view-name-1>','<your-database-name-2>.<your-schema-name-2>.<your-physical-table-or-view-name-2>']
 SNOWFLAKE_ACCOUNT = "<your-snowflake-account>"
 SEMANTIC_MODEL_NAME = "<a-meaningful-semantic-model-name>"
 
-generate_base_semantic_context_from_snowflake(
+generate_base_semantic_model_from_snowflake(
     physical_tables=PHYSICAL_TABLES,
     snowflake_account=SNOWFLAKE_ACCOUNT,
     semantic_model_name=SEMANTIC_MODEL_NAME

semantic_model_generator/generate_model.py

@@ -1,5 +1,5 @@
 from datetime import datetime
-from typing import Optional
+from typing import List, Optional
 
 import jsonargparse
 from loguru import logger
@@ -20,7 +20,7 @@
 _FILL_OUT_TOKEN = " # <FILL-OUT>"
 
 
-def _get_placeholder_filter() -> list[semantic_model_pb2.NamedFilter]:
+def _get_placeholder_filter() -> List[semantic_model_pb2.NamedFilter]:
     return [
         semantic_model_pb2.NamedFilter(
             name=_PLACEHOLDER_COMMENT,
@@ -97,8 +97,18 @@ def _raw_table_to_semantic_context_table(
                 )
             )
         else:
-            raise ValueError(
-                f"Column datatype does not map to a known datatype. Input was = {col.column_type}. If this is a new datatype, please update the constants in snowflake_connector.py."
+            logger.warning(
+                f"Column datatype does not map to a known datatype. Input was = {col.column_type}. We are going to place as a Dimension for now."
+            )
+            dimensions.append(
+                semantic_model_pb2.Dimension(
+                    name=col.column_name,
+                    expr=col.column_name,
+                    data_type=col.column_type,
+                    sample_values=col.values,
+                    synonyms=[_PLACEHOLDER_COMMENT],
+                    description=_PLACEHOLDER_COMMENT,
+                )
             )
 
     return semantic_model_pb2.Table(
@@ -116,7 +126,7 @@ def _raw_table_to_semantic_context_table(
 
 
 def raw_schema_to_semantic_context(
-    physical_tables: list[str], snowflake_account: str, semantic_model_name: str
+    physical_tables: List[str], snowflake_account: str, semantic_model_name: str
 ) -> semantic_model_pb2.SemanticModel:
     """
     Converts a list of fully qualified Snowflake table names into a semantic model.
@@ -143,7 +153,7 @@ def raw_schema_to_semantic_context(
     )
     # For FQN tables, create a new snowflake connection per table in case the db/schema is different.
     table_objects = []
-    unique_database_schema: list[str] = []
+    unique_database_schema: List[str] = []
     for table in physical_tables:
         # Verify this is a valid FQN table. For now, we check that the table follows the following format.
         # {database}.{schema}.{table}
@@ -241,8 +251,8 @@ def _to_snake_case(s: str) -> str:
     return snake_case_str
 
 
-def generate_base_semantic_context_from_snowflake(
-    physical_tables: list[str],
+def generate_base_semantic_model_from_snowflake(
+    physical_tables: List[str],
     snowflake_account: str,
     semantic_model_name: str,
     output_yaml_path: Optional[str] = None,
@@ -313,7 +323,7 @@ def generate_base_semantic_context_from_snowflake(
 
     args = parser.parse_args()
 
-    generate_base_semantic_context_from_snowflake(
+    generate_base_semantic_model_from_snowflake(
         physical_tables=args.physical_tables,
         snowflake_account=args.snowflake_account,
         semantic_model_name=args.semantic_model_name,

semantic_model_generator/snowflake_utils/snowflake_connector.py

@@ -1,7 +1,7 @@
 import concurrent.futures
 from collections import defaultdict
 from contextlib import contextmanager
-from typing import Any, Generator, Optional, TypeVar
+from typing import Any, Dict, Generator, List, Optional, TypeVar
 
 import pandas as pd
 from loguru import logger
@@ -183,8 +183,8 @@ def _get_df(query: str) -> pd.DataFrame:
 
 def get_valid_schemas_tables_columns_df(
     conn: SnowflakeConnection,
-    table_schema: str | None = None,
-    table_names: list[str] | None = None,
+    table_schema: Optional[str] = None,
+    table_names: Optional[List[str]] = None,
 ) -> pd.DataFrame:
     if table_names and not table_schema:
         logger.warning(
@@ -257,7 +257,7 @@ def _get_warehouse(self) -> str:
             )
         return warehouse
 
-    def _get_host(self) -> str | None:
+    def _get_host(self) -> Optional[str]:
         host = env_vars.SNOWFLAKE_HOST
         if not host:
             logger.info(
@@ -328,7 +328,7 @@ def execute(
         self,
         connection: SnowflakeConnection,
         query: str,
-    ) -> dict[str, list[Any]]:
+    ) -> Dict[str, List[Any]]:
         try:
             if connection.warehouse is None:
                 warehouse = self._get_warehouse()

semantic_model_generator/snowflake_utils/utils.py

@@ -27,7 +27,7 @@ def create_connection_parameters(
     schema: Optional[str] = None,
     authenticator: Optional[str] = None,
 ) -> Dict[str, str]:
-    connection_parameters: dict[str, str] = dict(
+    connection_parameters: Dict[str, str] = dict(
         user=user, password=password, account=account
     )
     if role:
@@ -45,7 +45,7 @@ def create_connection_parameters(
     return connection_parameters
 
 
-def _connection(connection_parameters: dict[str, str]) -> SnowflakeConnection:
+def _connection(connection_parameters: Dict[str, str]) -> SnowflakeConnection:
     # https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-connect
     return connect(**connection_parameters)
 
@@ -56,7 +56,7 @@ def snowflake_connection(
     account: str,
     role: str,
     warehouse: str,
-    host: str | None = None,
+    host: Optional[str] = None,
 ) -> SnowflakeConnection:
     """
     Returns a Snowflake Connection to the specified account.

semantic_model_generator/tests/generate_model_test.py

@@ -1,4 +1,4 @@
-from unittest.mock import MagicMock, mock_open, patch
+from unittest.mock import MagicMock, call, mock_open, patch
 
 import pandas as pd
 import pytest
@@ -8,7 +8,7 @@
 from semantic_model_generator.data_processing.data_types import Column, Table
 from semantic_model_generator.generate_model import (
     _to_snake_case,
-    generate_base_semantic_context_from_snowflake,
+    generate_base_semantic_model_from_snowflake,
     raw_schema_to_semantic_context,
 )
 from semantic_model_generator.protos import semantic_model_pb2
@@ -69,6 +69,43 @@ def mock_snowflake_connection():
     comment=None,
 )
 
+
+_CONVERTED_TABLE_ALIAS_NEW_DTYPE = Table(
+    id_=0,
+    name="ALIAS",
+    columns=[
+        Column(
+            id_=0,
+            column_name="ZIP_CODE",
+            column_type="OBJECT",
+            values=None,
+            comment=None,
+        ),
+        Column(
+            id_=1,
+            column_name="AREA_CODE",
+            column_type="VARIANT",
+            values=None,
+            comment=None,
+        ),
+        Column(
+            id_=2,
+            column_name="BAD_ALIAS",
+            column_type="TIMESTAMP",
+            values=None,
+            comment=None,
+        ),
+        Column(
+            id_=3,
+            column_name="CBSA",
+            column_type="NUMBER",
+            values=None,
+            comment=None,
+        ),
+    ],
+    comment=None,
+)
+
 _CONVERTED_TABLE_ZIP_CODE = Table(
     id_=0,
     name="PRODUCTS",
@@ -140,6 +177,40 @@ def mock_dependencies(mock_snowflake_connection):
         yield
 
 
+@pytest.fixture
+def mock_dependencies_new_dtype(mock_snowflake_connection):
+    valid_schemas_tables_columns_df_alias = pd.DataFrame(
+        {
+            "TABLE_NAME": ["ALIAS"] * 4,
+            "COLUMN_NAME": ["ZIP_CODE", "AREA_CODE", "BAD_ALIAS", "CBSA"],
+            "DATA_TYPE": ["VARCHAR", "INTEGER", "DATETIME", "DECIMAL"],
+        }
+    )
+    valid_schemas_tables_columns_df_zip_code = pd.DataFrame(
+        {
+            "TABLE_NAME": ["PRODUCTS"],
+            "COLUMN_NAME": ["SKU"],
+            "DATA_TYPE": ["NUMBER"],
+        }
+    )
+    valid_schemas_tables_representations = [
+        valid_schemas_tables_columns_df_alias,
+        valid_schemas_tables_columns_df_zip_code,
+    ]
+    table_representations = [
+        _CONVERTED_TABLE_ALIAS_NEW_DTYPE,  # Value returned on the first call.
+    ]
+
+    with patch(
+        "semantic_model_generator.generate_model.get_valid_schemas_tables_columns_df",
+        side_effect=valid_schemas_tables_representations,
+    ), patch(
+        "semantic_model_generator.generate_model.get_table_representation",
+        side_effect=table_representations,
+    ):
+        yield
+
+
 def test_raw_schema_to_semantic_context(
     mock_dependencies, mock_snowflake_connection, mock_snowflake_connection_env
 ):
@@ -185,7 +256,7 @@ def test_generate_base_context_with_placeholder_comments(
     output_path = "output_model_path.yaml"
     semantic_model_name = "my awesome semantic model"
 
-    generate_base_semantic_context_from_snowflake(
+    generate_base_semantic_model_from_snowflake(
         physical_tables=physical_tables,
         snowflake_account=snowflake_account,
         output_yaml_path=output_path,
@@ -215,7 +286,7 @@ def test_generate_base_context_with_placeholder_comments_cross_database_cross_sc
     output_path = "output_model_path.yaml"
     semantic_model_name = "Another Incredible Semantic Model"
 
-    generate_base_semantic_context_from_snowflake(
+    generate_base_semantic_model_from_snowflake(
         physical_tables=physical_tables,
         snowflake_account=snowflake_account,
         output_yaml_path=output_path,
@@ -229,6 +300,47 @@ def test_generate_base_context_with_placeholder_comments_cross_database_cross_sc
     )
 
 
+@patch("semantic_model_generator.generate_model.logger")
+@patch("builtins.open", new_callable=mock_open)
+def test_generate_base_context_with_placeholder_comments_missing_datatype(
+    mock_file,
+    mock_logger,
+    mock_dependencies_new_dtype,
+    mock_snowflake_connection,
+    mock_snowflake_connection_env,
+):
+
+    physical_tables = ["test_db.schema_test.ALIAS"]
+    snowflake_account = "test_account"
+    output_path = "output_model_path.yaml"
+    semantic_model_name = "Another Incredible Semantic Model with new dtypes"
+
+    generate_base_semantic_model_from_snowflake(
+        physical_tables=physical_tables,
+        snowflake_account=snowflake_account,
+        output_yaml_path=output_path,
+        semantic_model_name=semantic_model_name,
+    )
+
+    expected_calls = [
+        call(
+            "Column datatype does not map to a known datatype. Input was = OBJECT. We are going to place as a Dimension for now."
+        ),
+        call(
+            "Column datatype does not map to a known datatype. Input was = VARIANT. We are going to place as a Dimension for now."
+        ),
+    ]
+
+    # Assert that all expected calls were made in the exact order
+    mock_logger.warning.assert_has_calls(expected_calls, any_order=False)
+
+    mock_file.assert_called_once_with(output_path, "w")
+    # Assert file save called with placeholder comments added along with sample values and cross-database
+    mock_file().write.assert_called_once_with(
+        "name: Another Incredible Semantic Model with new dtypes\ntables:\n  - name: ALIAS\n    description: '  ' # <FILL-OUT>\n    base_table:\n      database: test_db\n      schema: schema_test\n      table: ALIAS\n    filters:\n      - name: '  ' # <FILL-OUT>\n        synonyms:\n          - '  ' # <FILL-OUT>\n        description: '  ' # <FILL-OUT>\n        expr: '  ' # <FILL-OUT>\n    dimensions:\n      - name: ZIP_CODE\n        synonyms:\n          - '  ' # <FILL-OUT>\n        description: '  ' # <FILL-OUT>\n        expr: ZIP_CODE\n        data_type: OBJECT\n      - name: AREA_CODE\n        synonyms:\n          - '  ' # <FILL-OUT>\n        description: '  ' # <FILL-OUT>\n        expr: AREA_CODE\n        data_type: VARIANT\n    time_dimensions:\n      - name: BAD_ALIAS\n        synonyms:\n          - '  ' # <FILL-OUT>\n        description: '  ' # <FILL-OUT>\n        expr: BAD_ALIAS\n        data_type: TIMESTAMP\n    measures:\n      - name: CBSA\n        synonyms:\n          - '  ' # <FILL-OUT>\n        description: '  ' # <FILL-OUT>\n        expr: CBSA\n        data_type: NUMBER\n"
+    )
+
+
 def test_semantic_model_to_yaml() -> None:
     want_yaml = "name: transaction_ctx\ntables:\n  - name: transactions\n    description: A table containing data about financial transactions. Each row contains\n      details of a financial transaction.\n    base_table:\n      database: my_database\n      schema: my_schema\n      table: transactions\n    dimensions:\n      - name: transaction_id\n        description: A unique id for this transaction.\n        expr: transaction_id\n        data_type: BIGINT\n        unique: true\n    time_dimensions:\n      - name: initiation_date\n        description: Timestamp when the transaction was initiated. In UTC.\n        expr: initiation_date\n        data_type: DATETIME\n    measures:\n      - name: amount\n        description: The amount of this transaction.\n        expr: amount\n        data_type: DECIMAL\n        default_aggregation: sum\n"
     got = semantic_model_pb2.SemanticModel(