diff --git a/altair/utils/core.py b/altair/utils/core.py
index 9fecc4aa1..f2ce1d4bc 100644
--- a/altair/utils/core.py
+++ b/altair/utils/core.py
@@ -586,7 +586,15 @@ def parse_shorthand(
                 unescaped_field = attrs["field"].replace("\\", "")
                 if unescaped_field in dfi.column_names():
                     column = dfi.get_column_by_name(unescaped_field)
-                    attrs["type"] = infer_vegalite_type_for_dfi_column(column)
+                    try:
+                        attrs["type"] = infer_vegalite_type_for_dfi_column(column)
+                    except NotImplementedError:
+                        # Fall back to pandas-based inference
+                        if isinstance(data, pd.DataFrame):
+                            attrs["type"] = infer_vegalite_type(data[unescaped_field])
+                        else:
+                            raise
+
                     if isinstance(attrs["type"], tuple):
                         attrs["sort"] = attrs["type"][1]
                         attrs["type"] = attrs["type"][0]
diff --git a/tests/utils/test_core.py b/tests/utils/test_core.py
index 62203524d..75db18769 100644
--- a/tests/utils/test_core.py
+++ b/tests/utils/test_core.py
@@ -134,7 +134,8 @@ def check(s, **kwargs):
     )
 
 
-def test_parse_shorthand_with_data():
+@pytest.mark.parametrize("object_dtype", [False, True])
+def test_parse_shorthand_with_data(object_dtype):
     def check(s, data, **kwargs):
         assert parse_shorthand(s, data) == kwargs
 
@@ -147,6 +148,9 @@ def check(s, data, **kwargs):
         }
     )
 
+    if object_dtype:
+        data = data.astype("object")
+
     check("x", data, field="x", type="quantitative")
     check("y", data, field="y", type="nominal")
     check("z", data, field="z", type="temporal")