diff --git a/src/prefect/tasks.py b/src/prefect/tasks.py
index d4e7d2fbce52..0d6610009847 100644
--- a/src/prefect/tasks.py
+++ b/src/prefect/tasks.py
@@ -474,6 +474,10 @@ def __init__(
 
         if callable(retry_delay_seconds):
             self.retry_delay_seconds = retry_delay_seconds(retries)
+        elif not isinstance(retry_delay_seconds, (list, int, float, type(None))):
+            raise TypeError(
+                f"Invalid `retry_delay_seconds` provided; must be an int, float, list or callable. Received type {type(retry_delay_seconds)}"
+            )
         else:
             self.retry_delay_seconds = retry_delay_seconds
 
diff --git a/tests/test_tasks.py b/tests/test_tasks.py
index 88cc1c4d086a..06d41abd6423 100644
--- a/tests/test_tasks.py
+++ b/tests/test_tasks.py
@@ -4172,6 +4172,21 @@ def my_flow():
 
 
 class TestTaskConstructorValidation:
+    async def test_task_cannot_configure_poorly_typed_retry_delay(self):
+        with pytest.raises(TypeError, match="Invalid"):
+
+            @task(retries=42, retry_delay_seconds=dict(x=4))
+            async def insanity():
+                raise RuntimeError("try again!")
+
+        with pytest.raises(TypeError, match="Invalid"):
+
+            @task(retries=42, retry_delay_seconds=2)
+            async def sanity():
+                raise RuntimeError("try again!")
+
+            more_insanity = sanity.with_options(retry_delay_seconds=dict(x=4))  # noqa: F841
+
     async def test_task_cannot_configure_too_many_custom_retry_delays(self):
         with pytest.raises(ValueError, match="Can not configure more"):