diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 000000000..acc54c487
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,18 @@
+[pytest]
+asyncio_mode = auto
+addopts =
+  -v
+filterwarnings =
+  error
+; botocore has this warning that is reported by them to be irrelevant
+  ignore:.*urllib3.contrib.pyopenssl.*:DeprecationWarning:botocore.*
+; latest main of aioresponses does not have this problem, but current package uses deprecated pkg_resources API
+  ignore:.*pkg_resources.*:DeprecationWarning
+; SQLAlchemy uses deprecated APIs internally
+  ignore:.*dbapi().*:DeprecationWarning
+; aiogoogle inherits on top of AioHttpSession, which is not recommended by aiohttp
+  ignore:Inheritance class AiohttpSession from ClientSession is discouraged:DeprecationWarning
+; aiogoogle inherits on top of RetryableAioHttpSession, which is not recommended by aiohttp
+  ignore:Inheritance class RetryableAiohttpSession from ClientSession is discouraged:DeprecationWarning
+; pytest may generate its own warnings in some situations, such as improper usage or deprecated features.
+  ignore::pytest.PytestUnraisableExceptionWarning
diff --git a/setup.cfg b/setup.cfg
deleted file mode 100644
index 6a56fd2e1..000000000
--- a/setup.cfg
+++ /dev/null
@@ -1,2 +0,0 @@
-[tools:pytest]
-asyncio_mode = auto
diff --git a/tests/sources/test_slack.py b/tests/sources/test_slack.py
index 1b2b9e744..9fee6a08d 100644
--- a/tests/sources/test_slack.py
+++ b/tests/sources/test_slack.py
@@ -192,12 +192,17 @@ async def test_slack_data_source_get_docs(slack_data_source, mock_responses):
     channels_response = [{"id": "1", "name": "channel1", "is_member": True}]
     messages_response = [{"text": "message1", "type": "message", "ts": 123456}]
 
-    slack_client = AsyncMock()
-    slack_client.list_users = AsyncIterator(users_response)
-    slack_client.list_channels = AsyncIterator(channels_response)
-    slack_client.list_messages = AsyncIterator(messages_response)
-    slack_client.close = AsyncMock()
-    slack_data_source.slack_client = slack_client
+    # A bit weird, but Slack connector actually inits client in its __init__
+    # So we need to close it before redefining
+    original_client = slack_data_source.slack_client
+    await original_client.close()
+
+    mock_client = AsyncMock()
+    mock_client.list_users = AsyncIterator(users_response)
+    mock_client.list_channels = AsyncIterator(channels_response)
+    mock_client.list_messages = AsyncIterator(messages_response)
+    mock_client.close = AsyncMock()
+    slack_data_source.slack_client = mock_client
 
     docs = []
     async for doc, _ in slack_data_source.get_docs():