update wait; use dff.stats command

deeppavlov · Nov 15, 2023 · 73d8d69 · 73d8d69
1 parent 80d954b
commit 73d8d69
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 34 deletions.
diff --git a/makefile b/makefile
@@ -55,6 +55,7 @@ docker_up:
 
 wait_db: docker_up
 	while ! docker-compose exec psql pg_isready; do sleep 1; done > /dev/null
+	while ! docker-compose exec dashboard /bin/bash -c "curl localhost:8088/health | grep OK"; do sleep 1; done > /dev/null
 	while ! docker-compose exec mysql bash -c 'mysql -u $$MYSQL_USERNAME -p$$MYSQL_PASSWORD -e "select 1;"'; do sleep 1; done &> /dev/null
 .PHONY: wait_db
 

diff --git a/tests/stats/test_charts.py b/tests/stats/test_charts.py
@@ -10,7 +10,6 @@
     from requests import Session
     import omegaconf  # noqa: F401
     import tqdm  # noqa: F401
-    from dff.stats.__main__ import main
     from dff.stats.utils import get_superset_session
     from dff.stats.cli import DEFAULT_SUPERSET_URL
     from aiochclient import ChClient
@@ -31,6 +30,8 @@
 CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER")
 CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD")
 CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB")
+SUPERSET_USERNAME = os.getenv("SUPERSET_USERNAME")
+SUPERSET_PASSWORD = os.getenv("SUPERSET_PASSWORD")
 
 
 async def transitions_data_test(session: Session, headers: dict, base_url=DEFAULT_SUPERSET_URL):
@@ -89,20 +90,6 @@ async def numbered_data_test(session: Session, headers: dict, base_url=DEFAULT_S
     session.close()
 
 
-config_namespace = Namespace(
-    **{
-        "db.driver": "clickhousedb+connect",
-        "db.host": "clickhouse",
-        "db.port": "8123",
-        "db.name": "test",
-        "db.table": "otel_logs",
-        "host": "localhost",
-        "port": "8088",
-        "file": f"tutorials/{dot_path_to_addon}/example_config.yaml",
-    }
-)
-
-
 @pytest.mark.skipif(not SUPERSET_ACTIVE, reason="Superset server not active")
 @pytest.mark.skipif(not CLICKHOUSE_AVAILABLE, reason="Clickhouse unavailable.")
 @pytest.mark.skipif(not COLLECTOR_AVAILABLE, reason="OTLP collector unavailable.")
@@ -111,22 +98,14 @@ async def numbered_data_test(session: Session, headers: dict, base_url=DEFAULT_S
 )
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
-    ["args", "pipeline", "func"],
+    ["pipeline", "func"],
     [
-        (config_namespace, numbered_test_pipeline, numbered_data_test),
-        (config_namespace, transition_test_pipeline, transitions_data_test),
+        (numbered_test_pipeline, numbered_data_test),
+        (transition_test_pipeline, transitions_data_test),
     ],
 )
 @pytest.mark.docker
-async def test_charts(args, pipeline, func, otlp_log_exp_provider, otlp_trace_exp_provider):
-    args.__dict__.update(
-        {
-            "db.password": os.environ["CLICKHOUSE_PASSWORD"],
-            "username": os.environ["SUPERSET_USERNAME"],
-            "password": os.environ["SUPERSET_PASSWORD"],
-            "db.user": os.environ["CLICKHOUSE_USER"],
-        }
-    )
+async def test_charts(pipeline, func, otlp_log_exp_provider, otlp_trace_exp_provider):
     _, tracer_provider = otlp_trace_exp_provider
     _, logger_provider = otlp_log_exp_provider
 
@@ -140,11 +119,18 @@ async def test_charts(args, pipeline, func, otlp_log_exp_provider, otlp_trace_ex
     num_records = 0
 
     attempts = 0
-    while num_records == 0 and attempts < 10:
+    while num_records < 10 and attempts < 10:
         attempts += 1
         await asyncio.sleep(2)
         num_records = await ch_client.fetchval(f"SELECT COUNT (*) FROM {table}")
 
-    main(args)
-    session, headers = get_superset_session(args, DEFAULT_SUPERSET_URL)
+    os.system(
+        f"dff.stats tutorials/stats/example_config.yaml \
+            -U {SUPERSET_USERNAME} \
+            -P {SUPERSET_PASSWORD} \
+            -dP {CLICKHOUSE_PASSWORD}"
+    )
+    session, headers = get_superset_session(
+        Namespace(**{"username": SUPERSET_USERNAME, "password": SUPERSET_PASSWORD}), DEFAULT_SUPERSET_URL
+    )
     await func(session, headers)  # run with a test-specific function with equal signature
diff --git a/tutorials/stats/3_sample_data_provider.py b/tutorials/stats/3_sample_data_provider.py
@@ -20,22 +20,32 @@
     default_extractors,
     OtelInstrumentor,
 )
-from dff.utils.testing.toy_script import MULTIFLOW_SCRIPT, MULTIFLOW_REQUEST_OPTIONS
+from dff.utils.testing.toy_script import (
+    MULTIFLOW_SCRIPT,
+    MULTIFLOW_REQUEST_OPTIONS,
+)
 
 # %%
 # instrumentation code
-dff_instrumentor = OtelInstrumentor.from_url("grpc://localhost:4317", insecure=True)
+dff_instrumentor = OtelInstrumentor.from_url(
+    "grpc://localhost:4317", insecure=True
+)
 dff_instrumentor.instrument()
 
 
 def slot_processor_1(ctx: Context):
-    ctx.misc["slots"] = {**ctx.misc.get("slots", {}), "rating": random.randint(1, 10)}
+    ctx.misc["slots"] = {
+        **ctx.misc.get("slots", {}),
+        "rating": random.randint(1, 10),
+    }
 
 
 def slot_processor_2(ctx: Context):
     ctx.misc["slots"] = {
         **ctx.misc.get("slots", {}),
-        "current_topic": random.choice(["films", "games", "books", "smalltalk"]),
+        "current_topic": random.choice(
+            ["films", "games", "books", "smalltalk"]
+        ),
     }