Use Docker postgres as test db, and add top sites to analytics

api/main.py

@@ -11,51 +11,10 @@
 
 EVENTS_TABLE = "keyevents"
 
-@app.get("/api/events/{userId}/statistics")
-def get_events_statistics(request: Request, userId: str, interval: str = '1 hour', offset_count: int = 0, db = Depends(get_db)):
-  #No SQL injection possible as the type is enforced... actually yes, as interval string can be bad #TODO fix urgently
-  offset = ' '.join([f"- INTERVAL '{interval}'"] * offset_count) 
-  data = db.query(f"""
-    with user_keyevents as (
-      select * from keyevents where user_id=%s
-    ), last_hour_events as (
-        select 
-            * 
-        from user_keyevents 
-        where 
-          record_time > NOW() - INTERVAL %s {offset} and
-          record_time <= NOW() {offset}
-    ), word_count as (
-        select 
-            count(*) as word_count
-        from last_hour_events 
-        where is_end_of_word is true or is_end_of_line is true 
-    ),
-    returns_count as (
-        select
-            count(*) as error_count
-        from last_hour_events
-        where is_return is true
-    ),
-    total_count as (
-        select
-            count(*) as total_count
-        from last_hour_events
-    )
-    select 
-        total_count,
-        error_count,
-        word_count
-    from word_count, returns_count, total_count
-  """, (userId, interval))
-  return {
-    'data': data
-  }
-
 #TODO add weeks offset
 @app.get("/api/events/{userId}/analytics/time-of-day")
 def get_events_statistics(request: Request, userId: str, interval: str = '1 hour', bucket_width='15 minutes', db = Depends(get_db)):
-  data = db.query(f"""
+  data = db.query("""
     with user_keyevents as (
       select * from keyevents
       where 
@@ -102,7 +61,7 @@ def get_events_statistics(request: Request, userId: str, interval: str = '1 hour
 
 @app.get("/api/events/{userId}/analytics/day-of-week")
 def get_events_statistics(request: Request, userId: str, interval='1 month', db = Depends(get_db)):
-  data = db.query(f"""
+  data = db.query("""
     with user_keyevents as (
       select * from keyevents where user_id=%s  and record_time > now() - interval %s
     )
@@ -117,27 +76,7 @@ def get_events_statistics(request: Request, userId: str, interval='1 month', db
     'data': data
   }
 
-@app.get("/api/events/{userId}/analytics/top-sites")
-def get_events_statistics(request: Request, userId: str, interval='1 month', db = Depends(get_db)):
-  data = db.query(f"""
-    with keyevents as (
-        select 
-            (regexp_matches(source_url, '^(?:https?:\/\/)?(?:[^@\/\n]+@)?([^:\/\n]+)', 'g'))[1] as url,
-            *
-        from keyevents 
-        where 
-          user_id=%s and 
-          record_time > now() - interval %s
-    )
-    select 
-        url, count(*) 
-    from keyevents
-    group by url
-    order by count(*) desc
-  """, (userId,interval))
-  return {
-    'data': data
-  }
+
 
 @app.get("/api/version")
 def get_version():

api/routers/analytics.py

@@ -11,4 +11,26 @@ def get_typing_speed_current(request: Request, x_user_id: str = Header(default=N
     "stats": pd.read_sql(f"""
       select * from typing_speed_current where user_id = '{x_user_id}'
     """, db.bind).to_json(orient="records")
+  }
+
+@router.get("/api/stats/top-sites")
+def get_events_statistics(request: Request, x_user_id: str = Header(default=None), interval='1 month', db = Depends(get_db)):
+  data = pd.read_sql("""
+    with keyevents as (
+        select 
+            (regexp_matches(source_url, '^(?:https?:\/\/)?(?:[^@\/\n]+@)?([^:\/\n]+)', 'g'))[1] as url,
+            *
+        from typing_events 
+        where 
+          user_id=%(user_id)s and 
+          record_time > now() - interval %(interval)s
+    )
+    select 
+        url, count(*) 
+    from keyevents
+    group by url
+    order by count(*) desc
+  """, db.bind, params={"user_id": x_user_id, "interval": interval}).to_dict(orient="records")
+  return {
+    'data': data
   }

api/tests/conftest.py

@@ -0,0 +1,48 @@
+# conftest.py
+import pytest
+import subprocess
+import time
+from sqlalchemy import create_engine, exc
+from models import Base
+import pandas as pd
+import os
+
+os.environ["JWT_SECRET_KEY"] = "oth42obgtwknbwl2ngl2np2non24ongtonh2gn2ngpwn"
+
+@pytest.fixture(scope='session')
+def postgres_db():
+    # Start a new PostgreSQL instance in Docker
+    container_id = subprocess.getoutput(
+        "docker run -d -e POSTGRES_PASSWORD=secret -e POSTGRES_DB=test_db -p 5432:5432 postgres:13"
+    )
+    print(f"Started PostgreSQL container: {container_id}")
+
+    # Initialize a connection engine
+    engine = create_engine("postgresql://postgres:secret@localhost:5432/test_db")
+
+    # Poll database to check if it's ready
+    max_retries = 10
+    retry_count = 0
+    while retry_count < max_retries:
+        try:
+            # Attempt to connect to the database
+            pd.read_sql("select 1", engine)
+            print("Database is ready.")
+            break  # Database is ready, break out of loop
+        except exc.SQLAlchemyError:
+            print("Database not ready yet. Retrying...")
+            retry_count += 1
+            time.sleep(2)  # Wait for 2 seconds before retrying
+
+    if retry_count == max_retries:
+        print("Max retries reached. Exiting.")
+        raise Exception("Could not connect to database.")
+
+    Base.metadata.create_all(bind=engine)
+
+    yield engine  # Provides the database connection to the test
+
+    # Teardown: Stop and remove the container
+    subprocess.run(["docker", "container", "stop", container_id])
+    subprocess.run(["docker", "container", "rm", container_id])
+    print(f"Removed PostgreSQL container: {container_id}")

api/tests/test_analytics.py

@@ -0,0 +1,52 @@
+from tests.utils import get_docker_engine
+from main import app
+from fastapi.testclient import TestClient
+from dependencies import get_db
+from datetime import datetime
+from uuid import uuid4
+from random import randint
+client = TestClient(app)
+
+app.dependency_overrides[get_db] = get_docker_engine
+
+def get_test_user():
+    res = client.post("/api/signup", data={"username": f"alice{randint(1000, 2000)}@topsites.com", "password": "1234"})
+    assert res.status_code == 200
+    headers = {"Authorization": "Bearer " + res.json()["access_token"] }
+    return headers
+
+def test_top_sites_empty(postgres_db):
+    headers = get_test_user()    
+    res = client.get("/api/stats/top-sites", 
+        headers=headers
+    )
+    assert res.status_code == 200
+    assert len(res.json()["data"]) == 0, res.json()
+
+def generate_event(url):
+    return {
+        "record_time": str(datetime.now()),
+        "session_id": uuid4().hex,
+        "source_url": url,
+        "is_end_of_word": False,
+        "is_end_of_line": False,
+        "is_return": False
+    }
+
+def test_top_sites_ranking(postgres_db):
+    headers = get_test_user()
+    test_request = [generate_event("google.com") for _ in range(3)] + [generate_event("facebook.com")]
+    res = client.post(
+        "/api/events", 
+        json={"events": test_request},
+        headers=headers
+    )
+    assert res.status_code
+    res = client.get("/api/stats/top-sites",
+        headers=headers
+    )
+    assert res.status_code == 200
+    stats = res.json()["data"]
+    assert len(stats) == 2, stats
+    assert stats[0]["url"] == "google.com"
+    assert stats[1]["url"] == "facebook.com"

api/tests/test_auth.py

@@ -1,4 +1,4 @@
-from tests.utils import override_get_db
+from tests.utils import get_docker_engine
 from main import app
 from dependencies import get_db
 from fastapi.testclient import TestClient
@@ -10,11 +10,11 @@
 from models import Base, User
 from pytest import raises
 
-app.dependency_overrides[get_db] = override_get_db
+app.dependency_overrides[get_db] = get_docker_engine
 client = TestClient(app)
 client.raise_server_exceptions = False
 
-def test_user_signup():
+def test_user_signup(postgres_db):
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 200
   response = res.json()
@@ -23,23 +23,23 @@ def test_user_signup():
   res = client.post("/api/login", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 200
 
-def test_login_unauthenticated_user():
+def test_login_unauthenticated_user(postgres_db):
   res = client.post("/api/login", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 400
 
-def test_signup_existing_user():
+def test_signup_existing_user(postgres_db):
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 200
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 400
 
-def test_wrong_password():
+def test_wrong_password(postgres_db):
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 200
   res = client.post("/api/login", data={"username": "[email protected]", "password": "abcd"})
   assert res.status_code == 400
 
-def test_bearer_token():
+def test_bearer_token(postgres_db):
   #with raises(HTTPException) as e:
   #  assert client.get("/api/version").status_code == 400
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
@@ -63,7 +63,7 @@ def test_tokens_eventually_expire():
 def test_tokens_contain_relevant_claims():
   pass
 
-def test_token_refresh():
+def test_token_refresh(postgres_db):
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 200
   credentials1 = res.json()
@@ -75,7 +75,7 @@ def test_token_refresh():
   assert "access_token" in credentials2
   assert "refresh_token" in credentials2
 
-def test_refresh_token_with_access_token_fails():
+def test_refresh_token_with_access_token_fails(postgres_db):
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   assert res.status_code == 200
   credentials1 = res.json()

api/tests/test_events.py

@@ -1,4 +1,4 @@
-from tests.utils import override_get_db
+from tests.utils import get_docker_engine
 from uuid import uuid4
 from datetime import datetime
 from main import app
@@ -7,10 +7,10 @@
 
 client = TestClient(app)
 
-app.dependency_overrides[get_db] = override_get_db
+app.dependency_overrides[get_db] = get_docker_engine
 
 
-def test_insert_single_event():
+def test_insert_single_event(postgres_db):
   res = client.post("/api/signup", data={"username": "[email protected]", "password": "1234"})
   token = res.json()["access_token"]
   test_request = [

api/tests/utils.py

@@ -1,25 +1,14 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
-from sqlalchemy.pool import StaticPool
-from dependencies import get_db, engine
-from models import Base, User
-import os
+from dependencies import engine
 
-os.environ["JWT_SECRET_KEY"] = "oth42obgtwknbwl2ngl2np2non24ongtonh2gn2ngpwn"
 
-SQLALCHEMY_DATABASE_URL = "sqlite://"
+engine = create_engine("postgresql://postgres:secret@localhost:5432/test_db")
+Session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
 
-engine = create_engine(
-    SQLALCHEMY_DATABASE_URL,
-    connect_args={"check_same_thread": False},
-    poolclass=StaticPool,
-)
-Base.metadata.create_all(bind=engine)
-TestingSessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
-
-def override_get_db():
+def get_docker_engine():
   try:
-    db = TestingSessionLocal()
+    db = Session()
     yield db
   finally:
     db.close()