[QA] Score tests improvements

.github/workflows/test-score-vizro-ai.yml

@@ -5,7 +5,13 @@ defaults:
     working-directory: vizro-ai
 
 on:
+  schedule:
+    - cron: "30 10 * * 1" # run every Monday at 10:30 UTC
   workflow_dispatch:
+  #temporary for development
+  pull_request:
+    branches:
+      - main
 
 env:
   PYTHONUNBUFFERED: 1
@@ -20,17 +26,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - python-version: "3.9"
-            hatch-env: all.py3.9
-          - python-version: "3.10"
-            hatch-env: all.py3.10
-          - python-version: "3.11"
-            hatch-env: all.py3.11
           - python-version: "3.12"
             hatch-env: all.py3.12
-          - python-version: "3.9"
-            hatch-env: lower-bounds
-            label: lower bounds
 
     steps:
       - uses: actions/checkout@v4
@@ -46,17 +43,8 @@ jobs:
       fail-fast: false
       matrix:
         config:
-          - python-version: "3.9"
-            hatch-env: all.py3.9
-          - python-version: "3.10"
-            hatch-env: all.py3.10
-          - python-version: "3.11"
-            hatch-env: all.py3.11
           - python-version: "3.12"
             hatch-env: all.py3.12
-          - python-version: "3.9"
-            hatch-env: lower-bounds
-            label: lower bounds
 
     steps:
       - uses: actions/checkout@v4
@@ -77,6 +65,8 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }}
           VIZRO_TYPE: pypi
           BRANCH: ${{ github.head_ref }}
           PYTHON_VERSION: ${{ matrix.config.python-version }}
@@ -88,6 +78,8 @@ jobs:
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+          ANTHROPIC_BASE_URL: ${{ secrets.ANTHROPIC_BASE_URL }}
           VIZRO_TYPE: local
           BRANCH: ${{ github.head_ref }}
           PYTHON_VERSION: ${{ matrix.config.python-version }}
@@ -131,7 +123,7 @@ jobs:
       - name: Create one csv report
         run: |
           cd /home/runner/work/vizro/vizro/
-          head -n 1 Report-3.11-/report_model_gpt-4o-mini_pypi.csv > report-aggregated-${{ steps.date.outputs.date }}.csv && tail -n+2 -q */*.csv >> report-aggregated-${{ steps.date.outputs.date }}.csv
+          head -n 1 Report-3.12-/report_model_gpt-4o-mini_pypi.csv > report-aggregated-${{ steps.date.outputs.date }}.csv && tail -n+2 -q */*.csv >> report-aggregated-${{ steps.date.outputs.date }}.csv
           gawk -F, -i inplace 'FNR>1 {$1="${{ steps.date.outputs.date }}"} {print}' OFS=, report-aggregated-${{ steps.date.outputs.date }}.csv
 
       - name: Report artifacts

vizro-ai/changelog.d/20241224_135602_alexey_snigir_score_tests_improvements.md

@@ -0,0 +1,48 @@
+<!--
+A new scriv changelog fragment.
+
+Uncomment the section that is right (remove the HTML comment wrapper).
+-->
+
+<!--
+### Highlights ✨
+
+- A bullet item for the Highlights ✨ category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Removed
+
+- A bullet item for the Removed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Added
+
+- A bullet item for the Added category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Changed
+
+- A bullet item for the Changed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Deprecated
+
+- A bullet item for the Deprecated category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Fixed
+
+- A bullet item for the Fixed category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->
+<!--
+### Security
+
+- A bullet item for the Security category with a link to the relevant PR at the end of your entry, e.g. Enable feature XXX. ([#1](https://github.com/mckinsey/vizro/pull/1))
+
+-->

vizro-ai/hatch.toml

@@ -49,7 +49,7 @@ prep-release = [
 pypath = "hatch run python -c 'import sys; print(sys.executable)'"
 test = "pytest tests {args}"
 test-integration = "pytest -vs --reruns 1 tests/integration --headless {args}"
-test-score = "pytest -vs --reruns 1 tests/score --headless {args}"
+test-score = "pytest -vs tests/score --headless {args}"
 test-unit = "pytest tests/unit {args}"
 test-unit-coverage = [
   "coverage run -m pytest tests/unit {args}",

vizro-ai/tests/score/prompts.py

@@ -0,0 +1,105 @@
+easy_prompt = """
+I need a page with 1 table.
+The table shows the tech companies stock data.
+
+I need a second page showing 2 cards and one chart.
+The first card says 'The Gapminder dataset provides historical data on countries' development indicators.'
+The chart is an scatter plot showing life expectancy vs. GDP per capita by country.
+Life expectancy on the y axis, GDP per capita on the x axis, and colored by continent.
+The second card says 'Data spans from 1952 to 2007 across various countries.'
+The layout uses a grid of 3 columns and 2 rows.
+
+Row 1: The first row has three columns:
+The first column is occupied by the first card.
+The second and third columns are spanned by the chart.
+
+Row 2: The second row mirrors the layout of the first row with respect to chart,
+but the first column is occupied by the second card.
+
+Add a filter to filter the scatter plot by continent.
+Add a second filter to filter the chart by year.
+"""
+
+medium_prompt = """
+<Page 1>
+I need a page with 1 table and 1 line chart.
+The chart shows the stock price trends of GOOG and AAPL.
+The table shows the stock prices data details.
+
+<Page 2>
+I need a second page showing 3 cards and 4 charts.
+The cards says 'The Gapminder dataset provides historical data on countries' development indicators.'
+The charts are the scatter plots showing GDP per capita vs. life expectancy.
+GDP per capita on the x axis, life expectancy on the y axis, and colored by continent.
+Layout the cards on the left and the chart on the right.
+Add a filter to filter the scatter plots by continent.
+Add a second filter to filter the charts by year.
+
+<Page 3>
+This page displays the tips dataset. use four different charts to show data
+distributions. one chart should be a bar chart. the other should be a scatter plot.
+next chart should be a line chart. last one should be an area plot.
+first and second charts are on the left and the third and fourth charts are on the right.
+Add a filter to filter data in every plot by smoker.
+
+<Page 4>
+Create 3 cards on this page:
+1. The first card on top says "This page combines data from various sources
+ including tips, stock prices, and global indicators."
+2. The second card says "Insights from Gapminder dataset."
+3. The third card says "Stock price trends over time."
+
+Layout these 3 cards in this way:
+create a grid with 3 columns and 2 rows.
+Row 1: The first row has three columns:
+- The first column is empty.
+- The second and third columns span the area for card 1.
+
+Row 2: The second row also has three columns:
+- The first column is empty.
+- The second column is occupied by the area for card 2.
+- The third column is occupied by the area for card 3.
+    """
+
+
+complex_prompt = """
+<Page 1>
+I need a page with 1 table and 3 line charts.
+The chart shows the stock price trends of GOOG and AAPL.
+The table shows the stock prices data details.
+Add 3 filters to filter the line chart by companies.
+
+<Page 2>
+I need a second page showing 1 card and 1 chart.
+The card says 'The Gapminder dataset provides historical data on countries' development indicators.'
+The chart is a scatter plot showing GDP per capita vs. life expectancy.
+GDP per capita on the x axis, life expectancy on the y axis, and colored by continent.
+Layout the card on the left and the chart on the right. The card takes 1/3 of the whole space on the left.
+The chart takes 2/3 of the whole space and is on the right.
+Add a filter to filter the scatter plot by continent.
+Add a second filter to filter the chart by year.
+
+<Page 3>
+This page displays the tips dataset. use two different charts to show data
+distributions. one chart should be a bar chart and the other should be a scatter plot.
+first chart is on the left and the second chart is on the right.
+Add a filter to filter data in the scatter plot by smoker.
+
+<Page 4>
+Create 3 cards on this page:
+1. The first card on top says "This page combines data from various sources
+ including tips, stock prices, and global indicators."
+2. The second card says "Insights from Gapminder dataset."
+3. The third card says "Stock price trends over time."
+
+Layout these 3 cards in this way:
+create a grid with 3 columns and 2 rows.
+Row 1: The first row has three columns:
+- The first column is empty.
+- The second and third columns span the area for card 1.
+
+Row 2: The second row also has three columns:
+- The first column is empty.
+- The second column is occupied by the area for card 2.
+- The third column is occupied by the area for card 3.
+    """

vizro-ai/tests/score/pytest.ini

@@ -2,3 +2,9 @@
 markers =
     easy_dashboard: mark test with easy prompt for dashboard creation.
     medium_dashboard: mark test with medium prompt for dashboard creation.
+    complex_dashboard: mark test with complex prompt for dashboard creation.
+
+filterwarnings =
+    ignore::UserWarning
+    # Ignore deprecation warning until this is solved: https://github.com/plotly/dash/issues/2590:
+    ignore:HTTPResponse.getheader():DeprecationWarning