From 8159569abf78db5fff35d38a04b8a69da86da65e Mon Sep 17 00:00:00 2001 From: atheo89 Date: Mon, 23 Oct 2023 11:09:39 +0200 Subject: [PATCH] Fine tunings and logical fixes --- Makefile | 8 +------ .../ubi8-python-3.8/test/test_notebook.ipynb | 21 ++++++++++--------- .../ubi9-python-3.9/test/test_notebook.ipynb | 20 +++++++++--------- 3 files changed, 22 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 1e6db94c3..c84b7e42a 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ CONTAINER_ENGINE ?= podman IMAGE_REGISTRY ?= quay.io/opendatahub/workbench-images -RELEASE ?= 2023a +RELEASE ?= 2023b DATE ?= $(shell date +'%Y%m%d') IMAGE_TAG ?= $(RELEASE)_$(DATE) KUBECTL_BIN ?= bin/kubectl @@ -350,12 +350,6 @@ test-%: bin/kubectl $(call test_with_papermill,minimal,ubi8,python-3.8) \ elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "datascience-ubi8"; then \ $(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \ - elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "pytorch-ubi8"; then \ - $(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \ - $(call test_with_papermill,pytorch,ubi8,python-3.8) \ - elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "tensorflow-ubi8"; then \ - $(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \ - $(call test_with_papermill,tensorflow,ubi8,python-3.8) \ elif echo "$(FULL_NOTEBOOK_NAME)" | grep -q "trustyai-ubi8"; then \ $(MAKE) validate-ubi8-datascience -e FULL_NOTEBOOK_NAME=$(FULL_NOTEBOOK_NAME); \ $(call test_with_papermill,trustyai,ubi8,python-3.8) \ diff --git a/jupyter/trustyai/ubi8-python-3.8/test/test_notebook.ipynb b/jupyter/trustyai/ubi8-python-3.8/test/test_notebook.ipynb index ed4c4d985..f4201ce7b 100644 --- a/jupyter/trustyai/ubi8-python-3.8/test/test_notebook.ipynb +++ b/jupyter/trustyai/ubi8-python-3.8/test/test_notebook.ipynb @@ -13,7 +13,6 @@ "from platform import python_version\n", "from trustyai.metrics.fairness.group import statistical_parity_difference\n", "from trustyai.model import output\n", - "from trustyai.metrics.fairness.group import disparate_impact_ratio\n", "\n", "class TestTrustyaiNotebook(unittest.TestCase):\n", "\n", @@ -28,7 +27,7 @@ " self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n", " \n", " def test_fairnessmetrics(self):\n", - " url_unbiased = \"https://raw.githubusercontent.com/dibryant/notebooks/trustyai/jupyter/trustyai/ubi8-python-3.8/test/income-unbiased.csv\"\n", + " url_unbiased = \"https://raw.githubusercontent.com/opendatahub-io/notebooks/main/jupyter/trustyai/ubi8-python-3.8/test/income-unbiased.csv\"\n", " nobias = pd.read_csv(url_unbiased, index_col=False)\n", " \n", " nobias = pd.read_csv(url_unbiased, index_col=False)\n", @@ -43,23 +42,25 @@ " favorable=[favorable])\n", " self.assertTrue(score >= 0.0036255104824703954) \n", " print(\"On the test_fairness_metrics test case the statistical_parity_difference score for this dataset is between the threshold [-0.1,0.1], which classifies the model as reasonably fair.\")\n", - " \n", - " def test_datafairness(self):\n", - " url_biased = \"https://raw.githubusercontent.com/dibryant/notebooks/trustyai/jupyter/trustyai/ubi8-python-3.8/test/income-biased.csv\"\n", - " bias = pd.read_csv(url_biased, index_col=False)\n", " \n", + " def test_datafairness(self):\n", + " url_biased = \"https://raw.githubusercontent.com/opendatahub-io/notebooks/main/jupyter/trustyai/ubi8-python-3.8/test/income-biased.csv\"\n", " bias = pd.read_csv(url_biased, index_col=False)\n", - " bias.groupby(['gender', 'income'])['income'].count()\n", - " bias.groupby(['gender', 'income'])['income'].count().unstack().plot.bar()\n", " \n", + " # Perform the data manipulations \n", + " grouped_counts = bias.groupby(['gender', 'income'])['income'].count()\n", + " unstacked_counts = bias.groupby(['gender', 'income'])['income'].count().unstack()\n", + " unstacked_counts.plot.bar()\n", + "\n", " bias_privileged = bias[bias.gender == 1]\n", " bias_unprivileged = bias[bias.gender == 0]\n", " favorable = output(\"income\", dtype=\"number\", value=1)\n", " score = statistical_parity_difference(privileged=bias_privileged,\n", " unprivileged=bias_unprivileged,\n", " favorable=[favorable])\n", - " self.assertTrue(score >= -0.15670061634672994) \n", - " print(\"On the test_fairness_metrics test case the statistical_parity_difference score for this dataset is between the threshold [-0.1,0.1], which classifies the model as reasonably fair.\")\n", + " self.assertTrue(score <= -0.15670061634672994)\n", + " print(\"On the test_bias_metrics test case the statistical_parity_difference score for this dataset, as expected, is outside the threshold [-0.1,0.1], which classifies the model as unfair.\")\n", + " \n", "\n", "suite = unittest.TestLoader().loadTestsFromTestCase(TestTrustyaiNotebook)\n", "unittest.TextTestRunner().run(suite)" diff --git a/jupyter/trustyai/ubi9-python-3.9/test/test_notebook.ipynb b/jupyter/trustyai/ubi9-python-3.9/test/test_notebook.ipynb index ba43ae59b..7d7ea68ee 100644 --- a/jupyter/trustyai/ubi9-python-3.9/test/test_notebook.ipynb +++ b/jupyter/trustyai/ubi9-python-3.9/test/test_notebook.ipynb @@ -13,7 +13,6 @@ "from platform import python_version\n", "from trustyai.metrics.fairness.group import statistical_parity_difference\n", "from trustyai.model import output\n", - "from trustyai.metrics.fairness.group import disparate_impact_ratio\n", "\n", "class TestTrustyaiNotebook(unittest.TestCase):\n", "\n", @@ -28,7 +27,7 @@ " self.assertEqual(actual_major_minor, expected_major_minor, \"incorrect version\")\n", "\n", " def test_fairnessmetrics(self):\n", - " url_unbiased = \"https://raw.githubusercontent.com/dibryant/notebooks/trustyai/jupyter/trustyai/ubi9-python-3.9/test/income-unbiased.csv\"\n", + " url_unbiased = \"https://raw.githubusercontent.com/opendatahub-io/notebooks/main/jupyter/trustyai/ubi9-python-3.9/test/income-unbiased.csv\"\n", " nobias = pd.read_csv(url_unbiased, index_col=False)\n", " \n", " nobias = pd.read_csv(url_unbiased, index_col=False)\n", @@ -45,24 +44,25 @@ " print(\"On the test_fairness_metrics test case the statistical_parity_difference score for this dataset is between the threshold [-0.1,0.1], which classifies the model as reasonably fair.\")\n", " \n", " def test_datafairness(self):\n", - " url_biased = \"https://raw.githubusercontent.com/dibryant/notebooks/trustyai/jupyter/trustyai/ubi9-python-3.9/test/income-biased.csv\"\n", + " url_biased = \"https://raw.githubusercontent.com/opendatahub-io/notebooks/main/jupyter/trustyai/ubi9-python-3.9/test/income-biased.csv\"\n", " bias = pd.read_csv(url_biased, index_col=False)\n", " \n", - " bias = pd.read_csv(url_biased, index_col=False)\n", - " bias.groupby(['gender', 'income'])['income'].count()\n", - " bias.groupby(['gender', 'income'])['income'].count().unstack().plot.bar()\n", - " \n", + " # Perform the data manipulations \n", + " grouped_counts = bias.groupby(['gender', 'income'])['income'].count()\n", + " unstacked_counts = bias.groupby(['gender', 'income'])['income'].count().unstack()\n", + " unstacked_counts.plot.bar()\n", + "\n", " bias_privileged = bias[bias.gender == 1]\n", " bias_unprivileged = bias[bias.gender == 0]\n", " favorable = output(\"income\", dtype=\"number\", value=1)\n", " score = statistical_parity_difference(privileged=bias_privileged,\n", " unprivileged=bias_unprivileged,\n", " favorable=[favorable])\n", - " self.assertTrue(score >= -0.15670061634672994) \n", - " print(\"On the test_fairness_metrics test case the statistical_parity_difference score for this dataset is between the threshold [-0.1,0.1], which classifies the model as reasonably fair.\")\n", + " self.assertTrue(score <= -0.15670061634672994)\n", + " print(\"On the test_bias_metrics test case the statistical_parity_difference score for this dataset, as expected, is outside the threshold [-0.1,0.1], which classifies the model as unfair.\")\n", " \n", "suite = unittest.TestLoader().loadTestsFromTestCase(TestTrustyaiNotebook)\n", - "unittest.TextTestRunner().run(suite)\n" + "unittest.TextTestRunner().run(suite)" ] } ],