Rerun Croissant Health reports for Hugging Face and OpenML (#660)

mlcommons · Jun 5, 2024 · 0f95e04 · 0f95e04
1 parent d65b6ce
commit 0f95e04
Show file tree

Hide file tree

Showing 3 changed files with 51 additions and 805 deletions.
diff --git a/health/README.md b/health/README.md
@@ -17,7 +17,7 @@ pip install -r requirements.txt
 
 # Test the spider locally.
 # In huggingface.py you can uncomment the line in
-# `start_requests` to produce crawl fake data.
+# `list_datasets` to produce crawl fake data.
 scrapy crawl huggingface
 
 # When you're ready, the following commands launch a new job:

diff --git a/health/crawler/spiders/openml.py b/health/crawler/spiders/openml.py
@@ -20,4 +20,6 @@ def list_datasets(self):
 
     def get_url(self, dataset_id: str):
         """See base class."""
-        return f"https://openml1.win.tue.nl/dataset{dataset_id}/croissant.json"
+        return (
+            f"https://openml1.win.tue.nl/{dataset_id // 10000:04d}/{dataset_id:04d}/dataset_{dataset_id}_croissant.json"
+        )
diff --git a/health/visualizer/report_huggingface.ipynb b/health/visualizer/report_huggingface.ipynb