fix: misc

.gitignore

@@ -153,3 +153,4 @@ cython_debug/
 
 
 **.DS_Store
+BLEURT-20/

README.md

@@ -20,9 +20,8 @@ This repository contains utilities to download and work with the dataset in Pyth
 evaluation metrics presented in our paper. Alternatively, you can download the dev and test sets in JSON format and
 generate completions to submit to us for evaluation.
 
-## Leaderboards
-
-TODO: move to website
+To view the leaderboards and more documentation about how to use this dataset, check out our website
+at <https://fanoutqa.com>!
 
 ## Requirements and Installation
 
@@ -130,7 +129,8 @@ To evaluate a model's generation, first ensure that you have installed all the e
 
 To use the GPT-as-judge metric, you will need to provide your OpenAI API key. We intentionally do not read
 the `OPENAI_API_KEY` environment variable by default to prevent accidentally spending money; you must set the
-`FANOUTQA_OPENAI_API_KEY` environment variable instead.
+`FANOUTQA_OPENAI_API_KEY` environment variable instead. You can use `export FANOUTQA_OPENAI_API_KEY=$OPENAI_API_KEY` to
+quickly copy it over.
 
 You should record your model/system's outputs as a list of dicts with the following schema:
 

fanoutqa/models.py

@@ -110,7 +110,7 @@ class TestQuestion:
 
     @classmethod
     def from_dict(cls, d):
-        evidence = [Evidence.from_dict(e) for e in d["evidence"]]
+        evidence = [Evidence.from_dict(e) for e in d["necessary_evidence"]]
         return cls(
             id=d["id"],
             question=d["question"],

fanoutqa/wiki.py

@@ -14,7 +14,9 @@
 WIKI_CACHE_DIR.mkdir(exist_ok=True, parents=True)
 
 log = logging.getLogger(__name__)
-wikipedia = httpx.Client(base_url="https://en.wikipedia.org/w/api.php", headers={"User-Agent": USER_AGENT})
+wikipedia = httpx.Client(
+    base_url="https://en.wikipedia.org/w/api.php", headers={"User-Agent": USER_AGENT}, follow_redirects=True
+)
 
 
 class LazyEvidence(Evidence):
@@ -48,7 +50,7 @@ def revid(self):
         )
         resp.raise_for_status()
         data = resp.json()
-        page = data["query"]["pages"][self.pageid]
+        page = data["query"]["pages"][str(self.pageid)]
         return page["revisions"][0]["revid"]