Skip to content

Commit

Permalink
Modified name and subset npz assignment to fix mp
Browse files Browse the repository at this point in the history
  • Loading branch information
shenoynikhil committed Sep 27, 2023
1 parent 5ec94bd commit a5f52c5
Showing 1 changed file with 6 additions and 4 deletions.
10 changes: 6 additions & 4 deletions src/openqdc/datasets/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,10 +159,12 @@ def read_preprocess(self):
for key in ["name", "subset"]:
filename = p_join(self.preprocess_path, f"{key}.npz")
pull_locally(filename)
# with open(filename, "rb") as f:
self.data[key] = np.load(open(filename, "rb"))
for k in self.data[key]:
print(f"Loaded {key}_{k} with shape {self.data[key][k].shape}, dtype {self.data[key][k].dtype}")
self.data[key] = dict()
with open(filename, "rb") as f:
tmp = np.load(f)
for k in tmp:
self.data[key][k] = tmp[k]
print(f"Loaded {key}_{k} with shape {self.data[key][k].shape}, dtype {self.data[key][k].dtype}")

def is_preprocessed(self):
predicats = [copy_exists(p_join(self.preprocess_path, f"{key}.mmap")) for key in self.data_keys]
Expand Down

0 comments on commit a5f52c5

Please sign in to comment.