Skip to content

Commit

Permalink
Merge pull request #341 from nasaharvest/get-new-months
Browse files Browse the repository at this point in the history 
Get new data for Ethiopia Tigray 2021
  • Loading branch information
@ivanzvonkov
ivanzvonkov authored Sep 5, 2023
2 parents 5797685 + 28150d7 commit fc63975
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 92 deletions.
4 changes: 2 additions & 2 deletions data/datasets.dvc
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
outs:
- md5: 001feb4ecdaa108deaf43002ef840c11.dir
size: 663324332
- md5: 5e9f23a90c0dd631f249251ac7b68f26.dir
size: 659946253
nfiles: 46
path: datasets
hash: md5
29 changes: 29 additions & 0 deletions data/report.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,15 @@ eo_data_export_failed 1



Ethiopia (Timesteps: 24)
----------------------------------------------------------------------------
disagreement: 0.0%
eo_data_complete 3649
eo_data_duplicate 864
✔ training amount: 3649, positive class: 55.0%



Ethiopia_Tigray_2020 (Timesteps: 24)
----------------------------------------------------------------------------
disagreement: 14.4%
Expand All @@ -131,6 +140,26 @@ eo_data_skipped 173



Ethiopia_Tigray_2021 (Timesteps: 24)
----------------------------------------------------------------------------
disagreement: 19.0%
eo_data_complete 718
eo_data_skipped 168
✔ validation amount: 351, positive class: 27.9%
✔ testing amount: 367, positive class: 32.7%



Ethiopia_Bure_Jimma_2019 (Timesteps: 24)
----------------------------------------------------------------------------
disagreement: 17.8%
eo_data_complete 986
eo_data_skipped 214
✔ validation amount: 488, positive class: 38.7%
✔ testing amount: 498, positive class: 32.3%



Ethiopia_Bure_Jimma_2020 (Timesteps: 24)
----------------------------------------------------------------------------
disagreement: 21.8%
Expand Down
180 changes: 90 additions & 90 deletions datasets.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -724,44 +724,44 @@ def load_labels(self) -> pd.DataFrame:
),
),
),
# CustomLabeledDataset(
# dataset="Ethiopia",
# country="Ethiopia",
# raw_labels=(
# RawLabels(filename="tigray/tigrayWW_crop.shp", class_prob=1.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_crop2.shp", class_prob=1.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_forest.shp", class_prob=0.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_forest2.shp", class_prob=0.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_shrub.shp", class_prob=0.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_shrub2.shp", class_prob=0.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_sparse.shp", class_prob=0.0, start_year=2019),
# RawLabels(filename="tigray/tigrayWW_sparse2.shp", class_prob=0.0, start_year=2019),
# RawLabels(
# filename="tigray_non_fallow_crop/nonFallowCrop2019.shp",
# class_prob=1.0,
# start_year=2019,
# ),
# RawLabels(
# filename="tigray_non_fallow_crop/nonFallowCrop2020.shp",
# class_prob=1.0,
# start_year=2020,
# ),
# RawLabels(
# filename="tigray_corrective_2020/non_crop.shp", class_prob=0.0, start_year=2020
# ),
# RawLabels(filename="tigray_corrective_2020/crop.shp", class_prob=1.0, start_year=2020),
# RawLabels(
# filename="tigray_corrective_2021/non_crop.shp",
# class_prob=0.0,
# start_year=2021,
# ),
# RawLabels(
# filename="tigray_corrective_2021/crop.shp",
# class_prob=1.0,
# start_year=2021,
# ),
# ),
# ),
CustomLabeledDataset(
dataset="Ethiopia",
country="Ethiopia",
raw_labels=(
RawLabels(filename="tigray/tigrayWW_crop.shp", class_prob=1.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_crop2.shp", class_prob=1.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_forest.shp", class_prob=0.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_forest2.shp", class_prob=0.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_shrub.shp", class_prob=0.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_shrub2.shp", class_prob=0.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_sparse.shp", class_prob=0.0, start_year=2019),
RawLabels(filename="tigray/tigrayWW_sparse2.shp", class_prob=0.0, start_year=2019),
RawLabels(
filename="tigray_non_fallow_crop/nonFallowCrop2019.shp",
class_prob=1.0,
start_year=2019,
),
RawLabels(
filename="tigray_non_fallow_crop/nonFallowCrop2020.shp",
class_prob=1.0,
start_year=2020,
),
RawLabels(
filename="tigray_corrective_2020/non_crop.shp", class_prob=0.0, start_year=2020
),
RawLabels(filename="tigray_corrective_2020/crop.shp", class_prob=1.0, start_year=2020),
RawLabels(
filename="tigray_corrective_2021/non_crop.shp",
class_prob=0.0,
start_year=2021,
),
RawLabels(
filename="tigray_corrective_2021/crop.shp",
class_prob=1.0,
start_year=2021,
),
),
),
CustomLabeledDataset(
dataset="Ethiopia_Tigray_2020",
country="Ethiopia",
Expand Down Expand Up @@ -790,58 +790,58 @@ def load_labels(self) -> pd.DataFrame:
),
),
),
# CustomLabeledDataset(
# dataset="Ethiopia_Tigray_2021",
# country="Ethiopia",
# raw_labels=(
# RawLabels(
# filename="ceo-2021-Ethiopia-Tigray-(Set-1-Fixed)-sample-data-2022-02-24.csv",
# class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
# start_year=2021,
# latitude_col="lat",
# longitude_col="lon",
# train_val_test=(0.0, 0.5, 0.5),
# filter_df=clean_ceo_data,
# labeler_name="email",
# label_duration="analysis_duration",
# ),
# RawLabels(
# filename="ceo-2021-Ethiopia-Tigray-(Set-2-Fixed)-sample-data-2022-02-24.csv",
# class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
# start_year=2021,
# latitude_col="lat",
# longitude_col="lon",
# train_val_test=(0.0, 0.5, 0.5),
# filter_df=clean_ceo_data,
# labeler_name="email",
# label_duration="analysis_duration",
# ),
# ),
# ),
# CustomLabeledDataset(
# dataset="Ethiopia_Bure_Jimma_2019",
# country="Ethiopia",
# raw_labels=(
# RawLabels(
# filename="ceo-2019-Ethiopia---Bure-Jimma-(Set-1)-sample-data-2021-11-24.csv",
# class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
# start_year=2019,
# latitude_col="lat",
# longitude_col="lon",
# train_val_test=(0.0, 0.5, 0.5),
# filter_df=clean_ceo_data,
# ),
# RawLabels(
# filename="ceo-2019-Ethiopia---Bure-Jimma-(Set-2)-sample-data-2021-11-24.csv",
# class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
# start_year=2019,
# latitude_col="lat",
# longitude_col="lon",
# train_val_test=(0.0, 0.5, 0.5),
# filter_df=clean_ceo_data,
# ),
# ),
# ),
CustomLabeledDataset(
dataset="Ethiopia_Tigray_2021",
country="Ethiopia",
raw_labels=(
RawLabels(
filename="ceo-2021-Ethiopia-Tigray-(Set-1-Fixed)-sample-data-2022-02-24.csv",
class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
start_year=2021,
latitude_col="lat",
longitude_col="lon",
train_val_test=(0.0, 0.5, 0.5),
filter_df=clean_ceo_data,
labeler_name="email",
label_duration="analysis_duration",
),
RawLabels(
filename="ceo-2021-Ethiopia-Tigray-(Set-2-Fixed)-sample-data-2022-02-24.csv",
class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
start_year=2021,
latitude_col="lat",
longitude_col="lon",
train_val_test=(0.0, 0.5, 0.5),
filter_df=clean_ceo_data,
labeler_name="email",
label_duration="analysis_duration",
),
),
),
CustomLabeledDataset(
dataset="Ethiopia_Bure_Jimma_2019",
country="Ethiopia",
raw_labels=(
RawLabels(
filename="ceo-2019-Ethiopia---Bure-Jimma-(Set-1)-sample-data-2021-11-24.csv",
class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
start_year=2019,
latitude_col="lat",
longitude_col="lon",
train_val_test=(0.0, 0.5, 0.5),
filter_df=clean_ceo_data,
),
RawLabels(
filename="ceo-2019-Ethiopia---Bure-Jimma-(Set-2)-sample-data-2021-11-24.csv",
class_prob=lambda df: (df["Does this pixel contain active cropland?"] == "Crop"),
start_year=2019,
latitude_col="lat",
longitude_col="lon",
train_val_test=(0.0, 0.5, 0.5),
filter_df=clean_ceo_data,
),
),
),
CustomLabeledDataset(
dataset="Ethiopia_Bure_Jimma_2020",
country="Ethiopia",
Expand Down

0 comments on commit fc63975

Please sign in to comment.