Add input data for simulations and supporting notebooks to create out…

…put charts from results.
fidelity · Dec 27, 2023 · 377756a · 377756a
1 parent 83f1782
commit 377756a
Show file tree

Hide file tree

Showing 8 changed files with 10,719 additions and 11 deletions.
diff --git a/examples/probabilistic_fairness/input_data/sampled_surrogate_inputs.csv b/examples/probabilistic_fairness/input_data/sampled_surrogate_inputs.csv
diff --git a/..._accuracy/input_data/surrogate_inputs.csv → ..._fairness/input_data/surrogate_inputs.csv b/..._accuracy/input_data/surrogate_inputs.csv → ..._fairness/input_data/surrogate_inputs.csv
diff --git a/examples/probabilistic_fairness/notebooks/analyze_prob_vs_model.ipynb b/examples/probabilistic_fairness/notebooks/analyze_prob_vs_model.ipynb
diff --git a/examples/probabilistic_fairness/notebooks/analyze_sample_size_sim.ipynb b/examples/probabilistic_fairness/notebooks/analyze_sample_size_sim.ipynb
diff --git a/...ess_accuracy/python_scripts/simulation.py → ...tic_fairness/python_scripts/simulation.py b/...ess_accuracy/python_scripts/simulation.py → ...tic_fairness/python_scripts/simulation.py
@@ -6,8 +6,10 @@
 sys.path.append('../../jurity/tests')
 sys.path.append('../../jurity/jurity')
 from jurity.fairness import BinaryFairnessMetrics as bfm
-
 from test_utils_proba import UtilsProbaSimulator
+
+output_path='~/Documents/data/jurity_tests/simulations/'
+
 testing_simulation=False
 n_runs=30
 avg_counts=[30,50]
@@ -22,11 +24,11 @@
            "moderately_unfair":moderately_unfair_sim,
            "very_unfair":very_unfair_sim,
            "extremely_unfair":extremely_unfair_sim}
-surrogates=pd.read_csv('./supporting_data/surrogate_inputs.csv')
+surrogates=pd.read_csv('../input_data/surrogate_inputs.csv')
 if testing_simulation:
-    output_string = '~/Documents/data/jurity_tests/simulations/{0}_simulation_count_{1}_surrogates_{2}_test.csv'
+    output_string = output_path+'{0}_simulation_count_{1}_surrogates_{2}_test.csv'
 else:
-    output_string = '~/Documents/data/jurity_tests/simulations/{0}_simulation_count_{1}_surrogates_{2}.csv'
+    output_string = output_path+'{0}_simulation_count_{1}_surrogates_{2}.csv'
 
 def run_one_sim(simulator, membership_df,count_mean,rng=np.random.default_rng()):
     membership_df["count"]=pd.Series(rng.poisson(lam=count_mean,size=membership_df.shape[0]))
@@ -36,7 +38,6 @@ def run_one_sim(simulator, membership_df,count_mean,rng=np.random.default_rng())
     prob_metrics=bfm.get_all_scores(test_data["label"],test_data["prediction"],
                    membership_df.set_index("ZIP")[["not_protected","protected"]],
                    test_data["ZIP"],[1]).rename(columns={"Value":"probabilistic_estimate"})
-    #TODO: Add simulation for argmax
     predicted_class=test_data[["not_protected","protected"]].values.tolist()
     argmax_metrics=bfm.get_all_scores(test_data["label"].values,test_data["prediction"].values,
                                      predicted_class).rename(columns={"Value":"argmax_estimate"})

diff --git a/...on_scripts/simulation_compare_to_model.py → ...on_scripts/simulation_compare_to_model.py b/...on_scripts/simulation_compare_to_model.py → ...on_scripts/simulation_compare_to_model.py
diff --git a/...uracy/python_scripts/simulation_counts.py → ...rness/python_scripts/simulation_counts.py b/...uracy/python_scripts/simulation_counts.py → ...rness/python_scripts/simulation_counts.py
@@ -6,8 +6,8 @@
 sys.path.append('../../jurity/tests')
 sys.path.append('../../jurity/jurity')
 from jurity.fairness import BinaryFairnessMetrics as bfm
-
 from test_utils_proba import UtilsProbaSimulator
+output_path='~/Documents/data/jurity_tests/simulations/sample_size/min_weight_0/'
 testing_simulation=False
 n_runs=30
 avg_counts=[5,10,20,30,40]
@@ -26,9 +26,9 @@
 surrogates=pd.read_csv('./supporting_data/surrogate_inputs.csv')
 surrogates["ZIP"]=surrogates["ZIP"].astype(int)
 if testing_simulation:
-    output_string = '~/Documents/data/jurity_tests/simulations/sample_size/min_weight_0/{0}_simulation_count_{1}_test_surrogates_{2}.csv'
+    output_string = output_path+'{0}_simulation_count_{1}_test_surrogates_{2}.csv'
 else:
-    output_string = '~/Documents/data/jurity_tests/simulations/sample_size/min_weight_0/{0}_simulation_count_{1}_surrogates_{2}.csv'
+    output_string = output_path+'sample_size/min_weight_0/{0}_simulation_count_{1}_surrogates_{2}.csv'
 
 def run_one_sim(test_data,membership_df):
     #Sometimes the sub-sampling leads to data errors.

diff --git a/examples/prob_fairness_accuracy/readme.md → examples/probabilistic_fairness/readme.md b/examples/prob_fairness_accuracy/readme.md → examples/probabilistic_fairness/readme.md
@@ -3,9 +3,9 @@ This folder contains the files necessary to reproduce the results from
 academic papers demonstrating the accuracy of probabilistic 
 fairness under different sample sizes. Probabilistic fairness is a technique 
 unique to jurity that allows users to calculate fairness metrics when protected status is 
-unknown but a <i>surrogate class</i> is available. A surrogate class is 
-an alternative feature that divides the population into groups and where the probability 
-of protected class membership is known given the surrogate class.
+unknown but a surrogate class feature is available. A <i>surrogate</i> class divides 
+the population into groups where 
+the probability of protected class membership is known given the surrogate class membership.
 
 Probabilistic fairness, its accuracy, and the simulation method used in
 these demonstrations are detailed in