Process diversity data

open-innovations · Sep 3, 2024 · edc3ad3 · edc3ad3
1 parent 32673ed
commit edc3ad3
Show file tree

Hide file tree

Showing 7 changed files with 600 additions and 23 deletions.
diff --git a/scripts/lcip/transform.py b/scripts/lcip/transform.py
@@ -27,6 +27,11 @@
     'CARER (FUNDED)'
 ]
 
+project_files = ['inspire', 'grow_project', 'activate']
+revenue_files = ['grow_revenue', 'thrive', 'cultural_anchors']
+project_aggregated = pd.DataFrame()
+revenue_aggregated = pd.DataFrame()
+
 # Function to fuzzy match and merge dataframes
 def fuzzy_merge(df1, df2, key1, key2, threshold=90):
     s = df2[key2].tolist()
@@ -75,29 +80,31 @@ def process_wards(ward_data, data, out_path, theme, output_file):
                     theme_df.to_csv(os.path.join(out_path, theme_filename), index=True)
 
                 if data['THEME'].isin(diversity_metrics).any():
-                    # Split data into applied and funded
-                    applied = data[data['THEME'].str.contains(r'\(APPLIED\)')]
-                    funded = data[data['THEME'].str.contains(r'\(FUNDED\)')]
-
-                    # Remove the (APPLIED) and (FUNDED) from the THEME
-                    applied['THEME'] = applied['THEME'].str.replace(r' \(APPLIED\)', '', regex=True)
-                    funded['THEME'] = funded['THEME'].str.replace(r' \(FUNDED\)', '', regex=True)
-
-                    # Merge applied and funded on THEME and METRIC
-                    diversity = pd.merge(
-                        applied[['THEME', 'METRIC', 'R1 Q1']],
-                        funded[['THEME', 'METRIC', 'R1 Q1']],
-                        on=['THEME', 'METRIC'],
-                        how='outer',
-                        suffixes=('_APPLIED', '_FUNDED')
-                    )
-
-                    # Rename columns to metric, applied, funded
-                    diversity.rename(columns={'THEME': 'METRIC', 'R1 Q1_APPLIED': 'APPLIED', 'R1 Q1_FUNDED': 'FUNDED'}, inplace=True)
-
-                    # Save to CSV
-                    diversity.to_csv(os.path.join(OUT_DIR, 'diversity.csv'), index=False)
-
+                        applied = data[data['THEME'].str.contains(r'\(APPLIED\)')]
+                        funded = data[data['THEME'].str.contains(r'\(FUNDED\)')]
+
+                        applied['THEME'] = applied['THEME'].str.replace(r' \(APPLIED\)', '', regex=True)
+                        funded['THEME'] = funded['THEME'].str.replace(r' \(FUNDED\)', '', regex=True)
+
+                        diversity = pd.merge(
+                            applied[['THEME', 'METRIC', 'R1 Q1']],
+                            funded[['THEME', 'METRIC', 'R1 Q1']],
+                            on=['THEME', 'METRIC'],
+                            how='outer',
+                            suffixes=('_APPLIED', '_FUNDED')
+                        )
+
+                        diversity.rename(columns={'THEME': 'METRIC', 'R1 Q1_APPLIED': 'APPLIED', 'R1 Q1_FUNDED': 'FUNDED'}, inplace=True)
+
+                        output_file_name = f"{filename_stem}_diversity.csv"
+                        if filename_stem in project_files:
+                            output_path = os.path.join(OUT_DIR, 'diversity', 'project', output_file_name)
+                        elif filename_stem in revenue_files:
+                            output_path = os.path.join(OUT_DIR, 'diversity', 'revenue', output_file_name)
+
+                        if output_path:  
+                            diversity.to_csv(output_path, index=False)
+
                 process_wards(ward_data, data, out_path, 'WARDS - APPLICANT BASED', 'applications_by_ward.csv')
                 process_wards(ward_data, data, out_path, 'WARDS - RECEIVING ACTIVITY', 'received_by_ward.csv')
 

diff --git a/src/_data/viz/lcip/diversity/project/activate_diversity.csv b/src/_data/viz/lcip/diversity/project/activate_diversity.csv
@@ -0,0 +1,95 @@
+METRIC,METRIC,APPLIED,FUNDED
+AGE,18 - 24,11.0,1.0
+AGE,25 - 34,16.0,3.0
+AGE,35 - 44,24.0,17.0
+AGE,45 - 54,14.0,3.0
+AGE,55 - 64,9.0,7.0
+AGE,65 - 74,8.0,4.0
+AGE,75 - 84,13.0,8.0
+AGE,85+,1.0,1.0
+AGE,Not known,2.0,0.0
+AGE,Prefer not to say,0.0,0.0
+ARTFORM,Circus,0.0,0.0
+ARTFORM,Combined Arts,4.0,0.0
+ARTFORM,Crafts,1.0,1.0
+ARTFORM,Dance,1.0,0.0
+ARTFORM,Design,0.0,0.0
+ARTFORM,Digital Media,2.0,2.0
+ARTFORM,Festival/Carnival,1.0,0.0
+ARTFORM,Film,2.0,0.0
+ARTFORM,Literature / Poetry / Spoken Word,1.0,0.0
+ARTFORM,Live Art,0.0,0.0
+ARTFORM,Music,7.0,2.0
+ARTFORM,Photography,2.0,1.0
+ARTFORM,Theatre,1.0,0.0
+ARTFORM,Visual Art,5.0,3.0
+CARER,Do you consider yourself to be a carer?,,
+CARER,No,18.0,5.0
+CARER,Not known,61.0,12.0
+CARER,Prefer not to say,10.0,10.0
+CARER,Yes,5.0,4.0
+DISABILITY,Hearing impairment,1.0,1.0
+DISABILITY,Learning disability,7.0,2.0
+DISABILITY,Long-standing illness or health condition,4.0,2.0
+DISABILITY,Mental health condition,5.0,2.0
+DISABILITY,Neuro Diverse,8.0,3.0
+DISABILITY,Not known,39.0,4.0
+DISABILITY,Other,0.0,0.0
+DISABILITY,Physical impairment,6.0,6.0
+DISABILITY,Prefer not to say,10.0,10.0
+DISABILITY,Visual impairment,0.0,0.0
+ETHNIC ORIGIN,African,1.0,
+ETHNIC ORIGIN,Any other Asian background,,
+ETHNIC ORIGIN,Any other Black background,,
+ETHNIC ORIGIN,Any other background,,
+ETHNIC ORIGIN,Any other mixed/multiple ethnic group,,
+ETHNIC ORIGIN,Any other white background,8.0,3.0
+ETHNIC ORIGIN,Arab,,
+ETHNIC ORIGIN,Asian or Asian British,,
+ETHNIC ORIGIN,Bangladeshi,,
+ETHNIC ORIGIN,Black or Black British,,
+ETHNIC ORIGIN,Caribbean,2.0,2.0
+ETHNIC ORIGIN,Chinese,1.0,
+ETHNIC ORIGIN,Indian,1.0,
+ETHNIC ORIGIN,Kashmiri,,
+ETHNIC ORIGIN,Mixed/ multiple ethnic group,,
+ETHNIC ORIGIN,Other ethnic groups,,
+ETHNIC ORIGIN,Pakistani,,
+ETHNIC ORIGIN,Prefer not to say,8.0,
+ETHNIC ORIGIN,White,,
+ETHNIC ORIGIN,White British,7.0,1.0
+ETHNIC ORIGIN,White English,64.0,31.0
+ETHNIC ORIGIN,White Gypsy or Traveller,,
+ETHNIC ORIGIN,White Irish,,
+ETHNIC ORIGIN,White Northern Irish,,
+ETHNIC ORIGIN,White Scottish,3.0,
+ETHNIC ORIGIN,White Welsh,3.0,1.0
+ETHNIC ORIGIN,White and Asian,1.0,1.0
+ETHNIC ORIGIN,White and Black African,1.0,
+ETHNIC ORIGIN,White and Black Caribbean,1.0,
+GENDER REGISTERED AT BIRTH,Not known,0.0,0.0
+GENDER REGISTERED AT BIRTH,Prefer not to say,0.0,0.0
+GENDER REGISTERED AT BIRTH,Total,0.0,0.0
+RELIGIOUS BELIEF,Buddhist,0.0,0.0
+RELIGIOUS BELIEF,Christian,15.0,13.0
+RELIGIOUS BELIEF,Hindu,0.0,0.0
+RELIGIOUS BELIEF,Jewish,1.0,0.0
+RELIGIOUS BELIEF,Muslim,0.0,0.0
+RELIGIOUS BELIEF,No religion/belief,21.0,13.0
+RELIGIOUS BELIEF,Not known,54.0,16.0
+RELIGIOUS BELIEF,Other,0.0,0.0
+RELIGIOUS BELIEF,Prefer not to say,16.0,12.0
+RELIGIOUS BELIEF,Sikh,0.0,0.0
+SEX,Female,52.0,26.0
+SEX,Male,44.0,17.0
+SEX,Non Binary,4.0,1.0
+SEX,Not known,8.0,0.0
+SEX,Other,0.0,0.0
+SEX,Prefer not to say,0.0,0.0
+SEXUAL ORIENTATION,Bi-sexual,7.0,1.0
+SEXUAL ORIENTATION,Gay,3.0,1.0
+SEXUAL ORIENTATION,Heterosexual or Straight,38.0,32.0
+SEXUAL ORIENTATION,Lesbian,2.0,2.0
+SEXUAL ORIENTATION,Not known,54.0,7.0
+SEXUAL ORIENTATION,Other,1.0,1.0
+SEXUAL ORIENTATION,Prefer not to say,3.0,
diff --git a/src/_data/viz/lcip/diversity/project/grow_project_diversity.csv b/src/_data/viz/lcip/diversity/project/grow_project_diversity.csv
@@ -0,0 +1,95 @@
+METRIC,METRIC,APPLIED,FUNDED
+AGE,18 - 24,11.0,1.0
+AGE,25 - 34,16.0,3.0
+AGE,35 - 44,24.0,17.0
+AGE,45 - 54,14.0,3.0
+AGE,55 - 64,9.0,7.0
+AGE,65 - 74,8.0,4.0
+AGE,75 - 84,13.0,8.0
+AGE,85+,1.0,1.0
+AGE,Not known,2.0,0.0
+AGE,Prefer not to say,0.0,0.0
+ARTFORM,Circus,0.0,0.0
+ARTFORM,Combined Arts,4.0,0.0
+ARTFORM,Crafts,1.0,1.0
+ARTFORM,Dance,1.0,0.0
+ARTFORM,Design,0.0,0.0
+ARTFORM,Digital Media,2.0,2.0
+ARTFORM,Festival/Carnival,1.0,0.0
+ARTFORM,Film,2.0,0.0
+ARTFORM,Literature / Poetry / Spoken Word,1.0,0.0
+ARTFORM,Live Art,0.0,0.0
+ARTFORM,Music,7.0,2.0
+ARTFORM,Photography,2.0,1.0
+ARTFORM,Theatre,1.0,0.0
+ARTFORM,Visual Art,5.0,3.0
+CARER,Do you consider yourself to be a carer?,,
+CARER,No,18.0,5.0
+CARER,Not known,61.0,12.0
+CARER,Prefer not to say,10.0,10.0
+CARER,Yes,5.0,4.0
+DISABILITY,Hearing impairment,1.0,1.0
+DISABILITY,Learning disability,7.0,2.0
+DISABILITY,Long-standing illness or health condition,4.0,2.0
+DISABILITY,Mental health condition,5.0,2.0
+DISABILITY,Neuro Diverse,8.0,3.0
+DISABILITY,Not known,39.0,4.0
+DISABILITY,Other,0.0,0.0
+DISABILITY,Physical impairment,6.0,6.0
+DISABILITY,Prefer not to say,10.0,10.0
+DISABILITY,Visual impairment,0.0,0.0
+ETHNIC ORIGIN,African,1.0,
+ETHNIC ORIGIN,Any other Asian background,,
+ETHNIC ORIGIN,Any other Black background,,
+ETHNIC ORIGIN,Any other background,,
+ETHNIC ORIGIN,Any other mixed/multiple ethnic group,,
+ETHNIC ORIGIN,Any other white background,8.0,3.0
+ETHNIC ORIGIN,Arab,,
+ETHNIC ORIGIN,Asian or Asian British,,
+ETHNIC ORIGIN,Bangladeshi,,
+ETHNIC ORIGIN,Black or Black British,,
+ETHNIC ORIGIN,Caribbean,2.0,2.0
+ETHNIC ORIGIN,Chinese,1.0,
+ETHNIC ORIGIN,Indian,1.0,
+ETHNIC ORIGIN,Kashmiri,,
+ETHNIC ORIGIN,Mixed/ multiple ethnic group,,
+ETHNIC ORIGIN,Other ethnic groups,,
+ETHNIC ORIGIN,Pakistani,,
+ETHNIC ORIGIN,Prefer not to say,8.0,
+ETHNIC ORIGIN,White,,
+ETHNIC ORIGIN,White British,7.0,1.0
+ETHNIC ORIGIN,White English,64.0,31.0
+ETHNIC ORIGIN,White Gypsy or Traveller,,
+ETHNIC ORIGIN,White Irish,,
+ETHNIC ORIGIN,White Northern Irish,,
+ETHNIC ORIGIN,White Scottish,3.0,
+ETHNIC ORIGIN,White Welsh,3.0,1.0
+ETHNIC ORIGIN,White and Asian,1.0,1.0
+ETHNIC ORIGIN,White and Black African,1.0,
+ETHNIC ORIGIN,White and Black Caribbean,1.0,
+GENDER REGISTERED AT BIRTH,Not known,0.0,0.0
+GENDER REGISTERED AT BIRTH,Prefer not to say,0.0,0.0
+GENDER REGISTERED AT BIRTH,Total,0.0,0.0
+RELIGIOUS BELIEF,Buddhist,0.0,0.0
+RELIGIOUS BELIEF,Christian,15.0,13.0
+RELIGIOUS BELIEF,Hindu,0.0,0.0
+RELIGIOUS BELIEF,Jewish,1.0,0.0
+RELIGIOUS BELIEF,Muslim,0.0,0.0
+RELIGIOUS BELIEF,No religion/belief,21.0,13.0
+RELIGIOUS BELIEF,Not known,54.0,16.0
+RELIGIOUS BELIEF,Other,0.0,0.0
+RELIGIOUS BELIEF,Prefer not to say,16.0,12.0
+RELIGIOUS BELIEF,Sikh,0.0,0.0
+SEX,Female,52.0,26.0
+SEX,Male,44.0,17.0
+SEX,Non Binary,4.0,1.0
+SEX,Not known,8.0,0.0
+SEX,Other,0.0,0.0
+SEX,Prefer not to say,0.0,0.0
+SEXUAL ORIENTATION,Bi-sexual,7.0,1.0
+SEXUAL ORIENTATION,Gay,3.0,1.0
+SEXUAL ORIENTATION,Heterosexual or Straight,38.0,32.0
+SEXUAL ORIENTATION,Lesbian,2.0,2.0
+SEXUAL ORIENTATION,Not known,54.0,7.0
+SEXUAL ORIENTATION,Other,1.0,1.0
+SEXUAL ORIENTATION,Prefer not to say,3.0,
diff --git a/src/_data/viz/lcip/diversity/project/inspire_diversity.csv b/src/_data/viz/lcip/diversity/project/inspire_diversity.csv
@@ -0,0 +1,95 @@
+METRIC,METRIC,APPLIED,FUNDED
+AGE,18 - 24,11.0,1.0
+AGE,25 - 34,16.0,3.0
+AGE,35 - 44,24.0,17.0
+AGE,45 - 54,14.0,3.0
+AGE,55 - 64,9.0,7.0
+AGE,65 - 74,8.0,4.0
+AGE,75 - 84,13.0,8.0
+AGE,85+,1.0,1.0
+AGE,Not known,2.0,0.0
+AGE,Prefer not to say,0.0,0.0
+ARTFORM,Circus,0.0,0.0
+ARTFORM,Combined Arts,4.0,0.0
+ARTFORM,Crafts,1.0,1.0
+ARTFORM,Dance,1.0,0.0
+ARTFORM,Design,0.0,0.0
+ARTFORM,Digital Media,2.0,2.0
+ARTFORM,Festival/Carnival,1.0,0.0
+ARTFORM,Film,2.0,0.0
+ARTFORM,Literature / Poetry / Spoken Word,1.0,0.0
+ARTFORM,Live Art,0.0,0.0
+ARTFORM,Music,7.0,2.0
+ARTFORM,Photography,2.0,1.0
+ARTFORM,Theatre,1.0,0.0
+ARTFORM,Visual Art,5.0,3.0
+CARER,Do you consider yourself to be a carer?,,
+CARER,No,18.0,5.0
+CARER,Not known,61.0,12.0
+CARER,Prefer not to say,10.0,10.0
+CARER,Yes,5.0,4.0
+DISABILITY,Hearing impairment,1.0,1.0
+DISABILITY,Learning disability,7.0,2.0
+DISABILITY,Long-standing illness or health condition,4.0,2.0
+DISABILITY,Mental health condition,5.0,2.0
+DISABILITY,Neuro Diverse,8.0,3.0
+DISABILITY,Not known,39.0,4.0
+DISABILITY,Other,0.0,0.0
+DISABILITY,Physical impairment,6.0,6.0
+DISABILITY,Prefer not to say,10.0,10.0
+DISABILITY,Visual impairment,0.0,0.0
+ETHNIC ORIGIN,African,1.0,
+ETHNIC ORIGIN,Any other Asian background,,
+ETHNIC ORIGIN,Any other Black background,,
+ETHNIC ORIGIN,Any other background,,
+ETHNIC ORIGIN,Any other mixed/multiple ethnic group,,
+ETHNIC ORIGIN,Any other white background,8.0,3.0
+ETHNIC ORIGIN,Arab,,
+ETHNIC ORIGIN,Asian or Asian British,,
+ETHNIC ORIGIN,Bangladeshi,,
+ETHNIC ORIGIN,Black or Black British,,
+ETHNIC ORIGIN,Caribbean,2.0,2.0
+ETHNIC ORIGIN,Chinese,1.0,
+ETHNIC ORIGIN,Indian,1.0,
+ETHNIC ORIGIN,Kashmiri,,
+ETHNIC ORIGIN,Mixed/ multiple ethnic group,,
+ETHNIC ORIGIN,Other ethnic groups,,
+ETHNIC ORIGIN,Pakistani,,
+ETHNIC ORIGIN,Prefer not to say,8.0,
+ETHNIC ORIGIN,White,,
+ETHNIC ORIGIN,White British,7.0,1.0
+ETHNIC ORIGIN,White English,64.0,31.0
+ETHNIC ORIGIN,White Gypsy or Traveller,,
+ETHNIC ORIGIN,White Irish,,
+ETHNIC ORIGIN,White Northern Irish,,
+ETHNIC ORIGIN,White Scottish,3.0,
+ETHNIC ORIGIN,White Welsh,3.0,1.0
+ETHNIC ORIGIN,White and Asian,1.0,1.0
+ETHNIC ORIGIN,White and Black African,1.0,
+ETHNIC ORIGIN,White and Black Caribbean,1.0,
+GENDER REGISTERED AT BIRTH,Not known,0.0,0.0
+GENDER REGISTERED AT BIRTH,Prefer not to say,0.0,0.0
+GENDER REGISTERED AT BIRTH,Total,0.0,0.0
+RELIGIOUS BELIEF,Buddhist,0.0,0.0
+RELIGIOUS BELIEF,Christian,15.0,13.0
+RELIGIOUS BELIEF,Hindu,0.0,0.0
+RELIGIOUS BELIEF,Jewish,1.0,0.0
+RELIGIOUS BELIEF,Muslim,0.0,0.0
+RELIGIOUS BELIEF,No religion/belief,21.0,13.0
+RELIGIOUS BELIEF,Not known,54.0,16.0
+RELIGIOUS BELIEF,Other,0.0,0.0
+RELIGIOUS BELIEF,Prefer not to say,16.0,12.0
+RELIGIOUS BELIEF,Sikh,0.0,0.0
+SEX,Female,52.0,26.0
+SEX,Male,44.0,17.0
+SEX,Non Binary,4.0,1.0
+SEX,Not known,8.0,0.0
+SEX,Other,0.0,0.0
+SEX,Prefer not to say,0.0,0.0
+SEXUAL ORIENTATION,Bi-sexual,7.0,1.0
+SEXUAL ORIENTATION,Gay,3.0,1.0
+SEXUAL ORIENTATION,Heterosexual or Straight,38.0,32.0
+SEXUAL ORIENTATION,Lesbian,2.0,2.0
+SEXUAL ORIENTATION,Not known,54.0,7.0
+SEXUAL ORIENTATION,Other,1.0,1.0
+SEXUAL ORIENTATION,Prefer not to say,3.0,