Run the data pipelines

open-innovations · Oct 10, 2023 · f6e76f1 · f6e76f1
1 parent 921b755
commit f6e76f1
Show file tree

Hide file tree

Showing 7 changed files with 205 additions and 220 deletions.
diff --git a/data/metrics/events/master/all.csv b/data/metrics/events/master/all.csv
@@ -927,7 +927,6 @@ actual_audience_size_number_of_participants_in_person,actual_audience_size_numbe
 ,,466,2023-11-26,The Weight of Words,['Partner Programme - Event'],Henry Moore Institute: The Weight of Words,,['LS1 3AH'],Henry Moore Institute,Partner,"['Season 2 - Playing', 'Season 3 - Dreaming']",2023-07-07,['Little London and Woodhouse']
 ,,284,2023-07-08,"""70 x 7: The Meal"": Harewood Big Picnic","['Public Event - Large Scale Outdoor Event', 'CLE - Community Engagement session']","Hidden Stories: ""70 x 7: The Meal"": Harewood Big Picnic",,['LS17 9LG'],Hidden Stories,CLE,['Season 2 - Playing'],2023-07-08,['Harewood']
 78.0,,1770,2023-07-08,African and Caribbean Enterprise ,"['CLE - Skills and Talent session', 'CLE - Community Engagement session']",Leeds International African Arts Festival : African and Caribbean Enterprise ,,['LS1 3HE'],Leeds International African Arts Festival ,,['Season 2 - Playing'],2023-07-08,['Little London and Woodhouse']
-78.0,,1770,2023-07-08,African and Caribbean Enterprise ,"['CLE - Skills and Talent session', 'CLE - Community Engagement session']",Leeds International African Arts Festival : African and Caribbean Enterprise ,,['LS1 3HE'],Leeds International African Arts Festival ,,['Season 2 - Playing'],2023-07-08,['Little London and Woodhouse']
 143.0,,428,2023-07-08,My LEEDS Summer: Gipton & Harehills,['Public Event - Performance'],12 - My LEEDS 2023: My LEEDS Summer: Gipton & Harehills,,"['LS9 6NL', 'LS9 7BG']",12 - My LEEDS 2023,Produce (Signature),['Season 2 - Playing'],2023-07-08,"['Gipton and Harehills', 'Gipton and Harehills']"
 ,,265,2023-07-09,My Leeds Event Day 1 (Live Activation),['Public Event - Performance'],12 - My LEEDS 2023: My Leeds Event Day 1 (Live Activation),,,12 - My LEEDS 2023,Produce (Signature),['Season 2 - Playing'],2023-07-08,
 ,,1145,,,,: ,,,,,,2023-07-08,
@@ -938,21 +937,16 @@ actual_audience_size_number_of_participants_in_person,actual_audience_size_numbe
 100.0,,1771,2023-07-09,Step into Africa,['CLE - Community Engagement session'],Leeds International African Arts Festival : Step into Africa,,['LS18 5HD'],Leeds International African Arts Festival ,CLE,['Season 2 - Playing'],2023-07-09,['Horsforth']
 60.0,,1772,2023-07-10,Anthology Launch ,['CLE - Community Engagement session'],Leeds International African Arts Festival : Anthology Launch ,,['LS7 3EX'],Leeds International African Arts Festival ,CLE,,2023-07-10,
 265.0,,1773,2023-07-14,Cinema Africa ,['CLE - Community Engagement session'],Leeds International African Arts Festival : Cinema Africa ,,['LS6 1JD'],Leeds International African Arts Festival ,CLE,,2023-07-11,['Headingley and Hyde Park']
-100.0,,1771,2023-07-09,Step into Africa,['CLE - Community Engagement session'],Leeds International African Arts Festival : Step into Africa,,['LS18 5HD'],Leeds International African Arts Festival ,CLE,['Season 2 - Playing'],2023-07-09,['Horsforth']
-60.0,,1772,2023-07-10,Anthology Launch ,['CLE - Community Engagement session'],Leeds International African Arts Festival : Anthology Launch ,,['LS7 3EX'],Leeds International African Arts Festival ,CLE,,2023-07-10,
-265.0,,1773,2023-07-14,Cinema Africa ,['CLE - Community Engagement session'],Leeds International African Arts Festival : Cinema Africa ,,['LS6 1JD'],Leeds International African Arts Festival ,CLE,,2023-07-11,['Headingley and Hyde Park']
 7.0,,1852,2023-07-11,Leeds Book Awards Oracy Project: Blenheim Primary,['CLE - Community Engagement session'],Creative Learning S2: Leeds Book Awards Oracy Project: Blenheim Primary,,,Creative Learning S2,CLE,['Season 2 - Playing'],2023-07-11,
 ,,516,2023-07-14,JULIE: The Musical ,['Public Event - Performance'],Carriageworks Theatre: JULIE: The Musical ,,['LS2 3AD'],Carriageworks Theatre,Promote,['Season 2 - Playing'],2023-07-12,['Little London and Woodhouse']
 7.0,,243,2023-07-12,NEST: Protagonist Auditions 2nd Round,"['CLE - Skills and Talent session', 'CLE - Participant / Volunteer thank you event']",10 - Melt: NEST: Protagonist Auditions 2nd Round,,['LS10 1NE'],10 - Melt,Produce (Signature),['Season 1 - Awakening'],2023-07-12,['Hunslet and Riverside']
 250.0,,1774,2023-07-13,Jollof Cook Off,['CLE - Community Engagement session'],Leeds International African Arts Festival : Jollof Cook Off,,['LS10 1JY'],Leeds International African Arts Festival ,CLE,,2023-07-13,
-250.0,,1774,2023-07-13,Jollof Cook Off,['CLE - Community Engagement session'],Leeds International African Arts Festival : Jollof Cook Off,,['LS10 1JY'],Leeds International African Arts Festival ,CLE,,2023-07-13,
 1050.0,,218,2023-07-14,Children's Day Event Day,['Public Event - Large Scale Outdoor Event'],07 - Children's Day: Children's Day Event Day,,['LS8 1DF'],07 - Children's Day,Produce (Signature),['Season 2 - Playing'],2023-07-14,['Roundhay']
 6.0,,1127,2023-07-14,Digital Upskilling Workshop 14/20,['CLE - Skills and Talent session'],Creative Learning S2: Digital Upskilling Workshop 14/20,,['LS25 1EH'],Creative Learning S2,CLE,['Season 2 - Playing'],2023-07-14,['Garforth and Swillington']
 7.0,,1853,2023-07-14,Leeds Book Awards Oracy Project: Cockburn John Charles Academy,['CLE - Community Engagement session'],Creative Learning S2: Leeds Book Awards Oracy Project: Cockburn John Charles Academy,,,Creative Learning S2,CLE,['Season 2 - Playing'],2023-07-14,
 7.0,,1854,2023-07-14,Leeds Book Awards Oracy Project: Moor Allerton Hall Primary,['CLE - Community Engagement session'],Creative Learning S2: Leeds Book Awards Oracy Project: Moor Allerton Hall Primary,,,Creative Learning S2,CLE,['Season 2 - Playing'],2023-07-14,
 ,,320,2023-07-14,"Ntantu, 2 Worlds 1 Song EP release","['Public Event - Digital Experience', 'Partner Programme - Event']","Ntantu, 2 Worlds 1 Voice: Ntantu, 2 Worlds 1 Song EP release",,,"Ntantu, 2 Worlds 1 Voice",Partner,['Season 2 - Playing'],2023-07-14,
 160.0,,1775,2023-07-15,Awards and Dinner,['CLE - Community Engagement session'],Leeds International African Arts Festival : Awards and Dinner,,['LS7 3EJ'],Leeds International African Arts Festival ,CLE,,2023-07-15,
-160.0,,1775,2023-07-15,Awards and Dinner,['CLE - Community Engagement session'],Leeds International African Arts Festival : Awards and Dinner,,['LS7 3EJ'],Leeds International African Arts Festival ,CLE,,2023-07-15,
 ,,416,2023-07-15,LCEP CPD event ,['CLE - Youth Voice session'],Youth Voice Programmes: LCEP CPD event ,,,Youth Voice Programmes,CLE,['Season 2 - Playing'],2023-07-15,
 3000.0,,433,2023-07-15,My LEEDS Summer: Adel and Wharfdale Ralph Thoresby Summer Festival with LEEDS 2023,['Public Event - Performance'],12 - My LEEDS 2023: My LEEDS Summer: Adel and Wharfdale Ralph Thoresby Summer Festival with LEEDS 2023,,['LS16 7RX'],12 - My LEEDS 2023,Produce (Signature),['Season 2 - Playing'],2023-07-15,['Adel and Wharfedale']
 ,,336,2023-07-15,"Yorkshire Dance, Performing Genders",['Partner Programme - Event'],"Yorkshire Dance, Performing Genders: Yorkshire Dance, Performing Genders",,['LS2 7UP'],"Yorkshire Dance, Performing Genders",Partner,['Season 2 - Playing'],2023-07-15,['Little London and Woodhouse']
@@ -1068,7 +1062,6 @@ actual_audience_size_number_of_participants_in_person,actual_audience_size_numbe
 ,,1262,2023-09-08,Culture Club,"['Public Event - Performance', 'CLE - Community Engagement session']",Hidden Stories: Culture Club,,['LS9 7LN'],Hidden Stories,CLE,['Season 3 - Dreaming'],2023-09-08,
 ,,1936,2023-09-09,Any Work That Wanted Doing: Drop-In Printing and Stitching,['CLE - Community Engagement session'],Hidden Stories: Any Work That Wanted Doing: Drop-In Printing and Stitching,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-09,['Little London and Woodhouse']
 250.0,,2109,2023-09-09,Family Flourish - Harvest ,,12 - My LEEDS 2023: Family Flourish - Harvest ,,,12 - My LEEDS 2023,Produce (Signature),,2023-09-09,
-250.0,,2109,2023-09-09,Family Flourish - Harvest ,,12 - My LEEDS 2023: Family Flourish - Harvest ,,,12 - My LEEDS 2023,Produce (Signature),,2023-09-09,
 175.0,,1792,2023-09-17,Heritage Open Day Hub ,"['CLE - Community Engagement session', 'Public Event - Exhibition']",Hidden Stories: Heritage Open Day Hub ,,['LS1 7BR'],Hidden Stories,CLE,,2023-09-09,['Little London and Woodhouse']
 ,,616,2023-09-09,Launchpad,['Public Event - Performance'],02 - Making A Stand: Launchpad,,['LS1 2HA'],02 - Making A Stand,Produce (Signature),['Season 3 - Dreaming'],2023-09-09,['Little London and Woodhouse']
 ,,1754,2023-09-09,MOON PALACE Daytime Observation Session at Victoria Sq,['Public Event - Installation'],09 - Smeaton300: Moon Palace East Leeds Observatory: MOON PALACE Daytime Observation Session at Victoria Sq,,,09 - Smeaton300: Moon Palace East Leeds Observatory,Produce (Signature),['Season 2 - Playing'],2023-09-09,
@@ -1101,13 +1094,11 @@ actual_audience_size_number_of_participants_in_person,actual_audience_size_numbe
 ,,1943,2023-09-15,Royal Armouries Takeover,,Hidden Stories: Royal Armouries Takeover,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-15,['Little London and Woodhouse']
 ,,293,2023-09-16,"AgeOfThe, Visions in the Map",['Partner Programme - Event'],"AgeOfThe, Visions in the Map: AgeOfThe, Visions in the Map",,,"AgeOfThe, Visions in the Map",Partner,['Season 3 - Dreaming'],2023-09-16,
 1083.0,,287,2023-09-16,Civic Skateboarding,"['Public Event - Large Scale Outdoor Event', 'CLE - Community Engagement session']",12 - My LEEDS 2023: Civic Skateboarding,,"['LS2 9JT', 'LS2 7BS', 'LS2 7UP']",12 - My LEEDS 2023,CLE,['Season 3 - Dreaming'],2023-09-16,"['Little London and Woodhouse', 'Little London and Woodhouse', 'Little London and Woodhouse']"
-1083.0,,287,2023-09-16,Civic Skateboarding,"['Public Event - Large Scale Outdoor Event', 'CLE - Community Engagement session']",12 - My LEEDS 2023: Civic Skateboarding,,"['LS2 9JT', 'LS2 7BS', 'LS2 7UP']",12 - My LEEDS 2023,CLE,['Season 3 - Dreaming'],2023-09-16,"['Little London and Woodhouse', 'Little London and Woodhouse', 'Little London and Woodhouse']"
 ,,1947,2023-09-16,Ken Morris: Wigs and Costume Workshop,['CLE - Community Engagement session'],Hidden Stories: Ken Morris: Wigs and Costume Workshop,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-16,['Little London and Woodhouse']
 ,,1255,2024-01-01,Lens Lab Exhibition,"['Public Event - Exhibition', 'CLE - Community Engagement session']",Hidden Stories: Lens Lab Exhibition,,,Hidden Stories,CLE,['Season 3 - Dreaming'],2023-09-16,
 ,,1944,2023-09-16,Try Your Hand at Ceramic Sculpture,['CLE - Community Engagement session'],Hidden Stories: Try Your Hand at Ceramic Sculpture,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-16,['Little London and Woodhouse']
 ,,1945,2023-09-16,Try Your Hand at Stone Carving,['CLE - Community Engagement session'],Hidden Stories: Try Your Hand at Stone Carving,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-16,['Little London and Woodhouse']
 203.0,,2108,2023-09-17,Civic Skateboarding,"['Public Event - Large Scale Outdoor Event', 'CLE - Community Engagement session']",12 - My LEEDS 2023: Civic Skateboarding,,"['LS10 1LT', 'LS12 2DR']",12 - My LEEDS 2023,CLE,['Season 3 - Dreaming'],2023-09-17,"['Hunslet and Riverside', 'Armley']"
-203.0,,2108,2023-09-17,Civic Skateboarding,"['Public Event - Large Scale Outdoor Event', 'CLE - Community Engagement session']",12 - My LEEDS 2023: Civic Skateboarding,,"['LS10 1LT', 'LS12 2DR']",12 - My LEEDS 2023,CLE,['Season 3 - Dreaming'],2023-09-17,"['Hunslet and Riverside', 'Armley']"
 ,,1946,2023-09-17,Pablo Fanque’s Circus Games,['CLE - Community Engagement session'],Hidden Stories: Pablo Fanque’s Circus Games,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-17,['Little London and Woodhouse']
 ,,1937,2023-09-17,Pablo Fanque’s Circus Games,['CLE - Community Engagement session'],Hidden Stories: Pablo Fanque’s Circus Games,,['LS1 7BR'],Hidden Stories,,['Season 3 - Dreaming'],2023-09-17,['Little London and Woodhouse']
 ,,1353,2023-09-20,Early Career Music Promoters CPWM gig - Caser ,['Partner Programme - Event'],Skills Development: Early Career Music Promoters CPWM gig - Caser ,,,Skills Development,,['Season 3 - Dreaming'],2023-09-20,
@@ -1137,7 +1128,6 @@ actual_audience_size_number_of_participants_in_person,actual_audience_size_numbe
 ,,1751,2025-10-31,Smeaton300 at Leeds Libraries: Code Club Smeaton Challenge [dates and location tbc],['Public Event - Installation'],09 - Smeaton300: Learning programme: Smeaton300 at Leeds Libraries: Code Club Smeaton Challenge [dates and location tbc],,,09 - Smeaton300: Learning programme,Produce (Signature),['Season 3 - Dreaming'],2023-09-25,
 ,,1744,2023-09-26, Community CPD Session - North West Workshop 2,"['CLE - Skills and Talent session', 'CLE - Community Engagement session']",General CLE:  Community CPD Session - North West Workshop 2,,['LS6 1LJ'],General CLE,CLE,['Season 3 - Dreaming'],2023-09-26,['Headingley and Hyde Park']
 ,18.0,1790,2023-09-26,Leeds Language Week - Programme Launch,,Creative Learning S3: Leeds Language Week - Programme Launch,,,Creative Learning S3,,,2023-09-26,
-,18.0,1790,2023-09-26,Leeds Language Week - Programme Launch,,Creative Learning S3: Leeds Language Week - Programme Launch,,,Creative Learning S3,,,2023-09-26,
 ,,1975,2023-09-26,Leeds Language Week: Programme Launch,['CLE - Creative Learning session'],Creative Learning S3: Leeds Language Week: Programme Launch,,,Creative Learning S3,CLE,['Season 3 - Dreaming'],2023-09-26,
 ,,1669,2023-09-27,The Magician and The Good Book double-screening,['Partner Programme - Event'],Slung Low: The Magician and The Good Book double-screening,,['LS1 5AT'],Slung Low,Partner,['Season 3 - Dreaming'],2023-09-27,['Little London and Woodhouse']
 ,,1131,2023-09-28,Digital Upskilling Workshop 17/20,['CLE - Skills and Talent session'],Creative Learning S3: Digital Upskilling Workshop 17/20,,['LS27 8PZ'],Creative Learning S3,CLE,['Season 3 - Dreaming'],2023-09-28,['Morley South']
@@ -1152,12 +1142,6 @@ actual_audience_size_number_of_participants_in_person,actual_audience_size_numbe
 1.0,,2104,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
 12.0,,626,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
 1.0,,2105,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
-1.0,,2101,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
-1.0,,2103,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
-1.0,,2102,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
-1.0,,2104,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
-12.0,,626,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
-1.0,,2105,2023-09-30,Smeaton 300: STEAM Conference ,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton 300: STEAM Conference ,,['LS17 8GS'],09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-09-30,['Alwoodley']
 ,,414,2023-10-31,East Street Arts: R,['Partner Programme - Event'],"East Street Arts, A City Less Grey 2023 - Add Fuel: East Street Arts: R",,,"East Street Arts, A City Less Grey 2023 - Add Fuel",Partner,['Season 3 - Dreaming'],2023-10-01,
 80.0,,2038,2023-12-31,Smeaton CREST Awards - Discovery Level,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton CREST Awards - Discovery Level,,,09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-10-02,
 280.0,,2037,2023-12-31,Smeaton CREST Awards - Discovery Level,['CLE - Creative Learning session'],09 - Smeaton300: Learning programme: Smeaton CREST Awards - Discovery Level,,,09 - Smeaton300: Learning programme,CLE,['Season 3 - Dreaming'],2023-10-02,

diff --git a/data/metrics/events/tickets/orders.csv b/data/metrics/events/tickets/orders.csv
@@ -4826,3 +4826,4 @@ created_at,event_id,event_date,number_of_tickets,ward_code
 2023-10-10 10:36:26,ev_3052122,2023-11-17,1,E05012647
 2023-10-10 10:38:46,ev_3052129,2023-12-19,1,E05013830
 2023-10-10 10:42:15,ev_3052125,2023-12-02,1,E05011399
+2023-10-10 11:32:05,ev_3052123,2023-11-25,1,E05011413
diff --git a/scripts/metrics/events/scheduled/dvc.lock b/scripts/metrics/events/scheduled/dvc.lock
@@ -14,7 +14,7 @@ stages:
     outs:
     - path: working/metrics/events/all.csv
       hash: md5
-      md5: f9cb49fa9127503b7dcd220476480e47
+      md5: 13b23d52156a18f353eaf80b30e98315
       size: 353572
   transform:
     cmd: PYTHONPATH=. python3 scripts/metrics/events/scheduled/transform.py
@@ -30,7 +30,7 @@ stages:
       size: 676
     - path: working/metrics/events/all.csv
       hash: md5
-      md5: f9cb49fa9127503b7dcd220476480e47
+      md5: 13b23d52156a18f353eaf80b30e98315
       size: 353572
     outs:
     - path: data/metrics/events/master/all.csv

diff --git a/scripts/metrics/events/signature/dvc.lock b/scripts/metrics/events/signature/dvc.lock
@@ -83,8 +83,8 @@ stages:
       size: 281269
     - path: ../../../../data/metrics/events/tickets/orders.csv
       hash: md5
-      md5: a122c429bead48e21cb862108a3a4435
-      size: 259805
+      md5: a72d72b5bb1648c8b8a190705f86c858
+      size: 259859
     - path: events.py
       hash: md5
       md5: bb7c8d94865aac1cdb8f8a5e3ac6045e

diff --git a/scripts/metrics/events/ticketing/dvc.lock b/scripts/metrics/events/ticketing/dvc.lock
@@ -18,15 +18,15 @@ stages:
     outs:
     - path: ../../../../working/metrics/ticketing/orders.csv
       hash: md5
-      md5: 0a267540de2e5ea03f658e657a98ebd8
-      size: 571913
+      md5: 5e3d28c2ec819f198a20b12da353c083
+      size: 572052
   transform:
     cmd: papermill tickets.ipynb ../../../../working/output/tickets-transform.ipynb
     deps:
     - path: ../../../../working/metrics/ticketing/orders.csv
       hash: md5
-      md5: 0a267540de2e5ea03f658e657a98ebd8
-      size: 571913
+      md5: 5e3d28c2ec819f198a20b12da353c083
+      size: 572052
     - path: pipes.py
       hash: md5
       md5: 564a48f276d11d6a64b72b71f5e50c66
@@ -38,5 +38,5 @@ stages:
     outs:
     - path: ../../../../data/metrics/events/tickets/orders.csv
       hash: md5
-      md5: a122c429bead48e21cb862108a3a4435
-      size: 259805
+      md5: a72d72b5bb1648c8b8a190705f86c858
+      size: 259859