-
Notifications
You must be signed in to change notification settings - Fork 2
/
cron.yaml
92 lines (92 loc) · 4.56 KB
/
cron.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
cron:
# These rely on 701004
#TODO: API CUTOFF
#- description: reload future facebook event data
# url: /tasks/reload_events?user_id=701004&allow_cache=0&only_if_updated=1&time_period=FUTURE&disable_updates=&queue=slow-queue&vertical=STREET
# schedule: every 3 hours
# If we are trying to run this quickly, we may be interested in using disable_updates=regeocode,geodata,photo,index depending:
# www.dancedeets.com/tasks/reload_events?user_id=701004&allow_cache=1&only_if_updated=0&disable_updates=regeocode,photo,geodata,index&queue=fast-queue
#
# These load potential events from all the various pages/groups out there
#TODO: API CUTOFF
#- description: load potential events from special users and sources
# url: /tasks/scrape_sources_and_process_events?user_id=dummy&queue=fast-queue
# schedule: every day 12:00
# These also load potential events, but from the /events of our logged-in users
#TODO: API CUTOFF
#- description: load potential events from our users
# url: /tasks/load_all_potential_events?user_id=dummy&queue=slow-queue
# schedule: every 8 hours
#
# These run through all our collected potential events and reload them, checking for what should be included.
# They will periodically refresh the event data to check for the latest values.
# Many times events get discovered in one of the earlier stages, but *too* early, when keywords/attendees don't trigger yet.
# This mapreduces is the "cleanup" that run through and reload them and guarantee they get rechecked.
# We could run it with expiry_days, but what's the point of re-running a classifier with cached data?
# Instead, let's run it with no cache, reloading all fb event/attending data, to then re-classify.
#TODO: API CUTOFF
#- description: auto-add events
# url: /tools/auto_add_potential_events?user_id=dummy&past_event=0&allow_cache=0&queue=slow-queue
# schedule: every 24 hours
#
#
- description: reload user data
url: /tasks/reload_all_users?user_id=dummy&allow_cache=0&queue=slow-queue
schedule: every day 18:00
- description: reload all user data
url: /tasks/reload_all_users?user_id=dummy&allow_cache=0&queue=slow-queue&all_users=1
schedule: every monday 10:00
- description: email users about upcoming events
url: /tasks/email_all_users?user_id=dummy
schedule: every monday 12:00
timezone: America/Los_Angeles
- description: post to social networks
url: /tasks/social_publisher
schedule: every 1 minutes
#TODO: API CUTOFF
#- description: search fb for more events
# url: /tools/search_fb_for_events?user_id=random
# schedule: every 8 hours
# This seems to be too spammy, and has a negative effect on users :(
# So let's disable it for now, until we can find a better threshold/metric for whether to notify users
#- description: notify about newly-added events
# url: /tasks/promote_new_events
# schedule: every 1 hours
# Disabled because we no longer show classes
#- description: scrape all classes and index them
# url: /tasks/crawl_and_index_classes
# schedule: every 6 hours
- description: scrape all websites for events
url: /tasks/web_events/start_spiders
schedule: every 12 hours
#TODO: API CUTOFF
#- description: calculate periodic event statistics and rankings
# url: /tasks/compute_event_rankings?vertical=STREET # runs on slow-queue
# schedule: every day 10:00
- description: calculate periodic user statistics and rankings
url: /tasks/compute_user_rankings # runs on slow-queue
schedule: every day 10:00
#- description: reload past facebook event data
# url: /tasks/reload_events?user_id=dummy&allow_cache=0&time_period=FUTURE
# schedule: every monday 2:00
# Disabled in favor of the Dataflow pipeline, that will need to be called on this schedule
#- description: compute popular organizers/dancers
# url: /tools/popular_people?user_id=701004
# # twice a month
# schedule: 1,15 of month 09:00
# Disabled since weekly posts seem to get super-low engagement, and this image stitching doesn't seem to be that engaging or work that well embedded on FB feeds
#- description: queue up our weekly city posts
# url: /tasks/weekly_posts?user_id=701004&limit=30
# schedule: every monday 12:00
- description: generate sitemaps/*-*.xml
url: /tasks/generate_sitemaps?user_id=701004&vertical=STREET
schedule: every tuesday 12:00
- description: generate sitemaps/recent.xml
url: /tasks/generate_sitemaps?user_id=701004&time_period=FUTURE&vertical=STREET
schedule: every day 12:00
- description: start posting about events N days out
url: /tasks/prepare_event_notifications?days=3&min_dancers=50&min_attendees=100
schedule: every day 10:00
- description: scrape jwjam
url: /tasks/scrape_apis/jwjam
schedule: every day 12:00