diff --git a/.cfignore b/.cfignore index f043920..5dee6e7 100644 --- a/.cfignore +++ b/.cfignore @@ -81,8 +81,8 @@ ENV/ # Spyder project settings .spyderproject -# DB Files -*/staticfiles/ +# StaticFiles +application/staticfiles/ # App Specific Files application/staticfiles/ @@ -90,49 +90,48 @@ cover/ test.py dummy_data.txt -#CodeDeploy -codedeploy/certificate.crt -codedeploy/data.zip -codedeploy/data_.zip -codedeploy/deploy.sh -codedeploy/feedcrunch.xyz.csr -codedeploy/https.key -https/deploy.sh -fieldkeys/* -!fieldkeys/__init__.py +# Fixtures +application/fixtures/feedcrunch_dump8.json +application/fixtures/feedcrunch_dump9.json application/fixtures/dataradar.json + +# SQLite DB *.sqlite3 + +# Media Files +media/ !media/images/user_photos/__init__.py !media/images/user_photos/dummy_user.png -media/images/user_photos/ -codedeploy/Encrypted/*.zip -codedeploy/Encrypted/*.txt -*.model -FeedCrunch.IO.zip -boto-develop/build/ -application/fixtures/feedcrunch_dump8.json -application/fixtures/feedcrunch_dump9.json -media/images/interest_photos/ -# CloudFoundry Remove +# Templates _unused_/ -codedeploy/ example_files/ + +# Keys and Certificates certificate/ -fieldkeys/ -lib_bin/ -media/ -scripts/ -venv/ +.crt +.key + +# CI Chain .codeclimate.yml -.coveragerc -.env -.env.dist -.gitignore -circle.yml .travis.yml -app.json -appspec.yml +.pyup.yml +*.yml + +# Python Testing pytest.ini +.coverage +.coveragerc + +# Git Files README.md -*.yml \ No newline at end of file +CODE_OF_CONDUCT.md +CONTRIBUTING.md +LICENSE.md +.env.dist +.gitignore +.github/ +.git/ + +# Binary Folders +lib_bin/ diff --git a/.coveragerc b/.coveragerc index 2e827d4..73694c6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,4 +2,10 @@ omit = */migrations/* venv/* - /home/travis/virtualenv/*/lib/*/site-packages/* + /home/travis/virtualenv/*/lib/*/site-packages/* + python3.4.6/* + python3.6.3/* + staticfiles/* + tests/* + lib/* + django_celery_monitor/* \ No newline at end of file diff --git a/.env.dist b/.env.dist index 75b1d33..a097a34 100644 --- a/.env.dist +++ b/.env.dist @@ -1,5 +1,8 @@ DATABASE_URL='postgres://user:password@server:port/dbname' -RABBITMQ_URL='amqp://username:password@server:port/instance_name' +RABBITMQ_URL='amqps://username:password@server:port/instance_name' +REDIS_URL='redis://username:password@server:port' +USE_RABBITMQ=True + SECRET_KEY='################################' DEBUG=True AWS_USER='################################' diff --git a/.gitignore b/.gitignore index 1232206..e3aae74 100644 --- a/.gitignore +++ b/.gitignore @@ -76,13 +76,14 @@ celerybeat-schedule # virtualenv venv/ +venv_linux/ ENV/ # Spyder project settings .spyderproject -# DB Files -*/staticfiles/ +# StaticFiles +application/staticfiles/ # App Specific Files application/staticfiles/ @@ -90,35 +91,41 @@ cover/ test.py dummy_data.txt -#CodeDeploy -codedeploy/certificate.crt -codedeploy/data.zip -codedeploy/data_.zip -codedeploy/deploy.sh -codedeploy/feedcrunch.xyz.csr -codedeploy/https.key -https/deploy.sh -fieldkeys/* -!fieldkeys/__init__.py -application/fixtures/dataradar.json +# Fixtures +application/fixtures/* +!application/fixtures/.gitkeep + +# SQLite DB *.sqlite3 -!media/images/user_photos/__init__.py + +# Media Files +!media/ +media/* +!media/*/ +media/*/* +!media/*/*/ +media/*/*/* +!media/images/user_photos/.gitkeep !media/images/user_photos/dummy_user.png -media/images/user_photos/ -codedeploy/Encrypted/*.zip -codedeploy/Encrypted/*.txt -*.model -FeedCrunch.IO.zip -boto-develop/build/ -application/fixtures/feedcrunch_dump8.json -application/fixtures/feedcrunch_dump9.json -media/images/interest_photos/ -scripts/django_q.sql -/flower* -/celery* -Procfile -venv_linux/ -.idea/ -.key +!media/images/interest_photos/.gitkeep +!media/images/interest_photos/dummy_stock.jpg +!media/estimators/.gitkeep + +# Templates +_unused_/ +example_files/ + +# Keys and Certificates +certificate/ .crt -coverage.xml +.key + +# Python Testing +.coverage + +# PyCharm +.idea +celeryev.pid +celerybeat.pid +fieldkeys/ +.crt* diff --git a/.travis.yml b/.travis.yml index 5d1cc83..129ba1b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,9 +12,11 @@ python: services: - rabbitmq +- redis-server - postgresql before_install: +- python -c "import fcntl; fcntl.fcntl(1, fcntl.F_SETFL, 0)" - export DJANGO_SETTINGS_MODULE=application.settings - export PYTHONPATH=$HOME/builds/DEKHTIARJonathan/FeedCrunch.IO - virtualenv venv @@ -51,6 +53,7 @@ env: global: - RABBITMQ_URL='amqp://guest:guest@localhost:5672/' + - REDIS_URL='redis://localhost:6379' - DATABASE_URL='postgres://postgres:@localhost:5432/travisci' - DEBUG='True' - EMAIL_DEFAULT_SENDER='local@local.host' @@ -60,13 +63,14 @@ env: - BLUEMIX_API_GATEWAY='https://api.eu-gb.bluemix.net' - BLUEMIX_API_ORGANISATION='FeedCrunch' - BLUEMIX_API_SPACE='prod' - + - USE_RABBITMQ='False' + ### == CODECOV_TOKEN === ### - secure: "sRuP0d5h9pZRVHMo+8TjOdx80rJEb9GiR26uEM2dZZvm3oh+jfHbiqSecaLa3yo6qgyLwz3doSZx1eu7ZW6RDujEHHSg/SdHkosLzo3qKE1Mr5XPJXLtwjXcvSVYhiMVS5E/IcrOKqEi5p694GsykLwz5jVb6orCh/O8UcHHb2NP99JhKWcxFRjEpPoitNmTcigd1VJjqf7MiMq8+GqEhe4vug9XCnyUgpUM7okOkbem2DPaABwEJfZC9f0HVlJa1X1r1mGAFJuPqJpJ0sOiimZo9k0yfBSGjTCWmr+aga1ASjMJUKBCiEk+yEAaLwu6w0Zr+nYuIhLeLWDxVwJpmUBs0QFM4RX4Q+Pt3hUs9ynCWrXfcR/1hNdQD76705Bs9/4TxBwT7sOMfdVsywcpWtUPtfFTYmQL1sBMnz1GOg4ueZQHL7+GaARaZs+5XuWiaNIP0/xmFE8C+EHm2OqesgwYD/rdJxYG5VMyfyrjsfEJrZLx0TV8e9xbhEx3x9ZeTJrBV24/qDyYT7Ekn2kY+t8nA7V8NSKkleMiT+5D81PyNLJ0+YJjw/Ot+DIT0U6IkuSi5Y0eQMGq+mRfLGGY3YNCSeo1MTxhy5EO3pX4mwVJZt2QwOQwzqoJyGM8foNu6NH1hBjX6IH3B0smpmLoPoY3TCEffZOYjpSoGOqBQsE=" - + ### === CODACY_PROJECT_TOKEN === ### - secure: "MJ6PAxlkD+Vch1QcAyJB/isYfzqs/fgVYp5dM/Rq2pj/sisl3TtpkdNyiddlFMONH+BkxLxUnBng4PaJewrnWc6ojUlarSolyYPO+nUiy0if3XPNHtcdoUgR97CGWfrm1U+3Rb/gDN3RNDTmWveISB8B2d3j00dUvQEwdYBmK48RsJ1z0a0bjc4E3D2Ns0RiOsp4kWi+CEsMG7r0AVWpdOgusiVnx4bWgiALQJclJS6xkszB63PjIjNyscWr79KE6YFmYxP0aEOtLh/u7FAuWCWM7oFEvXp+b4gOZi+dH1SeuKpNO1Cm+6O6mR0bfnrYlrJUr8m0USh4Uoqz8Qexn8SxRtR1cFQENyDzrTqi97aInmA9ZtiaAXRDy6gByOZ9uP0SYgqdF2+R0xNGQsVF6mo7vBqDynpqMp39Bi1DfceFP2/7qLIgsiBvSEH9fj903RAg4rlra16JiQKw2jwHN/S3I0PBvkV20AlgAyFjBAcIdvVpi1OjRIOXKPKPZPB3tyc940U0z3tHSJoWDlKWv5jq1W6YvmzV+bqVpHCAoKatQlcePhKv0WwQC0sFQnfs4JgYqBrpQunSzof4l62NsOsW/9Ji7NBQnWadFmTtoyC4pXRDYitq6ZURRasQS929V9OpzAFQbH55guB6RYyRuBdfCJlQlepZOy40ZV9A+5w=" - + ### === BluemixPassword === ### - secure: "pCcGxYi4rP0giVfguOMDrH6nq++v5xx0geb1pBQT6C6awF6NoFWfkrZXQSGqx5/G2bP5F4G+djTP0c441bzSSPrUT+oqW47vdrg6vqYRVin/YZFQRlutoHPcckjyyWep2aa5qAbp31yWXCol3MwQPupQl90yQeXKUk4GI9HP3+h9wEKeus0x+RZvLuKVINWNlRWMaIHtnR1J8xTrzCCFqQR4L6Kknv80sxOfTMFyaEKv8dUdAHzrV87J+tYQAaJoKAsMLqdagX88TBK9uRxXmfAX33RCUIdgDlG6zorRFcWG+xEvDcBaygxayZ3lc0niqJfAxJqFwzW5ORjYowzXqlfrswpp49djcTUe+fJePLUeApR92BAVXlr12xrbhYBG9BCaOlPvE5JCtmS2BZpGtuZa3XQ1PD9+R0mu9wFz3YgvFMS1WsFhqpauSp4sTmChvpV64jhkbZtjhIymEdOOxN7r9QNetr8P2JyMT5k86TYl0p7+P1O62ru8tqycujIMVLd5KeyH08K71aNmAjdn46Ft5g3sX+rZm7L19EsFRkojTSh7QY6hXW3YuCX3pVQTR25ZqkyZKFV6aFCP0kio7MdN5VcK9iIvDA15pV/ElFNEVMiGKwkC2MFzGD+Qb6MGqE7I4ShwjdU0qLoey3+KJX5pnGYX+CA6Jz2ac5vW6YM=" diff --git a/application/celery.py b/application/celery.py index 5f653a6..cedc477 100644 --- a/application/celery.py +++ b/application/celery.py @@ -29,7 +29,6 @@ def load_env(): from django.conf import settings from django.apps import AppConfig - # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'application.settings') @@ -48,12 +47,13 @@ def load_env(): app.conf.broker_use_ssl = settings.BROKER_USE_SSL app.conf.accept_content = settings.CELERY_ACCEPT_CONTENT app.conf.timezone = settings.CELERY_TIMEZONE +app.conf.enable_utc = settings.CELERY_ENABLE_UTC # Worker settings -app.conf.worker_concurrency = settings.CELERYD_CONCURRENCY +app.conf.worker_concurrency = settings.CELERY_CONCURRENCY # Results settings -#app.conf.result_backend = settings.CELERY_RESULT_BACKEND +#app.conf.result_backend = settings.CELERY_RESULT_BACKEND app.conf.result_serializer = settings.CELERY_RESULT_SERIALIZER app.conf.result_expires = settings.CELERY_TASK_RESULT_EXPIRES @@ -61,20 +61,25 @@ def load_env(): app.conf.task_serializer = settings.CELERY_TASK_SERIALIZER app.conf.task_acks_late = settings.CELERY_TASK_ACKS_LATE app.conf.task_reject_on_worker_lost = settings.CELERY_TASK_REJECT_ON_WORKER_LOST -app.conf.task_time_limit = settings.CELERYD_TASK_TIME_LIMIT -app.conf.task_soft_time_limit = settings.CELERYD_TASK_SOFT_TIME_LIMIT +app.conf.task_time_limit = settings.CELERY_TASK_TIME_LIMIT +app.conf.task_soft_time_limit = settings.CELERY_TASK_SOFT_TIME_LIMIT app.conf.task_always_eager = settings.CELERY_TASK_ALWAYS_EAGER +app.conf.task_queues = settings.CELERY_TASK_QUEUES + +# Event settings +app.conf.event_queue_ttl = settings.CELERY_EVENT_QUEUE_EXPIRES +app.conf.event_queue_expires = settings.CELERY_EVENT_QUEUE_TTL # Celery Beat Settings -app.conf.beat_scheduler = settings.CELERYBEAT_SCHEDULER -app.conf.beat_schedule = settings.CELERYBEAT_SCHEDULE -app.conf.beat_sync_every = settings.CELERYBEAT_SYNC_EVERY -app.conf.beat_max_loop_interval = settings.CELERYBEAT_MAX_LOOP_INTERVAL +app.conf.beat_scheduler = settings.CELERYBEAT_SCHEDULER +app.conf.beat_schedule = settings.CELERYBEAT_SCHEDULE +app.conf.beat_sync_every = settings.CELERYBEAT_SYNC_EVERY +app.conf.beat_max_loop_interval = settings.CELERYBEAT_MAX_LOOP_INTERVAL # Celery Monitor Settings -app.conf.monitors_expire_success = timedelta(hours=1) -app.conf.monitors_expire_error = timedelta(days=3) -app.conf.monitors_expire_pending = timedelta(days=5) +app.conf.monitors_expire_success = timedelta(hours=1) +app.conf.monitors_expire_error = timedelta(days=3) +app.conf.monitors_expire_pending = timedelta(days=5) class CeleryConfig(AppConfig): name = 'application' diff --git a/application/fixtures/.gitkeep b/application/fixtures/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/application/settings.py b/application/settings.py index 2134b2a..103c0a1 100644 --- a/application/settings.py +++ b/application/settings.py @@ -19,6 +19,7 @@ from datetime import timedelta from celery.schedules import crontab +from kombu import Exchange, Queue # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -119,8 +120,6 @@ def assign_env_value(var_name): ] THIRD_PARTY_APPS = [ - 'material', - 'material.admin', 'admin_view_permission', 'django_extensions', 'django_ses', @@ -309,32 +308,52 @@ def assign_env_value(var_name): ALLOWED_HOSTS = ['*'] # Celery Configuration -BROKER_URL = assign_env_value('RABBITMQ_URL') +if assign_env_value('USE_RABBITMQ'): + BROKER_URL = assign_env_value('RABBITMQ_URL') +else: + BROKER_URL = assign_env_value('REDIS_URL') + BROKER_USE_SSL=True CELERY_ACCEPT_CONTENT = ['application/json'] -CELERY_TIMEZONE = TIME_ZONE +CELERY_TIMEZONE = TIME_ZONE +CELERY_ENABLE_UTC = False -CELERYD_CONCURRENCY = 3 -#CELERY_RESULT_BACKEND = 'django-db' +CELERY_CONCURRENCY = 3 +#CELERY_RESULT_BACKEND = 'django-db' CELERY_RESULT_SERIALIZER = 'json' -CELERY_TASK_SERIALIZER = 'json' -CELERY_TASK_ACKS_LATE=True # Acknoledge pool when task is over -CELERY_TASK_REJECT_ON_WORKER_LOST=True -CELERY_TASK_RESULT_EXPIRES=7*24*30*30 +CELERY_TASK_SERIALIZER = 'json' +CELERY_TASK_ACKS_LATE = True # Acknoledge pool when task is over +CELERY_TASK_REJECT_ON_WORKER_LOST = True +CELERY_TASK_RESULT_EXPIRES = 3*24*60*60 # 3 Days -CELERYD_TASK_TIME_LIMIT=90 -CELERYD_TASK_SOFT_TIME_LIMIT=60 +CELERY_EVENT_QUEUE_EXPIRES = 60 +CELERY_EVENT_QUEUE_TTL = 5 + +CELERY_TASK_TIME_LIMIT = 90 +CELERY_TASK_SOFT_TIME_LIMIT = 60 if DEBUG or TESTING: CELERY_TASK_ALWAYS_EAGER = True else: CELERY_TASK_ALWAYS_EAGER = False -CELERYBEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' -CELERYBEAT_MAX_LOOP_INTERVAL=10 -CELERYBEAT_SYNC_EVERY=1 +CELERY_TASK_QUEUES = [ + Queue( + 'celery', + Exchange('celery'), + routing_key = 'celery', + queue_arguments = { + 'x-message-ttl': 60 * 1000 # 60 000 ms = 60 secs. + } + ) +] + +CELERYBEAT_SCHEDULER = 'django_celery_beat.schedulers:DatabaseScheduler' +CELERYBEAT_MAX_LOOP_INTERVAL = 10 +CELERYBEAT_SYNC_EVERY = 1 + CELERYBEAT_SCHEDULE = { 'refresh_all_rss_subscribers_count': { 'task': 'feedcrunch.tasks.refresh_all_rss_subscribers_count', diff --git a/application/urls.py b/application/urls.py index a6f0597..0f273f8 100644 --- a/application/urls.py +++ b/application/urls.py @@ -23,12 +23,14 @@ admin.autodiscover() urlpatterns = [ - url(r'^admin/', admin.site.urls), + url(r'^admin/django-ses/', include('django_ses.urls')), + url(r'^admin/', admin.site.urls), + url(r'^api/1.0/', include('feedcrunch_api_v1.urls')), url(r'^oauth/', include('oauth.urls')), url(r'^@(?P\w+)/admin/', include('feedcrunch_rssadmin.urls')), url(r'^@(?P\w+)/', include('feedcrunch_rssviewer.urls')), url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), url(r'', include('feedcrunch_home.urls')), -] +] \ No newline at end of file diff --git a/django_celery_monitor/__init__.py b/django_celery_monitor/__init__.py new file mode 100644 index 0000000..bd923bd --- /dev/null +++ b/django_celery_monitor/__init__.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +"""Celery monitor for Django.""" +# :copyright: (c) 2016, Ask Solem. +# All rights reserved. +# :license: BSD (3 Clause), see LICENSE for more details. + +from __future__ import absolute_import, unicode_literals + +import re + +from collections import namedtuple + +__version__ = '1.1.3' +__author__ = 'Jannis Leidel/Pieter De Decker' +__contact__ = 'jannis@leidel.info' +__homepage__ = 'https://github.com/pieterdd/django-celery-monitor' +__docformat__ = 'restructuredtext' + +# -eof meta- + +version_info_t = namedtuple('version_info_t', ( + 'major', 'minor', 'micro', 'releaselevel', 'serial', +)) + +# bumpversion can only search for {current_version} +# so we have to parse the version here. +_temp = re.match( + r'(\d+)\.(\d+).(\d+)(.+)?', __version__).groups() +VERSION = version_info = version_info_t( + int(_temp[0]), int(_temp[1]), int(_temp[2]), _temp[3] or '', '') +del(_temp) +del(re) + +__all__ = [] + +default_app_config = 'django_celery_monitor.apps.CeleryMonitorConfig' diff --git a/django_celery_monitor/admin.py b/django_celery_monitor/admin.py new file mode 100644 index 0000000..c6b5c6d --- /dev/null +++ b/django_celery_monitor/admin.py @@ -0,0 +1,260 @@ +"""Result Task Admin interface.""" +from __future__ import absolute_import, unicode_literals + +from __future__ import absolute_import, unicode_literals + +from django.contrib import admin +from django.contrib.admin import helpers +from django.contrib.admin.views import main as main_views +from django.shortcuts import render_to_response +from django.template import RequestContext +from django.utils.encoding import force_text +from django.utils.html import escape, format_html +from django.utils.translation import ugettext_lazy as _ + +from celery import current_app +from celery import states +from celery.task.control import broadcast, revoke, rate_limit + +from .models import TaskState, WorkerState +from .humanize import naturaldate +from .utils import action, display_field, fixedwidth, make_aware + + +TASK_STATE_COLORS = {states.SUCCESS: 'green', + states.FAILURE: 'red', + states.REVOKED: 'magenta', + states.STARTED: 'yellow', + states.RETRY: 'orange', + 'RECEIVED': 'blue'} +NODE_STATE_COLORS = {'ONLINE': 'green', + 'OFFLINE': 'gray'} + + +class MonitorList(main_views.ChangeList): + """A custom changelist to set the page title automatically.""" + + def __init__(self, *args, **kwargs): + super(MonitorList, self).__init__(*args, **kwargs) + self.title = self.model_admin.list_page_title + + +@display_field(_('state'), 'state') +def colored_state(task): + """Return the task state colored with HTML/CSS according to its level. + + See ``django_celery_monitor.admin.TASK_STATE_COLORS`` for the colors. + """ + state = escape(task.state) + color = TASK_STATE_COLORS.get(task.state, 'black') + return format_html('{1}', color, + state) + + +@display_field(_('state'), 'last_heartbeat') +def node_state(node): + """Return the worker state colored with HTML/CSS according to its level. + + See ``django_celery_monitor.admin.NODE_STATE_COLORS`` for the colors. + """ + state = node.is_alive() and 'ONLINE' or 'OFFLINE' + color = NODE_STATE_COLORS[state] + return format_html('{1}', color, + state) + + +@display_field(_('ETA'), 'eta') +def eta(task): + """Return the task ETA as a grey "none" if none is provided.""" + if not task.eta: + return format_html('none') + return escape(make_aware(task.eta)) + + +@display_field(_('when'), 'tstamp') +def tstamp(task): + """Better timestamp rendering. + + Converts the task timestamp to the local timezone and renders + it as a "natural date" -- a human readable version. + """ + value = make_aware(task.tstamp) + return format_html('
{1}
', str(value), + naturaldate(value)) + + +@display_field(_('name'), 'name') +def name(task): + """Return the task name and abbreviates it to maximum of 16 characters.""" + short_name = task.name + if task.name and len(task.name) > 16: + short_name = format_html('{0}…', task.name[0:15]) + return format_html('
{1}
', task.name, + short_name) + + +class ModelMonitor(admin.ModelAdmin): + """Base class for task and worker monitors.""" + + can_add = False + can_delete = False + + def get_changelist(self, request, **kwargs): + """Return the custom change list class we defined above.""" + return MonitorList + + def change_view(self, request, object_id, extra_context=None): + """Make sure the title is set correctly.""" + extra_context = extra_context or {} + extra_context.setdefault('title', self.detail_title) + return super(ModelMonitor, self).change_view( + request, object_id, extra_context=extra_context, + ) + + def has_delete_permission(self, request, obj=None): + """Short-circuiting the permission checks based on class attribute.""" + if not self.can_delete: + return False + return super(ModelMonitor, self).has_delete_permission(request, obj) + + def has_add_permission(self, request): + """Short-circuiting the permission checks based on class attribute.""" + if not self.can_add: + return False + return super(ModelMonitor, self).has_add_permission(request) + + +@admin.register(TaskState) +class TaskMonitor(ModelMonitor): + """The Celery task monitor.""" + + detail_title = _('Task detail') + list_page_title = _('Tasks') + rate_limit_confirmation_template = ( + 'django_celery_monitor/confirm_rate_limit.html' + ) + date_hierarchy = 'tstamp' + fieldsets = ( + (None, { + 'fields': ('state', 'task_id', 'name', 'args', 'kwargs', + 'eta', 'runtime', 'worker', 'tstamp'), + 'classes': ('extrapretty', ), + }), + ('Details', { + 'classes': ('collapse', 'extrapretty'), + 'fields': ('result', 'traceback', 'expires'), + }), + ) + list_display = ( + fixedwidth('task_id', name=_('UUID'), pt=8), + colored_state, + name, + fixedwidth('args', pretty=True), + fixedwidth('kwargs', pretty=True), + eta, + tstamp, + 'worker', + ) + readonly_fields = ( + 'state', 'task_id', 'name', 'args', 'kwargs', + 'eta', 'runtime', 'worker', 'result', 'traceback', + 'expires', 'tstamp', + ) + list_filter = ('state', 'name', 'tstamp', 'eta', 'worker') + search_fields = ('name', 'task_id', 'args', 'kwargs', 'worker__hostname') + actions = ['revoke_tasks', + 'terminate_tasks', + 'kill_tasks', + 'rate_limit_tasks'] + + class Media: + """Just some extra colors.""" + + css = {'all': ('django_celery_monitor/style.css', )} + + @action(_('Revoke selected tasks')) + def revoke_tasks(self, request, queryset): + with current_app.default_connection() as connection: + for state in queryset: + revoke(state.task_id, connection=connection) + + @action(_('Terminate selected tasks')) + def terminate_tasks(self, request, queryset): + with current_app.default_connection() as connection: + for state in queryset: + revoke(state.task_id, connection=connection, terminate=True) + + @action(_('Kill selected tasks')) + def kill_tasks(self, request, queryset): + with current_app.default_connection() as connection: + for state in queryset: + revoke(state.task_id, connection=connection, + terminate=True, signal='KILL') + + @action(_('Rate limit selected tasks')) + def rate_limit_tasks(self, request, queryset): + tasks = set([task.name for task in queryset]) + opts = self.model._meta + app_label = opts.app_label + if request.POST.get('post'): + rate = request.POST['rate_limit'] + with current_app.default_connection() as connection: + for task_name in tasks: + rate_limit(task_name, rate, connection=connection) + return None + + context = { + 'title': _('Rate limit selection'), + 'queryset': queryset, + 'object_name': force_text(opts.verbose_name), + 'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME, + 'opts': opts, + 'app_label': app_label, + } + + return render_to_response( + self.rate_limit_confirmation_template, context, + context_instance=RequestContext(request), + ) + + def get_actions(self, request): + actions = super(TaskMonitor, self).get_actions(request) + actions.pop('delete_selected', None) + return actions + + def get_queryset(self, request): + qs = super(TaskMonitor, self).get_queryset(request) + return qs.select_related('worker') + + +@admin.register(WorkerState) +class WorkerMonitor(ModelMonitor): + """The Celery worker monitor.""" + + can_add = True + detail_title = _('Node detail') + list_page_title = _('Worker Nodes') + list_display = ('hostname', node_state) + readonly_fields = ('last_heartbeat', ) + actions = ['shutdown_nodes', + 'enable_events', + 'disable_events'] + + @action(_('Shutdown selected worker nodes')) + def shutdown_nodes(self, request, queryset): + broadcast('shutdown', destination=[n.hostname for n in queryset]) + + @action(_('Enable event mode for selected nodes.')) + def enable_events(self, request, queryset): + broadcast('enable_events', + destination=[n.hostname for n in queryset]) + + @action(_('Disable event mode for selected nodes.')) + def disable_events(self, request, queryset): + broadcast('disable_events', + destination=[n.hostname for n in queryset]) + + def get_actions(self, request): + actions = super(WorkerMonitor, self).get_actions(request) + actions.pop('delete_selected', None) + return actions diff --git a/django_celery_monitor/apps.py b/django_celery_monitor/apps.py new file mode 100644 index 0000000..b2806bb --- /dev/null +++ b/django_celery_monitor/apps.py @@ -0,0 +1,15 @@ +"""Application configuration.""" +from __future__ import absolute_import, unicode_literals + +from django.apps import AppConfig +from django.utils.translation import ugettext_lazy as _ + +__all__ = ['CeleryMonitorConfig'] + + +class CeleryMonitorConfig(AppConfig): + """Default configuration for the django_celery_monitor app.""" + + name = 'django_celery_monitor' + label = 'celery_monitor' + verbose_name = _('Celery Monitor') diff --git a/django_celery_monitor/camera.py b/django_celery_monitor/camera.py new file mode 100644 index 0000000..08f3a6c --- /dev/null +++ b/django_celery_monitor/camera.py @@ -0,0 +1,139 @@ +"""The Celery events camera.""" +from __future__ import absolute_import, unicode_literals + +from datetime import timedelta + +from celery import states +from celery.events.snapshot import Polaroid +from celery.utils.imports import symbol_by_name +from celery.utils.log import get_logger +from celery.utils.time import maybe_iso8601 + +from .utils import fromtimestamp, correct_awareness + +WORKER_UPDATE_FREQ = 60 # limit worker timestamp write freq. +SUCCESS_STATES = frozenset([states.SUCCESS]) + +NOT_SAVED_ATTRIBUTES = frozenset(['name', 'args', 'kwargs', 'eta']) + +logger = get_logger(__name__) +debug = logger.debug + + +class Camera(Polaroid): + """The Celery events Polaroid snapshot camera.""" + + clear_after = True + worker_update_freq = WORKER_UPDATE_FREQ + + def __init__(self, *args, **kwargs): + super(Camera, self).__init__(*args, **kwargs) + # Expiry can be timedelta or None for never expire. + self.app.add_defaults({ + 'monitors_expire_success': timedelta(days=1), + 'monitors_expire_error': timedelta(days=3), + 'monitors_expire_pending': timedelta(days=5), + }) + + @property + def TaskState(self): + """Return the data model to store task state in.""" + return symbol_by_name('django_celery_monitor.models.TaskState') + + @property + def WorkerState(self): + """Return the data model to store worker state in.""" + return symbol_by_name('django_celery_monitor.models.WorkerState') + + def django_setup(self): + import django + django.setup() + + def install(self): + super(Camera, self).install() + self.django_setup() + + @property + def expire_task_states(self): + """Return a twople of Celery task states and expiration timedeltas.""" + return ( + (SUCCESS_STATES, self.app.conf.monitors_expire_success), + (states.EXCEPTION_STATES, self.app.conf.monitors_expire_error), + (states.UNREADY_STATES, self.app.conf.monitors_expire_pending), + ) + + def get_heartbeat(self, worker): + try: + heartbeat = worker.heartbeats[-1] + except IndexError: + return + return fromtimestamp(heartbeat) + + def handle_worker(self, hostname_worker): + hostname, worker = hostname_worker + return self.WorkerState.objects.update_heartbeat( + hostname, + heartbeat=self.get_heartbeat(worker), + update_freq=self.worker_update_freq, + ) + + def handle_task(self, uuid_task, worker=None): + """Handle snapshotted event.""" + uuid, task = uuid_task + if task.worker and task.worker.hostname: + worker = self.handle_worker( + (task.worker.hostname, task.worker), + ) + + defaults = { + 'name': task.name, + 'args': task.args, + 'kwargs': task.kwargs, + 'eta': correct_awareness(maybe_iso8601(task.eta)), + 'expires': correct_awareness(maybe_iso8601(task.expires)), + 'state': task.state, + 'tstamp': fromtimestamp(task.timestamp), + 'result': task.result or task.exception, + 'traceback': task.traceback, + 'runtime': task.runtime, + 'worker': worker + } + # Some fields are only stored in the RECEIVED event, + # so we should remove these from default values, + # so that they are not overwritten by subsequent states. + [defaults.pop(attr, None) for attr in NOT_SAVED_ATTRIBUTES + if defaults[attr] is None] + return self.update_task(task.state, task_id=uuid, defaults=defaults) + + def update_task(self, state, task_id, defaults=None): + defaults = defaults or {} + if not defaults.get('name'): + return + return self.TaskState.objects.update_state( + state=state, + task_id=task_id, + defaults=defaults, + ) + + def on_shutter(self, state): + + def _handle_tasks(): + for i, task in enumerate(state.tasks.items()): + self.handle_task(task) + + for worker in state.workers.items(): + self.handle_worker(worker) + _handle_tasks() + + def on_cleanup(self): + expired = ( + self.TaskState.objects.expire_by_states(states, expires) + for states, expires in self.expire_task_states + ) + dirty = sum(item for item in expired if item is not None) + if dirty: + debug('Cleanup: Marked %s objects as dirty.', dirty) + self.TaskState.objects.purge() + debug('Cleanup: %s objects purged.', dirty) + return dirty + return 0 diff --git a/django_celery_monitor/humanize.py b/django_celery_monitor/humanize.py new file mode 100644 index 0000000..1969c70 --- /dev/null +++ b/django_celery_monitor/humanize.py @@ -0,0 +1,84 @@ +"""Some helpers to humanize values.""" +from __future__ import absolute_import, unicode_literals + +from datetime import datetime + +from django.utils.translation import ungettext, ugettext as _ +from django.utils.timezone import now + + +def pluralize_year(n): + """Return a string with the number of yeargs ago.""" + return ungettext(_('{num} year ago'), _('{num} years ago'), n) + + +def pluralize_month(n): + """Return a string with the number of months ago.""" + return ungettext(_('{num} month ago'), _('{num} months ago'), n) + + +def pluralize_week(n): + """Return a string with the number of weeks ago.""" + return ungettext(_('{num} week ago'), _('{num} weeks ago'), n) + + +def pluralize_day(n): + """Return a string with the number of days ago.""" + return ungettext(_('{num} day ago'), _('{num} days ago'), n) + + +OLDER_CHUNKS = ( + (365.0, pluralize_year), + (30.0, pluralize_month), + (7.0, pluralize_week), + (1.0, pluralize_day), +) + + +def naturaldate(date, include_seconds=False): + """Convert datetime into a human natural date string.""" + if not date: + return '' + + right_now = now() + today = datetime(right_now.year, right_now.month, + right_now.day, tzinfo=right_now.tzinfo) + delta = right_now - date + delta_midnight = today - date + + days = delta.days + hours = delta.seconds // 3600 + minutes = delta.seconds // 60 + seconds = delta.seconds + + if days < 0: + return _('just now') + + if days == 0: + if hours == 0: + if minutes > 0: + return ungettext( + _('{minutes} minute ago'), + _('{minutes} minutes ago'), minutes + ).format(minutes=minutes) + else: + if include_seconds and seconds: + return ungettext( + _('{seconds} second ago'), + _('{seconds} seconds ago'), seconds + ).format(seconds=seconds) + return _('just now') + else: + return ungettext( + _('{hours} hour ago'), _('{hours} hours ago'), hours + ).format(hours=hours) + + if delta_midnight.days == 0: + return _('yesterday at {time}').format(time=date.strftime('%H:%M')) + + count = 0 + for chunk, pluralizefun in OLDER_CHUNKS: + if days >= chunk: + count = int(round((delta_midnight.days + 1) / chunk, 0)) + fmt = pluralizefun(count) + return fmt.format(num=count) diff --git a/django_celery_monitor/managers.py b/django_celery_monitor/managers.py new file mode 100644 index 0000000..2d21d5a --- /dev/null +++ b/django_celery_monitor/managers.py @@ -0,0 +1,110 @@ +"""The model managers.""" +from __future__ import absolute_import, unicode_literals +from datetime import timedelta + +from celery import states +from celery.events.state import Task +from celery.utils.time import maybe_timedelta +from django.db import models, router, transaction + +from .utils import Now + + +class ExtendedQuerySet(models.QuerySet): + """A custom model queryset that implements a few helpful methods.""" + + def select_for_update_or_create(self, defaults=None, **kwargs): + """Extend update_or_create with select_for_update. + + Look up an object with the given kwargs, updating one with defaults + if it exists, otherwise create a new one. + Return a tuple (object, created), where created is a boolean + specifying whether an object was created. + + This is a backport from Django 1.11 + (https://code.djangoproject.com/ticket/26804) to support + select_for_update when getting the object. + """ + defaults = defaults or {} + lookup, params = self._extract_model_params(defaults, **kwargs) + self._for_write = True + with transaction.atomic(using=self.db): + try: + obj = self.select_for_update().get(**lookup) + except self.model.DoesNotExist: + obj, created = self._create_object_from_params(lookup, params) + if created: + return obj, created + for k, v in defaults.items(): + setattr(obj, k, v() if callable(v) else v) + obj.save(using=self.db) + return obj, False + + +class WorkerStateQuerySet(ExtendedQuerySet): + """A custom model queryset for the WorkerState model with some helpers.""" + + def update_heartbeat(self, hostname, heartbeat, update_freq): + with transaction.atomic(): + # check if there was an update in the last n seconds? + interval = Now() - timedelta(seconds=update_freq) + recent_worker_updates = self.filter( + hostname=hostname, + last_update__gte=interval, + ) + if recent_worker_updates.exists(): + # if yes, get the latest update and move on + obj = recent_worker_updates.get() + else: + # if no, update the worker state and move on + obj, _ = self.select_for_update_or_create( + hostname=hostname, + defaults={'last_heartbeat': heartbeat}, + ) + return obj + + +class TaskStateQuerySet(ExtendedQuerySet): + """A custom model queryset for the TaskState model with some helpers.""" + + def active(self): + """Return all active task states.""" + return self.filter(hidden=False) + + def expired(self, states, expires): + """Return all expired task states.""" + return self.filter( + state__in=states, + tstamp__lte=Now() - maybe_timedelta(expires), + ) + + def expire_by_states(self, states, expires): + """Expire task with one of the given states.""" + if expires is not None: + return self.expired(states, expires).update(hidden=True) + + def purge(self): + """Purge all expired task states.""" + with transaction.atomic(): + self.using( + router.db_for_write(self.model) + ).filter(hidden=True).delete() + + def update_state(self, state, task_id, defaults): + with transaction.atomic(): + obj, created = self.select_for_update_or_create( + task_id=task_id, + defaults=defaults, + ) + if created: + return obj + + if states.state(state) < states.state(obj.state): + keep = Task.merge_rules[states.RECEIVED] + else: + keep = {} + for key, value in defaults.items(): + if key not in keep: + setattr(obj, key, value) + obj.save(update_fields=tuple(defaults.keys())) + return obj diff --git a/django_celery_monitor/migrations/0001_initial.py b/django_celery_monitor/migrations/0001_initial.py new file mode 100644 index 0000000..6d6a34c --- /dev/null +++ b/django_celery_monitor/migrations/0001_initial.py @@ -0,0 +1,133 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TaskState', + fields=[ + ('id', models.AutoField(auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID')), + ('state', models.CharField( + choices=[('FAILURE', 'FAILURE'), + ('PENDING', 'PENDING'), + ('RECEIVED', 'RECEIVED'), + ('RETRY', 'RETRY'), + ('REVOKED', 'REVOKED'), + ('STARTED', 'STARTED'), + ('SUCCESS', 'SUCCESS')], + db_index=True, + max_length=64, + verbose_name='state', + )), + ('task_id', models.CharField( + max_length=36, + unique=True, + verbose_name='UUID', + )), + ('name', models.CharField( + db_index=True, + max_length=200, + null=True, + verbose_name='name', + )), + ('tstamp', models.DateTimeField( + db_index=True, + verbose_name='event received at', + )), + ('args', models.TextField( + null=True, + verbose_name='Arguments', + )), + ('kwargs', models.TextField( + null=True, + verbose_name='Keyword arguments', + )), + ('eta', models.DateTimeField( + null=True, + verbose_name='ETA', + )), + ('expires', models.DateTimeField( + null=True, + verbose_name='expires', + )), + ('result', models.TextField( + null=True, + verbose_name='result', + )), + ('traceback', models.TextField( + null=True, + verbose_name='traceback', + )), + ('runtime', models.FloatField( + help_text='in seconds if task succeeded', + null=True, + verbose_name='execution time', + )), + ('retries', models.IntegerField( + default=0, + verbose_name='number of retries', + )), + ('hidden', models.BooleanField( + db_index=True, + default=False, + editable=False, + )), + ], + options={ + 'ordering': ['-tstamp'], + 'get_latest_by': 'tstamp', + 'verbose_name_plural': 'tasks', + 'verbose_name': 'task', + }, + ), + migrations.CreateModel( + name='WorkerState', + fields=[ + ('id', models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name='ID', + )), + ('hostname', models.CharField( + max_length=255, + unique=True, + verbose_name='hostname', + )), + ('last_heartbeat', models.DateTimeField( + db_index=True, + null=True, + verbose_name='last heartbeat', + )), + ], + options={ + 'ordering': ['-last_heartbeat'], + 'get_latest_by': 'last_heartbeat', + 'verbose_name_plural': 'workers', + 'verbose_name': 'worker', + }, + ), + migrations.AddField( + model_name='taskstate', + name='worker', + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to='celery_monitor.WorkerState', + verbose_name='worker' + ), + ), + ] diff --git a/django_celery_monitor/migrations/0002_workerstate_last_update.py b/django_celery_monitor/migrations/0002_workerstate_last_update.py new file mode 100644 index 0000000..7ec8e7d --- /dev/null +++ b/django_celery_monitor/migrations/0002_workerstate_last_update.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, unicode_literals + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('celery_monitor', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='workerstate', + name='last_update', + field=models.DateTimeField( + default=django.utils.timezone.now, + auto_now=True, + verbose_name='last update', + ), + preserve_default=False, + ), + ] diff --git a/django_celery_monitor/migrations/__init__.py b/django_celery_monitor/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_celery_monitor/models.py b/django_celery_monitor/models.py new file mode 100644 index 0000000..b5605cb --- /dev/null +++ b/django_celery_monitor/models.py @@ -0,0 +1,130 @@ +"""The data models for the task and worker states.""" +from __future__ import absolute_import, unicode_literals + +from time import time, mktime, gmtime + +from django.db import models +from django.utils.translation import ugettext_lazy as _ +from django.conf import settings + +from celery import states +from celery.events.state import heartbeat_expires +from celery.five import python_2_unicode_compatible + +from . import managers + +ALL_STATES = sorted(states.ALL_STATES) +TASK_STATE_CHOICES = sorted(zip(ALL_STATES, ALL_STATES)) + + +@python_2_unicode_compatible +class WorkerState(models.Model): + """The data model to store the worker state in.""" + + #: The hostname of the Celery worker. + hostname = models.CharField(_('hostname'), max_length=255, unique=True) + #: A :class:`~datetime.datetime` describing when the worker was last seen. + last_heartbeat = models.DateTimeField(_('last heartbeat'), null=True, + db_index=True) + last_update = models.DateTimeField(_('last update'), auto_now=True) + + #: A :class:`~django_celery_monitor.managers.ExtendedManager` instance + #: to query the :class:`~django_celery_monitor.models.WorkerState` model. + objects = managers.WorkerStateQuerySet.as_manager() + + class Meta: + """Model meta-data.""" + + verbose_name = _('worker') + verbose_name_plural = _('workers') + get_latest_by = 'last_heartbeat' + ordering = ['-last_heartbeat'] + + def __str__(self): + return self.hostname + + def __repr__(self): + return ''.format(self) + + def is_alive(self): + """Return whether the worker is currently alive or not.""" + if self.last_heartbeat: + # Use UTC timestamp if USE_TZ is true, or else use local timestamp + timestamp = mktime(gmtime()) if settings.USE_TZ else time() + return timestamp < heartbeat_expires(self.heartbeat_timestamp) + return False + + @property + def heartbeat_timestamp(self): + return mktime(self.last_heartbeat.timetuple()) + + +@python_2_unicode_compatible +class TaskState(models.Model): + """The data model to store the task state in.""" + + #: The :mod:`task state ` as returned by Celery. + state = models.CharField( + _('state'), max_length=64, choices=TASK_STATE_CHOICES, db_index=True, + ) + #: The task :func:`UUID `. + task_id = models.CharField(_('UUID'), max_length=36, unique=True) + #: The :ref:`task name `. + name = models.CharField( + _('name'), max_length=200, null=True, db_index=True, + ) + #: A :class:`~datetime.datetime` describing when the task was received. + tstamp = models.DateTimeField(_('event received at'), db_index=True) + #: The positional :ref:`task arguments `. + args = models.TextField(_('Arguments'), null=True) + #: The keyword :ref:`task arguments `. + kwargs = models.TextField(_('Keyword arguments'), null=True) + #: An optional :class:`~datetime.datetime` describing the + #: :ref:`ETA ` for its processing. + eta = models.DateTimeField(_('ETA'), null=True) + #: An optional :class:`~datetime.datetime` describing when the task + #: :ref:`expires `. + expires = models.DateTimeField(_('expires'), null=True) + #: The result of the task. + result = models.TextField(_('result'), null=True) + #: The Python error traceback if raised. + traceback = models.TextField(_('traceback'), null=True) + #: The task runtime in seconds. + runtime = models.FloatField( + _('execution time'), null=True, + help_text=_('in seconds if task succeeded'), + ) + #: The number of retries. + retries = models.IntegerField(_('number of retries'), default=0) + #: The worker responsible for the execution of the task. + worker = models.ForeignKey( + WorkerState, null=True, verbose_name=_('worker'), + on_delete=models.CASCADE, + ) + #: Whether the task has been expired and will be purged by the + #: event framework. + hidden = models.BooleanField(editable=False, default=False, db_index=True) + + #: A :class:`~django_celery_monitor.managers.TaskStateManager` instance + #: to query the :class:`~django_celery_monitor.models.TaskState` model. + objects = managers.TaskStateQuerySet.as_manager() + + class Meta: + """Model meta-data.""" + + verbose_name = _('task') + verbose_name_plural = _('tasks') + get_latest_by = 'tstamp' + ordering = ['-tstamp'] + + def __str__(self): + name = self.name or 'UNKNOWN' + s = '{0.state:<10} {0.task_id:<36} {1}'.format(self, name) + if self.eta: + s += ' eta:{0.eta}'.format(self) + return s + + def __repr__(self): + return ''.format( + self, self.name or 'UNKNOWN', + ) diff --git a/django_celery_monitor/static/django_celery_monitor/style.css b/django_celery_monitor/static/django_celery_monitor/style.css new file mode 100644 index 0000000..b4f4c6a --- /dev/null +++ b/django_celery_monitor/static/django_celery_monitor/style.css @@ -0,0 +1,4 @@ +.form-row.field-traceback p { + font-family: monospace; + white-space: pre; +} diff --git a/django_celery_monitor/templates/django_celery_monitor/confirm_rate_limit.html b/django_celery_monitor/templates/django_celery_monitor/confirm_rate_limit.html new file mode 100644 index 0000000..6152b76 --- /dev/null +++ b/django_celery_monitor/templates/django_celery_monitor/confirm_rate_limit.html @@ -0,0 +1,25 @@ +{% extends "admin/base_site.html" %} +{% load i18n %} + +{% block breadcrumbs %} +
+ {% trans "Home" %} › + {{ app_label|capfirst }} › + {{ opts.verbose_name_plural|capfirst }} › + {% trans 'Rate limit selected tasks' %} +
+{% endblock %} + +{% block content %} +
{% csrf_token %} +
+ {% for obj in queryset %} + + {% endfor %} + + + + +
+
+{% endblock %} diff --git a/django_celery_monitor/utils.py b/django_celery_monitor/utils.py new file mode 100644 index 0000000..1bf0047 --- /dev/null +++ b/django_celery_monitor/utils.py @@ -0,0 +1,116 @@ +"""Utilities.""" +# -- XXX This module must not use translation as that causes +# -- a recursive loader import! +from __future__ import absolute_import, unicode_literals + +from datetime import datetime +from pprint import pformat + +from django.conf import settings +from django.db.models import DateTimeField, Func +from django.utils import timezone +from django.utils.html import format_html +from six import string_types + +try: + from django.db.models.functions import Now +except ImportError: + + class Now(Func): + """A backport of the Now function from Django 1.9.x.""" + + template = 'CURRENT_TIMESTAMP' + + def __init__(self, output_field=None, **extra): + if output_field is None: + output_field = DateTimeField() + super(Now, self).__init__(output_field=output_field, **extra) + + def as_postgresql(self, compiler, connection): + # Postgres' CURRENT_TIMESTAMP means "the time at the start of the + # transaction". We use STATEMENT_TIMESTAMP to be cross-compatible + # with other databases. + self.template = 'STATEMENT_TIMESTAMP()' + return self.as_sql(compiler, connection) + + +def make_aware(value): + """Make the given datetime aware of a timezone.""" + if settings.USE_TZ: + # naive datetimes are assumed to be in UTC. + if timezone.is_naive(value): + value = timezone.make_aware(value, timezone.utc) + # then convert to the Django configured timezone. + default_tz = timezone.get_default_timezone() + value = timezone.localtime(value, default_tz) + return value + + +def correct_awareness(value): + """Fix the given datetime timezone awareness.""" + if isinstance(value, datetime): + if settings.USE_TZ: + return make_aware(value) + elif timezone.is_aware(value): + default_tz = timezone.get_default_timezone() + return timezone.make_naive(value, default_tz) + return value + + +def fromtimestamp(value): + """Return an aware or naive datetime from the given timestamp.""" + if settings.USE_TZ: + return make_aware(datetime.utcfromtimestamp(value)) + else: + return datetime.fromtimestamp(value) + + +FIXEDWIDTH_STYLE = '''\ +{2} \ +''' + + +def _attrs(**kwargs): + def _inner(fun): + for attr_name, attr_value in kwargs.items(): + setattr(fun, attr_name, attr_value) + return fun + return _inner + + +def display_field(short_description, admin_order_field, + allow_tags=True, **kwargs): + """Set some display_field attributes.""" + return _attrs(short_description=short_description, + admin_order_field=admin_order_field, + allow_tags=allow_tags, **kwargs) + + +def action(short_description, **kwargs): + """Set some admin action attributes.""" + return _attrs(short_description=short_description, **kwargs) + + +def fixedwidth(field, name=None, pt=6, width=16, maxlen=64, pretty=False): + """Render a field with a fixed width. + + :param bool pretty: + For field values that are not a string, use pretty printing if True. + """ + @display_field(name or field, field) + def f(task): + val = getattr(task, field) + + # Pretty printing only makes sense for things that aren't strings + if not isinstance(val, string_types) and pretty: + val = pformat(val, width=width) + if val.startswith("u'") or val.startswith('u"'): + val = val[2:-1] + shortval = val.replace(',', ',\n') + shortval = shortval.replace('\n', '
') + + if len(shortval) > maxlen: + shortval = shortval[:maxlen] + '...' + return format_html(FIXEDWIDTH_STYLE, val[:255], pt, shortval) + return f diff --git a/feedcrunch_home/views.py b/feedcrunch_home/views.py index ae68a7e..6487a5b 100644 --- a/feedcrunch_home/views.py +++ b/feedcrunch_home/views.py @@ -50,7 +50,7 @@ def loginView(request): else: return HttpResponseRedirect('/login/') else: - if request.user.is_authenticated(): + if request.user.is_authenticated: return HttpResponseRedirect('/@'+request.user.username+'/admin') else: return render(request, 'login.html', {}) @@ -89,7 +89,7 @@ def signUPView(request): else: - if request.user.is_authenticated(): + if request.user.is_authenticated: return HttpResponseRedirect('/@'+request.user.username+'/admin') else: country_list = Country.objects.all().order_by('name') diff --git a/feedcrunch_rssadmin/templates/admin/admin_post_listing.html b/feedcrunch_rssadmin/templates/admin/admin_post_listing.html index ac90a72..f3066b8 100644 --- a/feedcrunch_rssadmin/templates/admin/admin_post_listing.html +++ b/feedcrunch_rssadmin/templates/admin/admin_post_listing.html @@ -40,7 +40,7 @@ {{ post.id }} {{ post.title }} - {{ post.get_domain }} + {{ post.get_domain }} {{ post.get_shortdate }} {% if 'edit' in request.path %} diff --git a/functions/check_admin.py b/functions/check_admin.py index b293f8c..d5b3b6a 100644 --- a/functions/check_admin.py +++ b/functions/check_admin.py @@ -9,7 +9,7 @@ def check_admin(feedname, user, bypassOnboardingCheck = False): if feedname == None: return HttpResponse("Error") - elif not user.is_authenticated(): + elif not user.is_authenticated: return HttpResponseRedirect('/login/') elif not user.is_active: @@ -26,7 +26,7 @@ def check_admin(feedname, user, bypassOnboardingCheck = False): def check_admin_api(user): - if not user.is_authenticated(): + if not user.is_authenticated: return 'User Not authenticated' elif not user.is_active: diff --git a/lib_bin/windows/ephem-3.7.6.0-cp36-cp36m-win_amd64.whl b/lib_bin/windows/ephem-3.7.6.0-cp36-cp36m-win_amd64.whl new file mode 100644 index 0000000..a1680e9 Binary files /dev/null and b/lib_bin/windows/ephem-3.7.6.0-cp36-cp36m-win_amd64.whl differ diff --git a/lib_bin/windows/scipy-1.0.0-cp36-cp36m-win_amd64.whl b/lib_bin/windows/scipy-1.0.0-cp36-cp36m-win_amd64.whl new file mode 100644 index 0000000..093e2a8 Binary files /dev/null and b/lib_bin/windows/scipy-1.0.0-cp36-cp36m-win_amd64.whl differ diff --git a/manage.py b/manage.py index 7c35deb..37a71f8 100644 --- a/manage.py +++ b/manage.py @@ -10,13 +10,13 @@ if __name__ == "__main__": - platforms = ["TRAVIS", "HEROKU", "BLUEMIX"] + platforms = ["TRAVIS", "BLUEMIX"] if not any(x in os.environ for x in platforms): dotenv.read_dotenv() - + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "application.settings") - + try: from django.core.management import execute_from_command_line except ImportError: diff --git a/manifest-dev.yml b/manifest-dev.yml index 9db9e36..cc7845e 100644 --- a/manifest-dev.yml +++ b/manifest-dev.yml @@ -6,15 +6,16 @@ path: . stack: cflinuxfs2 buildpack: https://github.com/cloudfoundry/buildpack-python.git services: -- Feedcrunch-DB-dev -- RabbitMQ-dev +- Feedcrunch-DB-Dev +- RabbitMQ-Dev +- Redis-Dev applications: - name: Feedcrunch-Front-Dev host: feedcrunch-front-dev instances: 1 - command: chmod +x launch_server.sh && ./launch_server.sh + command: chmod +x ./scripts/bluemix/launch_server.sh && ./scripts/bluemix/launch_server.sh url: - dev.feedcrunch.io - feedcrunch-dev.eu-gb.mybluemix.net @@ -23,6 +24,6 @@ applications: - name: Feedcrunch-Worker-Dev host: feedcrunch-worker-dev instances: 1 - command: celery beat -A application --loglevel=info --detach && celery events -A application --loglevel=info --camera=django_celery_monitor.camera.Camera --frequency=2.0 --detach && celery worker -A application -l info --events + command: chmod +x ./scripts/bluemix/launch_orchester_dev.sh && ./scripts/bluemix/launch_orchester_dev.sh health-check-type: process no-route: true diff --git a/manifest.yml b/manifest.yml index 57cf8ca..70e0763 100644 --- a/manifest.yml +++ b/manifest.yml @@ -6,31 +6,32 @@ path: . stack: cflinuxfs2 buildpack: https://github.com/cloudfoundry/buildpack-python.git services: -- Feedcrunch-DB-prod -- RabbitMQ-prod +- Feedcrunch-DB-Prod +- RabbitMQ-Prod +- Redis-Prod applications: - name: Feedcrunch-Front-Prod host: feedcrunch-front-prod instances: 2 - command: chmod +x launch_server.sh && ./launch_server.sh + command: chmod +x ./scripts/bluemix/launch_server.sh && ./scripts/bluemix/launch_server.sh url: - www.feedcrunch.io - - feedcrunch.eu-gb.mybluemix.net + - feedcrunch.eu-gb.mybluemix.net - feedcrunch-api-prod.eu-gb.mybluemix.net - name: Feedcrunch-Orchester-Prod host: feedcrunch-orchester-prod memory: 1G instances: 1 - command: celery beat -A application --loglevel=info --detach && celery events -A application --loglevel=info --camera=django_celery_monitor.camera.Camera --frequency=2.0 --detach && celery worker -A application -l info --events + command: chmod +x ./scripts/bluemix/launch_orchester.sh && ./scripts/bluemix/launch_orchester.sh health-check-type: process no-route: true - name: Feedcrunch-Worker-Prod host: feedcrunch-worker-prod instances: 3 - command: celery worker -A application -l info --events + command: chmod +x ./scripts/bluemix/launch_worker.sh && ./scripts/bluemix/launch_worker.sh health-check-type: process no-route: true diff --git a/media/__init__.py b/media/__init__.py deleted file mode 100644 index faa18be..0000000 --- a/media/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/media/estimators/.gitkeep b/media/estimators/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/media/estimators/__init__.py b/media/estimators/__init__.py deleted file mode 100644 index faa18be..0000000 --- a/media/estimators/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/media/images/__init__.py b/media/images/__init__.py deleted file mode 100644 index faa18be..0000000 --- a/media/images/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/media/images/interest_photos/.gitkeep b/media/images/interest_photos/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/media/images/user_photos/.gitkeep b/media/images/user_photos/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/media/images/user_photos/__init__.py b/media/images/user_photos/__init__.py deleted file mode 100644 index faa18be..0000000 --- a/media/images/user_photos/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- diff --git a/requirements.txt b/requirements.txt index 5f93b3b..619fe7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,54 +1,55 @@ amqp==2.2.2 -asn1crypto==0.23.0 +asn1crypto==0.24.0 billiard==3.5.0.3 boto==2.48.0 -celery==4.1.0 -certifi==2017.11.5 -cffi==1.11.2 +celery==4.2.0rc1 +#git+https://github.com/celery/celery.git@be55de6#egg=celery +certifi==2018.1.18 +cffi==1.11.5 chardet==3.0.4 click==6.7 -coverage==4.4.2 -cryptography==2.1.3 -dj-database-url==0.4.2 -Django==1.11.7 -django-admin-view-permission==1.1 -django-celery-beat==1.1.0 -django-celery-monitor==1.1.2 +coverage==4.5.1 +cryptography==2.2.1 +dj-database-url==0.5.0 +Django==2.0.3 +django-admin-view-permission==1.6 +django-celery-beat==1.1.1 +#django-celery-monitor==1.1.2 django-choices==1.6.0 -django-dotenv==1.4.1 +django-dotenv==1.4.2 django-downloadview==1.9 django-encrypted-model-fields==0.5.3 -django-extensions==1.9.7 +django-extensions==2.0.6 django-getenv==1.3.2 -django-ipware==1.1.6 -django-material==1.1.1 -django-ses==0.8.3.1 +django-ipware==2.0.1 +django-ses==0.8.5 django-storages==1.6.5 -djangorestframework==3.7.3 +djangorestframework==3.7.7 dnspython==1.15.0 dnspython3==1.15.0 facebook-sdk==2.0.0 -factory-boy==2.9.2 -Faker==0.8.7 +factory-boy==2.10.0 +Faker==0.8.12 feedgen==0.6.1 feedparser==5.2.1 future==0.16.0 gunicorn==19.7.1 idna==2.6 kombu==4.1.0 -lxml==4.1.1 -numpy==1.13.3 -oauthlib==2.0.6 -olefile==0.44 -pandas==0.21.0 -Pillow==4.3.0 -psycopg2==2.7.3.2 +lxml==4.2.1 +numpy==1.14.2 +oauthlib==2.0.7 +olefile==0.45.1 +pandas==0.22.0 +Pillow==5.0.0 +psycopg2-binary==2.7.4 pycparser==2.18 pyIsEmail==1.3.1 -pyOpenSSL==17.3.0 -python-dateutil==2.6.1 +pyOpenSSL==17.5.0 +python-dateutil==2.7.0 python3-linkedin==1.0.2 -pytz==2017.3 +pytz==2018.3 +redis==2.10.6 requests==2.18.4 requests-oauthlib==0.8.0 scikit-learn==0.19.1 @@ -59,5 +60,5 @@ twython==3.6.0 untangle==1.1.1 urllib3==1.22 vine==1.1.4 -Werkzeug==0.12.2 +Werkzeug==0.14.1 whitenoise==3.3.1 diff --git a/runtime.txt b/runtime.txt index be711af..09dac98 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.6.3 \ No newline at end of file +python-3.6.4 \ No newline at end of file diff --git a/scripts/bluemix/launch_orchester.sh b/scripts/bluemix/launch_orchester.sh new file mode 100644 index 0000000..6f4e762 --- /dev/null +++ b/scripts/bluemix/launch_orchester.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +celery beat -A application --loglevel=info --detach +celery events -A application --loglevel=info --camera=django_celery_monitor.camera.Camera --frequency=2.0 --detach +celery worker -A application -l info --events \ No newline at end of file diff --git a/scripts/bluemix/launch_orchester_dev.sh b/scripts/bluemix/launch_orchester_dev.sh new file mode 100644 index 0000000..a6efc68 --- /dev/null +++ b/scripts/bluemix/launch_orchester_dev.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +celery beat -A application --loglevel=debug --detach +celery events -A application --loglevel=debug --camera=django_celery_monitor.camera.Camera --frequency=2.0 --detach +celery worker -A application --loglevel=debug --events \ No newline at end of file diff --git a/launch_server.sh b/scripts/bluemix/launch_server.sh similarity index 100% rename from launch_server.sh rename to scripts/bluemix/launch_server.sh diff --git a/scripts/bluemix/launch_worker.sh b/scripts/bluemix/launch_worker.sh new file mode 100644 index 0000000..17e54dc --- /dev/null +++ b/scripts/bluemix/launch_worker.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +celery worker -A application -l info --events \ No newline at end of file diff --git a/scripts/clean_celery.sql b/scripts/clean_celery.sql new file mode 100644 index 0000000..3b62466 --- /dev/null +++ b/scripts/clean_celery.sql @@ -0,0 +1,7 @@ +delete from celery_monitor_taskstate where 1=1; +delete from celery_monitor_workerstate where 1=1; +delete from django_celery_beat_periodictask where 1=1; +delete from django_celery_beat_periodictasks where 1=1; +delete from django_celery_beat_intervalschedule where 1=1; +delete from django_celery_beat_crontabschedule where 1=1; +delete from django_celery_beat_solarschedule where 1=1; \ No newline at end of file diff --git a/scripts/win/install_dependencies.bat b/scripts/win/install_dependencies.bat index 785d88b..b11c904 100644 --- a/scripts/win/install_dependencies.bat +++ b/scripts/win/install_dependencies.bat @@ -2,8 +2,8 @@ cd "%~p1" cd "../.." call venv\Scripts\activate.bat -call pip install --upgrade pip -call pip install "lib_bin\windows\scipy-0.19.1-cp36-cp36m-win_amd64.whl" -REM call pip install https://github.com/hairychris/django-material/archive/2b3d70347cf29bcc02b06d3319f9617b626502c8.zip +REM call pip install --upgrade pip +call pip install "lib_bin\windows\scipy-1.0.0-cp36-cp36m-win_amd64.whl" +call pip install "lib_bin\windows\ephem-3.7.6.0-cp36-cp36m-win_amd64.whl" call pip install -r requirements.txt PAUSE; diff --git a/scripts/win/collectStatic.bat b/scripts/win/launch_collectStatic.bat similarity index 100% rename from scripts/win/collectStatic.bat rename to scripts/win/launch_collectStatic.bat