From 6317bca158caaac63a159129cafa71df2fb675ee Mon Sep 17 00:00:00 2001 From: Beau Hargis Date: Fri, 2 Aug 2013 17:45:15 -0700 Subject: [PATCH 1/4] Work in progress: tweaks for deployment --- codalab/config/templates/freetds.conf | 4 +-- codalab/config/templates/odbc.ini | 2 +- codalab/config/templates/odbcinst.ini | 2 +- codalab/fabfile.py | 39 ++++++++++++++++++++------- codalab/runserver | 7 ++++- codalab/scripts/provision | 29 -------------------- codalab/scripts/ubuntu/provision | 22 +++++++-------- 7 files changed, 50 insertions(+), 55 deletions(-) delete mode 100644 codalab/scripts/provision diff --git a/codalab/config/templates/freetds.conf b/codalab/config/templates/freetds.conf index 7d7ace0a1..511f1cb2e 100644 --- a/codalab/config/templates/freetds.conf +++ b/codalab/config/templates/freetds.conf @@ -1,10 +1,10 @@ [global] - tds version = 8.0 + tds version = 7.1 character set = UTF-8 [{{DB_SERVER}}] port = {{DB_PORT}} - tds version = 8.0 + tds version = 7.1 character set = UTF-8 host = {{DB_HOST}} encryption = required diff --git a/codalab/config/templates/odbc.ini b/codalab/config/templates/odbc.ini index eff887bdd..f07b71ed7 100644 --- a/codalab/config/templates/odbc.ini +++ b/codalab/config/templates/odbc.ini @@ -9,5 +9,5 @@ Database = {{DB_NAME}} UID = {{DB_USER}} PWD = {{DB_PASSWORD}} Port = {{DB_PORT}} -TDS_Version = 8.0 +TDS_Version = 7.1 diff --git a/codalab/config/templates/odbcinst.ini b/codalab/config/templates/odbcinst.ini index d64454142..c247b90be 100644 --- a/codalab/config/templates/odbcinst.ini +++ b/codalab/config/templates/odbcinst.ini @@ -1,5 +1,5 @@ [ODBC] -Trace = Yes +Trace = No TraceFile = /tmp/odbc-{{DB_NAME}}.log [FreeTDS] diff --git a/codalab/fabfile.py b/codalab/fabfile.py index 204649e8c..15206bae6 100644 --- a/codalab/fabfile.py +++ b/codalab/fabfile.py @@ -7,32 +7,51 @@ VENV_PATH = os.path.join(THIS_DIR,'venv') THIS_SETTINGS_DIR = os.path.join(THIS_DIR,'codalab','settings') +env.conf.DEPLOY_USER = 'wwwuser' +env.conf.DEPLOY_PATH = 'codalab_src' + def env_setup(config=env.CONFIG,settings_module='codalab.settings'): + if config is None: + sys.path.append('.') os.environ['DJANGO_CONFIGURATION'] = config - os.environ["DJANGO_SETTINGS_MODULE"]= settings_module + os.environ["DJANGO_SETTINGS_MODULE"] = settings_module + from fabric.contrib import django from configurations import importer importer.install() - from fabric.contrib import django + django.settings_module(settings_module) - from django.conf import settings + from django.conf import settings as django_settings - #env.roledefs = settings.DEPLOY_ROLES - env.django_settings = settings -env_setup() + env.roledefs = django_settings.DEPLOY_ROLES + env.django_settings = django_settings @task -def local(): +def local(**kwargs): env.run = lrun - + env_setup(**kwargs) + @task -def remote(): +def remote(**kwargs): env.run = run + env_setup(**kwargs) @task -def bootstrap_on_linux(): +def clone_repo(repo_url='https://github.com/codalab/codalab.git',path=env.conf.DEPLOY_PATH): + env.run('git clone %s %s' % (repo_url, path)) + +@task +def provision(config='Dev'): + clone_repo() + with cd(env.conf.DEPLOY_PATH): + sudo('/bin/bash codalab/scripts/provision %s' % env.conf.DEPLOY_USER) + sudo('python manage.py config_gen --configuration=%s' % config, + user=env.conf.DEPLOY_USER) + +@task +def bootstrap(): make_virtualenv(path='venv', system_site_packages=False) with virtualenv('venv'): run('pip install --upgrade pip') diff --git a/codalab/runserver b/codalab/runserver index b867092aa..83531ba3c 100755 --- a/codalab/runserver +++ b/codalab/runserver @@ -1,7 +1,12 @@ #!/bin/bash +THIS_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +pushd `pwd` > /dev/null +cd $THIS_DIR + source ./config/generated/startup_env.sh echo " .... " echo "$DJANGO_CONFIGURATION Configuration" echo " .... " -python manage.py runserver $1 $2 \ No newline at end of file +/bin/bash manage runserver "$@" +popd diff --git a/codalab/scripts/provision b/codalab/scripts/provision deleted file mode 100644 index dc9b0d96b..000000000 --- a/codalab/scripts/provision +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash - -## This will eventually be a bit more configurable - -THIS_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd `pwd` > /dev/null -cd $THIS_DIR - -U=$1 - -if [[ "$U" == "" ]] ; then - echo "Specify user" - exit 1 -fi -if ! id -u "$U" > /dev/null; then - sudo useradd -m -s /bin/bash $U - sudo passwd $U - exit 1 -fi - -echo "Installing codalab dependency meta-package..." -sudo dpkg -i *.deb > /dev/null && \ -sudo apt-get -f install > /dev/null && \ -echo "Updaing some user permissions..." && \ -sudo chgrp -R www-data /var/log/nginx /var/lib/nginx && \ -sudo chmod -R g+rw /var/log/nginx /var/lib/nginx && \ -sudo usermod -a -G www-data $U - - diff --git a/codalab/scripts/ubuntu/provision b/codalab/scripts/ubuntu/provision index dc9b0d96b..9a4c0950c 100755 --- a/codalab/scripts/ubuntu/provision +++ b/codalab/scripts/ubuntu/provision @@ -3,27 +3,27 @@ ## This will eventually be a bit more configurable THIS_DIR="$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pushd `pwd` > /dev/null +pushd `pwd` 2>&1 > /dev/null cd $THIS_DIR U=$1 -if [[ "$U" == "" ]] ; then - echo "Specify user" - exit 1 -fi -if ! id -u "$U" > /dev/null; then - sudo useradd -m -s /bin/bash $U - sudo passwd $U +if [[ $U =~ [a-zA-Z0-9][\-\_a-zA-Z0-9]+[a-zA-Z0-9] ]] ; then + if ! id -u "$U" 2>&1 > /dev/null; then + echo "User will be added." + sudo useradd -m -s /bin/bash $U + sudo passwd $U + fi +else + echo "User not specified or username has bad characters" exit 1 fi echo "Installing codalab dependency meta-package..." sudo dpkg -i *.deb > /dev/null && \ sudo apt-get -f install > /dev/null && \ -echo "Updaing some user permissions..." && \ +echo "Updaing some group permissions..." && \ sudo chgrp -R www-data /var/log/nginx /var/lib/nginx && \ sudo chmod -R g+rw /var/log/nginx /var/lib/nginx && \ sudo usermod -a -G www-data $U - - +popd > /dev/null From 3a5a86ae51c266c6542bff84474325a34717d866 Mon Sep 17 00:00:00 2001 From: Beau Hargis Date: Fri, 2 Aug 2013 17:46:04 -0700 Subject: [PATCH 2/4] Added manage shell to make using environment variables easier --- codalab/manage | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 codalab/manage diff --git a/codalab/manage b/codalab/manage new file mode 100644 index 000000000..57a12803c --- /dev/null +++ b/codalab/manage @@ -0,0 +1,7 @@ +#!/bin/bash + +source ./config/generated/startup_env.sh +echo " .... " +echo "$DJANGO_CONFIGURATION Configuration" +echo " .... " +python manage.py "$@" From dec93d4d12e201c02f2f565d6e7cde2690a45e07 Mon Sep 17 00:00:00 2001 From: Beau Hargis Date: Mon, 5 Aug 2013 13:17:23 -0700 Subject: [PATCH 3/4] Celery tasks work in progress --- codalab/compsrv/celery.py | 15 -------------- codalab/compsrv/tasks.py | 32 ++++++++++++++++++++++++++--- codalab/config/templates/nginx.conf | 1 + 3 files changed, 30 insertions(+), 18 deletions(-) delete mode 100644 codalab/compsrv/celery.py diff --git a/codalab/compsrv/celery.py b/codalab/compsrv/celery.py deleted file mode 100644 index 55ac5eba9..000000000 --- a/codalab/compsrv/celery.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import absolute_import - -from celery import Celery - - -celery = Celery('tasks') -celery.config_from_object('settings') - -# Optional configuration, see the application user guide. -celery.conf.update( - CELERY_TASK_RESULT_EXPIRES=3600, -) - -if __name__ == '__main__': - celery.start() diff --git a/codalab/compsrv/tasks.py b/codalab/compsrv/tasks.py index 4e3385e48..11d39a9eb 100644 --- a/codalab/compsrv/tasks.py +++ b/codalab/compsrv/tasks.py @@ -1,6 +1,32 @@ +from __future__ import absolute_import + +from celery import Celery from celery import task +celery = Celery('tasks', backend='amqp', ) +celery.config_from_object('celeryconfig') + +# Optional configuration, see the application user guide. +celery.conf.update( + CELERY_TASK_RESULT_EXPIRES=3600, +) + + +# Tasks for competitions +# Currently stubs to test functionality @task -def evaluate_submission(location): - # evaluate(inputdir,standard,outputdir) - +def validate_submission(url): + """ + Will validate the format of a submission. + """ + return url + +@task +def evaluate_submission(url): + # evaluate(inputdir, standard, outputdir) + return url + + +# For starting the process +if __name__ == '__main__': + celery.start() diff --git a/codalab/config/templates/nginx.conf b/codalab/config/templates/nginx.conf index 176259d49..1c044a1b2 100644 --- a/codalab/config/templates/nginx.conf +++ b/codalab/config/templates/nginx.conf @@ -2,6 +2,7 @@ worker_processes 4; pid {{PROJECT_DIR}}/var/nginx-{{PORT}}.pid; #error_log /tmp/nginx-https.err.log; lock_file {{PROJECT_DIR}}/var/nginx-{{PORT}}.lock; +daemon off; events { worker_connections 768; From bc7b3a0875b1452390a423b5c40f73ec2110be7d Mon Sep 17 00:00:00 2001 From: Beau Hargis Date: Tue, 6 Aug 2013 13:40:15 -0700 Subject: [PATCH 4/4] Added management to add submission --- .../web/management/commands/add_submission.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 codalab/apps/web/management/commands/add_submission.py diff --git a/codalab/apps/web/management/commands/add_submission.py b/codalab/apps/web/management/commands/add_submission.py new file mode 100644 index 000000000..1c52445b4 --- /dev/null +++ b/codalab/apps/web/management/commands/add_submission.py @@ -0,0 +1,88 @@ +from django.core.management.base import BaseCommand, CommandError +from apps.web.models import Competition,CompetitionPhase,CompetitionParticipant,CompetitionSubmission +from django.contrib.auth import get_user_model +User = get_user_model() +from optparse import make_option + +from django.core.files.base import File + +class Command(BaseCommand): + help = "Creates a submission for a participant" + + option_list = BaseCommand.option_list + ( + make_option('--email', + dest='email', + help="Email of the user"), + make_option('--competition', + dest='competition', + default=None, + help="ID of the submission"), + make_option('--phase', + dest='phase', + default=None, + help="ID of the competition phase"), + make_option('--submission', + dest='submission', + default=None, + help="Path to the submission file"), + + ) + + def handle(self, *args, **options): + competition_id = options['competition'] + phase_id = options['phase'] + submission = options['submission'] + competition = None + phase = None + if not options['email']: + print " ERROR ... Email Required ... " + exit(1) + if not submission: + print " ERROR ... Submission File Required ... " + exit(1) + + user = User.objects.get(email=options['email']) + while not competition and not phase: + if competition_id and phase_id: + try: + phase = CompetitionPhase.objects.get(pk=phase_id, competition__pk=competition_id) + break + except Competition.DoesNotExist: + pass + else: + print "Competition/Phase not specified or not valid:\n" + + clist = CompetitionPhase.objects.order_by('competition__pk').all() + if not clist: + print " ... There are no competitions ..." + exit(1) + sel = [] + i = 0 + for c in clist: + sel.append((c.competition,c)) + print " %d) %s - %s" % (i+1,c.competition.title,c.label) + i = i + 1 + try: + inp = int(raw_input("\n Enter number --> ")) + idx = inp - 1 + competition = sel[idx][0] + phase = sel[idx][1] + except ValueError: + print " ... Bad Input ... " + competition_id = None + continue + except Exception as e: + print e + + part = CompetitionParticipant.objects.get(user=user, + competition=competition + ) + submission_file = File(open(options['submission'],'rb')) + + s = CompetitionSubmission.objects.create(participant=part,phase=phase,file=submission_file) + + print "OK. Submission Created" + + + +