WIP: Dockerisation

.gitignore

@@ -22,4 +22,5 @@ celerybeat-schedule
 run.sh
 .idea
 **/settings.yml
-
+src
+quantifiedcode/data

Dockerfile

@@ -0,0 +1,20 @@
+FROM debian:stretch-slim
+
+# -- Add some packages to the image
+RUN apt-get update
+RUN apt-get install -y \
+        curl \
+        git \
+        ipython \
+        nano \
+        python-pip \
+        sudo \
+        vim \
+        wget
+
+# -- Set up the source code
+ADD . /opt/quantifiedcode
+WORKDIR /opt/quantifiedcode
+
+# -- Pre-build requirements
+RUN pip install -r requirements.txt

README.md

@@ -34,7 +34,7 @@ The installation consists of three parts:
 QuantifiedCode requires the following external dependencies:
 
 * A message broker (required for the background tasks message queue). We recommend either RabbitMQ or Redis.
-* A database (required for the core application). We recommend PostgreSQL, but SQLite is supported as well. Other database systems might work too (e.g. MySQL), but are currently not officially supported. If you need to run QuantifiedCode on a non-supported database, please get in touch with us and we'll be happy to provide you some guidance.
+* A database (required for the core application). We recommend PostgreSQL, but SQLite is supported as well. Other database systems might work too (e.g. MySQL), but are currently not officially supported. If you need to run QuantifiedCode on a non-supported database, please get in touch with us and we'll be happy to provide you some guidance. See http://docs.sqlalchemy.org/en/latest/core/engines.html when editing the config file.
 
 ### Download the QuantifiedCode source code
 

docker-compose.yml

@@ -0,0 +1,44 @@
+qc:
+  build: .
+  volumes:
+    - .:/opt/quantifiedcode
+  environment:
+    QC_SETTINGS: /opt/quantifiedcode/quantifiedcode/settings/default.yml
+
+web:
+  extends:
+    service: qc
+  hostname: web
+  restart: always
+  links:
+    - db:db
+    - redis:redis
+  ports:
+    - "80:80"
+    - "443:443"
+    - "5000:5000"
+  command: "bash /opt/quantifiedcode/initiate_web.sh"
+
+celery:
+  extends:
+    service: qc
+  restart: always
+  links:
+    - db:db
+    - redis:redis
+  command: "bash /opt/quantifiedcode/initiate_workers.sh"
+
+
+db:
+  image: postgres:9.6.6
+  restart: always
+  environment:
+    - POSTGRES_USER=qc
+    - POSTGRES_DB=quantified
+    - POSTGRES_PASSWORD=password
+
+redis:
+  image: redis:3
+  restart: always
+  ports:
+    - "6379:6379"

initiate_web.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+pip install -r requirements.txt  # Refresh requirements
+python manage.py setup
+python manage.py runserver

initiate_workers.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+pip install -r requirements.txt  # Refresh requirements
+python manage.py setup
+python manage.py runworker

quantifiedcode/backend/api/v1/diff.py

@@ -154,7 +154,8 @@ def get(self, project_id, snapshot_a_id, snapshot_b_id, path=None):
                                      diff_issue_occurrence_table.c.issue_occurrence == issue_occurrence_table.c.pk,
                                      *issue_type_query))\
                        .join(issue_table, issue_table.c.pk == issue_occurrence_table.c.issue)\
-                       .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code)\
+                       .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code) \
+                       .alias('select_table')
 
         #we construct the table we will select issues from
         file_revision_query = select([fr_table.c.pk])\

quantifiedcode/backend/api/v1/public_projects.py

@@ -45,7 +45,7 @@ def get(self):
         form = PublicProjectsForm(request.args)
 
         if not form.validate():
-            return {'message' : 'please correct the errors mentioned below', errors: form.errors}, 400
+            return {'message' : 'please correct the errors mentioned below', 'errors': form.errors}, 400
 
         data = form.data
 

quantifiedcode/backend/api/v1/snapshot.py

@@ -106,6 +106,7 @@ def get(self, project_id, snapshot_id, path=None):
                        .join(issue_occurrence_table, issue_occurrence_table.c.file_revision == snapshot_file_revisions_table.c.filerevision)\
                        .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore) )\
                        .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code)\
+                       .alias('fr_select_table')
 
         #we construct the file revisions query
         file_revisions_query = select([snapshot_file_revisions_table.c.filerevision])\
@@ -124,6 +125,7 @@ def get(self, project_id, snapshot_id, path=None):
                        .join(issue_occurrence_table, issue_occurrence_table.c.file_revision == fr_table.c.pk)\
                        .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore))\
                        .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code)\
+                       .alias('select_table')
 
 
         #we construct the issues query

quantifiedcode/backend/tasks/helpers.py

@@ -70,7 +70,7 @@ def __init__(self, task, level=logging.INFO, backend=None, ping=True):
 
     def __enter__(self):
         kwargs = {
-            'filename': os.path.join(settings.get('backend.path'), settings.get('backend.paths.tasks'),
+            'filename': os.path.join(settings.get('project_path'), settings.get('backend.paths.tasks'),
                                      self.task.pk + '.log'),
             'encoding': "utf-8",
             'mode': "w",
@@ -139,10 +139,10 @@ def task_is_being_processed(current_task=None):
                     try:
                         res = celery.AsyncResult(task.celery_task_id)
                         with self.backend.transaction():
-                            if res.state == ResultState.Started:
+                            if res.state == self.ResultState.Started:
                                 self.backend.update(task, {'status': 'in_progress'})
                                 return True
-                            elif res.state == ResultState.Pending:
+                            elif res.state == self.ResultState.Pending:
                                 if task.get('last_ping'):
                                     # if we haven't heard from the task in a while, we mark it as failed...
                                     if datetime.datetime.now() - task.last_ping > datetime.timedelta(seconds=60 * 10):
@@ -151,9 +151,9 @@ def task_is_being_processed(current_task=None):
                                 elif datetime.datetime.now() - task.created_at < datetime.timedelta(seconds=60 * 20):
                                     self.backend.update(task, {'status': 'in_progress'})
                                     return True
-                            elif res.state == ResultState.Failure:
+                            elif res.state == self.ResultState.Failure:
                                 self.backend.update(task, {'status': 'failed'})
-                            elif res.state == ResultState.Success:
+                            elif res.state == self.ResultState.Success:
                                 self.backend.update(task, {'status': 'succeeded'})
                     except Task.DoesNotExist:
                         # sometimes a task object will get deleted by another worker while

quantifiedcode/backend/tasks/periodic.py

@@ -35,8 +35,8 @@ def delete_pending_project():
 
 @celery.task(queue="delete", ignore_result=False, time_limit=3600)
 def delete_pending_user():
-    pending_projects = backend.filter(User, {'delete': True}).sort('updated_at', 1).limit(100)
-    logger.debug("%d users marked for deletion" % len(pending_projects))
+    pending_users = backend.filter(User, {'delete': True}).sort('updated_at', 1).limit(100)
+    logger.debug("%d users marked for deletion" % len(pending_users))
     for pending_user in pending_users:
         return delete_user(pending_user.pk, task_id=delete_pending_user.request.id)
     logger.debug("No users left to delete...")

quantifiedcode/backend/tasks/project/delete.py

@@ -21,13 +21,15 @@
 from checkmate.management.commands.reset import Command as ResetCommand
 from quantifiedcode.settings import settings, backend
 from quantifiedcode.backend.settings import BACKEND_PATH
+from quantifiedcode.backend.tasks.helpers import ExclusiveTask, TaskLogger
 
 from ...worker import celery
 from ...models import (Project,
                        Task,
                        UserRole,
                        IssueClass,
                        ProjectIssueClass)
+from ..project.analyze import analyze_project
 
 @celery.task(queue="analysis", ignore_result=False)
 def delete_project(project_id, task_id=None):

quantifiedcode/backend/utils/api.py

@@ -11,7 +11,9 @@
 
 from quantifiedcode.settings import settings
 
+from flask import jsonify
 import logging
+import traceback
 logger = logging.getLogger(__name__)
 
 

quantifiedcode/frontend/app.py

@@ -136,4 +136,5 @@ def configure(app, settings):
 
 if __name__ == '__main__':
     app = get_app(settings)
+    debug = settings.get('debug')
     app.run(debug=debug, host='0.0.0.0', port=8000,threaded = False)

quantifiedcode/plugins/git/backend/api/v1/resources/git_snapshots.py

@@ -10,6 +10,7 @@
 from __future__ import print_function
 
 import logging
+import subprocess
 
 from datetime import datetime
 from flask import request

quantifiedcode/plugins/git/backend/api/v1/resources/project.py

@@ -79,11 +79,13 @@ def post(self):
             'url' : form.url.data
         }
 
-        try:
-            project = create_project(project_data, git_data, user)
-        except DuplicateProject:
-            return ({'message': 'A project with the same name already exists for your account.'},
-                    403)
+        project = create_project(project_data, git_data, user)
+        # FIXME: handle duplicate project exception
+        # try:
+        #     pass
+        # except DuplicateProject:
+        #     return ({'message': 'A project with the same name already exists for your account.'},
+        #             403)
         return ({'message': 'success!',
                  'project': self.export(project)},
                 200)

quantifiedcode/settings/default.yml

@@ -2,9 +2,10 @@
 backend:
   url: /api
   # by default, we use an in-memory instance of SQLite for testing.
-  db_url_test: 'sqlite+pysqlite:///:memory:'
+  db_url: 'postgresql+psycopg2://qc:password@db/quantified'
+  db_url_test: 'sqlite:///qc_test.sqlite'
   paths:
-    repositories: "data/repositories"
+    git_repositories: "data/repositories"
     tasks: "data/tasks"
   celery:
     config:
@@ -14,8 +15,8 @@ backend:
       timezone: Europe/Oslo
       enable_utc: true
       worker_hijack_root_logger: false
-      broker_url: 'amqp://qc:qc@localhost:5672/qc'
-      result_backend: 'amqp://qc:qc@localhost:5672/qc'
+      broker_url: 'redis://redis:6379/'
+      result_backend: 'redis://redis:6379/'
       task_default_queue: tasks
       task_queues:
         - {name: tasks, routing_key: 'task'}
@@ -67,6 +68,7 @@ frontend:
     explore: "/explore"
     project: "/project"
     projects: "/projects"
+url: http://localhost:5000
 render_context:
   # the current_year is added through Python settings#
   # (as this is a dynamic variable)

requirements.txt

@@ -1,5 +1,4 @@
 passlib==1.7.1
-#pysqlite
 alembic==0.9.3
 click==6.7
 flask==0.12.2
@@ -15,3 +14,5 @@ requests==2.18.1
 pyopenssl==17.2.0
 python-dateutil==2.6.1
 requirements-parser==0.1.0
+-e git://github.com/quantifiedcode/checkmate.git#egg=checkmate
+redis==2.10.6