From 621b2bf41db81df412c7d6e5b34eb2061eba3f2b Mon Sep 17 00:00:00 2001 From: Jonathan Le_Bonzec Date: Wed, 22 Nov 2017 11:55:21 +0100 Subject: [PATCH 1/6] Added checkmate as a dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 5906853..9b5a8c0 100755 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,4 @@ requests==2.18.1 pyopenssl==17.2.0 python-dateutil==2.6.1 requirements-parser==0.1.0 +-e git://github.com/quantifiedcode/checkmate.git#egg=checkmate From b45bab733de7f79f34fbb15c83bae3f35f49db6a Mon Sep 17 00:00:00 2001 From: Jonathan Le_Bonzec Date: Wed, 22 Nov 2017 11:56:21 +0100 Subject: [PATCH 2/6] Added empty settings.yml file --- settings.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 settings.yml diff --git a/settings.yml b/settings.yml new file mode 100644 index 0000000..e69de29 From f8fbb185c40d23bf15d06a7a405d7de730e10937 Mon Sep 17 00:00:00 2001 From: Jonathan Le_Bonzec Date: Wed, 22 Nov 2017 13:54:07 +0100 Subject: [PATCH 3/6] Ignore src and data directory This is where checkmate is pulled, or where repos are saved for instance, we don't want to cache it. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 7571c2c..5bcc4e3 100644 --- a/.gitignore +++ b/.gitignore @@ -22,4 +22,5 @@ celerybeat-schedule run.sh .idea **/settings.yml - +src +quantifiedcode/data From a96cd15a7dcc12da508fb2be6fde05d4191cc771 Mon Sep 17 00:00:00 2001 From: Jonathan Le_Bonzec Date: Wed, 22 Nov 2017 16:12:56 +0100 Subject: [PATCH 4/6] First attempt to dockerise The site is able to run. We can register new users and new projects. The analysis is not run though. To do absolutely: - Have the data (repos and stats) in a separate container. - It's saved in `quantified/data` for now, but is configurable through config - Have the celery worker run (in a separate container?) - Command is `python manage.py runworker` Not breaking, but important, especially for scaling: - Be able to read a special config file - It's currently all hardcoded - Have RabbitMQ (or Redis) in a separate container - Have the database (postgres advised) in a separate container Both are almost ready but I couldn't get it to work properly. So I used the local version intead. --- Dockerfile | 23 +++++++++++++++++++++++ docker-compose.yml | 25 +++++++++++++++++++++++++ initiate.sh | 7 +++++++ quantifiedcode/settings/default.yml | 8 +++++--- requirements.txt | 3 ++- 5 files changed, 62 insertions(+), 4 deletions(-) create mode 100644 Dockerfile create mode 100644 docker-compose.yml create mode 100755 initiate.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2878d22 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM debian:stretch-slim + +# -- Add some packages to the image +RUN apt-get update +RUN apt-get install -y \ + git \ + ipython \ + mysql-client \ + nano \ + python-pip \ + rabbitmq-server \ + sudo \ + vim \ + wget + +# -- Set up the source code +ADD . /opt/quantifiedcode +WORKDIR /opt/quantifiedcode + +# -- Pre-build requirements +RUN pip install -r requirements.txt + +RUN python manage.py setup diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..9e61db6 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,25 @@ +qc: + build: . + hostname: qc + environment: + QC_SETTINGS: /opt/quantifiedcode/quantifiedcode/settings/default.yml + links: + - db:db + ports: + - "80:80" + - "443:443" + - "5000:5000" + volumes: + - .:/opt/quantifiedcode + command: "/opt/quantifiedcode/initiate.sh" + + +db: + image: mysql:8 + restart: always + environment: + MYSQL_ROOT_PASSWORD: password + +rabbit: + image: rabbitmq:3.6.14 + restart: always \ No newline at end of file diff --git a/initiate.sh b/initiate.sh new file mode 100755 index 0000000..150f207 --- /dev/null +++ b/initiate.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +service rabbitmq-server start + +pip install -r requirements.txt # Refresh requirements +python manage.py setup +python manage.py runserver diff --git a/quantifiedcode/settings/default.yml b/quantifiedcode/settings/default.yml index 115f03f..19c1a18 100755 --- a/quantifiedcode/settings/default.yml +++ b/quantifiedcode/settings/default.yml @@ -2,7 +2,8 @@ backend: url: /api # by default, we use an in-memory instance of SQLite for testing. - db_url_test: 'sqlite+pysqlite:///:memory:' + db_url: 'sqlite:///qc.sqlite' + db_url_test: 'sqlite:///qc_test.sqlite' paths: repositories: "data/repositories" tasks: "data/tasks" @@ -14,8 +15,8 @@ backend: timezone: Europe/Oslo enable_utc: true worker_hijack_root_logger: false - broker_url: 'amqp://qc:qc@localhost:5672/qc' - result_backend: 'amqp://qc:qc@localhost:5672/qc' + broker_url: 'amqp://guest:guest@localhost:5672/' + result_backend: 'amqp://guest:guest@localhost:5672/' task_default_queue: tasks task_queues: - {name: tasks, routing_key: 'task'} @@ -67,6 +68,7 @@ frontend: explore: "/explore" project: "/project" projects: "/projects" +url: http://localhost:5000 render_context: # the current_year is added through Python settings# # (as this is a dynamic variable) diff --git a/requirements.txt b/requirements.txt index 9b5a8c0..6a7ccdc 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ passlib==1.7.1 -#pysqlite alembic==0.9.3 click==6.7 flask==0.12.2 @@ -16,3 +15,5 @@ pyopenssl==17.2.0 python-dateutil==2.6.1 requirements-parser==0.1.0 -e git://github.com/quantifiedcode/checkmate.git#egg=checkmate +# FIXME: NEEDED? +amqp From c23d747220483b2f9e08fffb5178f07446dea15d Mon Sep 17 00:00:00 2001 From: Jonathan Le_Bonzec Date: Wed, 29 Nov 2017 12:05:37 +0100 Subject: [PATCH 5/6] Used separate containers --- Dockerfile | 5 +--- README.md | 2 +- docker-compose.yml | 45 ++++++++++++++++++++--------- initiate.sh => initiate_web.sh | 2 -- initiate_workers.sh | 5 ++++ quantifiedcode/settings/default.yml | 6 ++-- requirements.txt | 3 +- 7 files changed, 43 insertions(+), 25 deletions(-) rename initiate.sh => initiate_web.sh (79%) create mode 100755 initiate_workers.sh diff --git a/Dockerfile b/Dockerfile index 2878d22..db10db1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,12 +3,11 @@ FROM debian:stretch-slim # -- Add some packages to the image RUN apt-get update RUN apt-get install -y \ + curl \ git \ ipython \ - mysql-client \ nano \ python-pip \ - rabbitmq-server \ sudo \ vim \ wget @@ -19,5 +18,3 @@ WORKDIR /opt/quantifiedcode # -- Pre-build requirements RUN pip install -r requirements.txt - -RUN python manage.py setup diff --git a/README.md b/README.md index dc3749e..b74977c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The installation consists of three parts: QuantifiedCode requires the following external dependencies: * A message broker (required for the background tasks message queue). We recommend either RabbitMQ or Redis. -* A database (required for the core application). We recommend PostgreSQL, but SQLite is supported as well. Other database systems might work too (e.g. MySQL), but are currently not officially supported. If you need to run QuantifiedCode on a non-supported database, please get in touch with us and we'll be happy to provide you some guidance. +* A database (required for the core application). We recommend PostgreSQL, but SQLite is supported as well. Other database systems might work too (e.g. MySQL), but are currently not officially supported. If you need to run QuantifiedCode on a non-supported database, please get in touch with us and we'll be happy to provide you some guidance. See http://docs.sqlalchemy.org/en/latest/core/engines.html when editing the config file. ### Download the QuantifiedCode source code diff --git a/docker-compose.yml b/docker-compose.yml index 9e61db6..b2a97c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,25 +1,44 @@ qc: build: . - hostname: qc + volumes: + - .:/opt/quantifiedcode environment: QC_SETTINGS: /opt/quantifiedcode/quantifiedcode/settings/default.yml + +web: + extends: + service: qc + hostname: web + restart: always links: - - db:db + - db:db + - redis:redis ports: - - "80:80" - - "443:443" - - "5000:5000" - volumes: - - .:/opt/quantifiedcode - command: "/opt/quantifiedcode/initiate.sh" + - "80:80" + - "443:443" + - "5000:5000" + command: "bash /opt/quantifiedcode/initiate_web.sh" + +celery: + extends: + service: qc + restart: always + links: + - db:db + - redis:redis + command: "bash /opt/quantifiedcode/initiate_workers.sh" db: - image: mysql:8 + image: postgres:9.6.6 restart: always environment: - MYSQL_ROOT_PASSWORD: password + - POSTGRES_USER=qc + - POSTGRES_DB=quantified + - POSTGRES_PASSWORD=password -rabbit: - image: rabbitmq:3.6.14 - restart: always \ No newline at end of file +redis: + image: redis:3 + restart: always + ports: + - "6379:6379" diff --git a/initiate.sh b/initiate_web.sh similarity index 79% rename from initiate.sh rename to initiate_web.sh index 150f207..a7358a8 100755 --- a/initiate.sh +++ b/initiate_web.sh @@ -1,7 +1,5 @@ #!/bin/bash -service rabbitmq-server start - pip install -r requirements.txt # Refresh requirements python manage.py setup python manage.py runserver diff --git a/initiate_workers.sh b/initiate_workers.sh new file mode 100755 index 0000000..0562ed3 --- /dev/null +++ b/initiate_workers.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +pip install -r requirements.txt # Refresh requirements +python manage.py setup +python manage.py runworker diff --git a/quantifiedcode/settings/default.yml b/quantifiedcode/settings/default.yml index 19c1a18..cb7aa70 100755 --- a/quantifiedcode/settings/default.yml +++ b/quantifiedcode/settings/default.yml @@ -2,7 +2,7 @@ backend: url: /api # by default, we use an in-memory instance of SQLite for testing. - db_url: 'sqlite:///qc.sqlite' + db_url: 'postgresql://qc:password@db/quantified' db_url_test: 'sqlite:///qc_test.sqlite' paths: repositories: "data/repositories" @@ -15,8 +15,8 @@ backend: timezone: Europe/Oslo enable_utc: true worker_hijack_root_logger: false - broker_url: 'amqp://guest:guest@localhost:5672/' - result_backend: 'amqp://guest:guest@localhost:5672/' + broker_url: 'amqp://guest:guest@rabbitmq:5672/' + result_backend: 'amqp://guest:guest@rabbitmq:5672/' task_default_queue: tasks task_queues: - {name: tasks, routing_key: 'task'} diff --git a/requirements.txt b/requirements.txt index 6a7ccdc..72e5b3f 100755 --- a/requirements.txt +++ b/requirements.txt @@ -15,5 +15,4 @@ pyopenssl==17.2.0 python-dateutil==2.6.1 requirements-parser==0.1.0 -e git://github.com/quantifiedcode/checkmate.git#egg=checkmate -# FIXME: NEEDED? -amqp +redis==2.10.6 From f8ede651ead5734419a788153592bbbf2f8a31c7 Mon Sep 17 00:00:00 2001 From: Jonathan Le_Bonzec Date: Wed, 29 Nov 2017 16:37:54 +0100 Subject: [PATCH 6/6] Fixed several mistakes Some variables were undeclared or badly used --- quantifiedcode/backend/api/v1/diff.py | 3 ++- quantifiedcode/backend/api/v1/public_projects.py | 2 +- quantifiedcode/backend/api/v1/snapshot.py | 2 ++ quantifiedcode/backend/tasks/helpers.py | 10 +++++----- quantifiedcode/backend/tasks/periodic.py | 4 ++-- quantifiedcode/backend/tasks/project/delete.py | 2 ++ quantifiedcode/backend/utils/api.py | 2 ++ quantifiedcode/frontend/app.py | 1 + .../git/backend/api/v1/resources/git_snapshots.py | 1 + .../plugins/git/backend/api/v1/resources/project.py | 12 +++++++----- quantifiedcode/settings/default.yml | 8 ++++---- 11 files changed, 29 insertions(+), 18 deletions(-) diff --git a/quantifiedcode/backend/api/v1/diff.py b/quantifiedcode/backend/api/v1/diff.py index 19f47b6..539ec8d 100755 --- a/quantifiedcode/backend/api/v1/diff.py +++ b/quantifiedcode/backend/api/v1/diff.py @@ -154,7 +154,8 @@ def get(self, project_id, snapshot_a_id, snapshot_b_id, path=None): diff_issue_occurrence_table.c.issue_occurrence == issue_occurrence_table.c.pk, *issue_type_query))\ .join(issue_table, issue_table.c.pk == issue_occurrence_table.c.issue)\ - .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code)\ + .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code) \ + .alias('select_table') #we construct the table we will select issues from file_revision_query = select([fr_table.c.pk])\ diff --git a/quantifiedcode/backend/api/v1/public_projects.py b/quantifiedcode/backend/api/v1/public_projects.py index b497008..5d19fae 100644 --- a/quantifiedcode/backend/api/v1/public_projects.py +++ b/quantifiedcode/backend/api/v1/public_projects.py @@ -45,7 +45,7 @@ def get(self): form = PublicProjectsForm(request.args) if not form.validate(): - return {'message' : 'please correct the errors mentioned below', errors: form.errors}, 400 + return {'message' : 'please correct the errors mentioned below', 'errors': form.errors}, 400 data = form.data diff --git a/quantifiedcode/backend/api/v1/snapshot.py b/quantifiedcode/backend/api/v1/snapshot.py index 7c2f68a..a7f6142 100755 --- a/quantifiedcode/backend/api/v1/snapshot.py +++ b/quantifiedcode/backend/api/v1/snapshot.py @@ -106,6 +106,7 @@ def get(self, project_id, snapshot_id, path=None): .join(issue_occurrence_table, issue_occurrence_table.c.file_revision == snapshot_file_revisions_table.c.filerevision)\ .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore) )\ .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code)\ + .alias('fr_select_table') #we construct the file revisions query file_revisions_query = select([snapshot_file_revisions_table.c.filerevision])\ @@ -124,6 +125,7 @@ def get(self, project_id, snapshot_id, path=None): .join(issue_occurrence_table, issue_occurrence_table.c.file_revision == fr_table.c.pk)\ .join(issue_table, and_(issue_table.c.pk == issue_occurrence_table.c.issue, issue_table.c.ignore == ignore))\ .join(issue_classes_cte, issue_classes_cte.c.analyzer_code == issue_table.c.analyzer + ':' + issue_table.c.code)\ + .alias('select_table') #we construct the issues query diff --git a/quantifiedcode/backend/tasks/helpers.py b/quantifiedcode/backend/tasks/helpers.py index c5ebb0d..9844ae9 100755 --- a/quantifiedcode/backend/tasks/helpers.py +++ b/quantifiedcode/backend/tasks/helpers.py @@ -70,7 +70,7 @@ def __init__(self, task, level=logging.INFO, backend=None, ping=True): def __enter__(self): kwargs = { - 'filename': os.path.join(settings.get('backend.path'), settings.get('backend.paths.tasks'), + 'filename': os.path.join(settings.get('project_path'), settings.get('backend.paths.tasks'), self.task.pk + '.log'), 'encoding': "utf-8", 'mode': "w", @@ -139,10 +139,10 @@ def task_is_being_processed(current_task=None): try: res = celery.AsyncResult(task.celery_task_id) with self.backend.transaction(): - if res.state == ResultState.Started: + if res.state == self.ResultState.Started: self.backend.update(task, {'status': 'in_progress'}) return True - elif res.state == ResultState.Pending: + elif res.state == self.ResultState.Pending: if task.get('last_ping'): # if we haven't heard from the task in a while, we mark it as failed... if datetime.datetime.now() - task.last_ping > datetime.timedelta(seconds=60 * 10): @@ -151,9 +151,9 @@ def task_is_being_processed(current_task=None): elif datetime.datetime.now() - task.created_at < datetime.timedelta(seconds=60 * 20): self.backend.update(task, {'status': 'in_progress'}) return True - elif res.state == ResultState.Failure: + elif res.state == self.ResultState.Failure: self.backend.update(task, {'status': 'failed'}) - elif res.state == ResultState.Success: + elif res.state == self.ResultState.Success: self.backend.update(task, {'status': 'succeeded'}) except Task.DoesNotExist: # sometimes a task object will get deleted by another worker while diff --git a/quantifiedcode/backend/tasks/periodic.py b/quantifiedcode/backend/tasks/periodic.py index 604b960..b91af46 100644 --- a/quantifiedcode/backend/tasks/periodic.py +++ b/quantifiedcode/backend/tasks/periodic.py @@ -35,8 +35,8 @@ def delete_pending_project(): @celery.task(queue="delete", ignore_result=False, time_limit=3600) def delete_pending_user(): - pending_projects = backend.filter(User, {'delete': True}).sort('updated_at', 1).limit(100) - logger.debug("%d users marked for deletion" % len(pending_projects)) + pending_users = backend.filter(User, {'delete': True}).sort('updated_at', 1).limit(100) + logger.debug("%d users marked for deletion" % len(pending_users)) for pending_user in pending_users: return delete_user(pending_user.pk, task_id=delete_pending_user.request.id) logger.debug("No users left to delete...") diff --git a/quantifiedcode/backend/tasks/project/delete.py b/quantifiedcode/backend/tasks/project/delete.py index 1366cd5..ec2c4b0 100644 --- a/quantifiedcode/backend/tasks/project/delete.py +++ b/quantifiedcode/backend/tasks/project/delete.py @@ -21,6 +21,7 @@ from checkmate.management.commands.reset import Command as ResetCommand from quantifiedcode.settings import settings, backend from quantifiedcode.backend.settings import BACKEND_PATH +from quantifiedcode.backend.tasks.helpers import ExclusiveTask, TaskLogger from ...worker import celery from ...models import (Project, @@ -28,6 +29,7 @@ UserRole, IssueClass, ProjectIssueClass) +from ..project.analyze import analyze_project @celery.task(queue="analysis", ignore_result=False) def delete_project(project_id, task_id=None): diff --git a/quantifiedcode/backend/utils/api.py b/quantifiedcode/backend/utils/api.py index 149a0bb..3bd553f 100644 --- a/quantifiedcode/backend/utils/api.py +++ b/quantifiedcode/backend/utils/api.py @@ -11,7 +11,9 @@ from quantifiedcode.settings import settings +from flask import jsonify import logging +import traceback logger = logging.getLogger(__name__) diff --git a/quantifiedcode/frontend/app.py b/quantifiedcode/frontend/app.py index dee5bf1..c3275cf 100644 --- a/quantifiedcode/frontend/app.py +++ b/quantifiedcode/frontend/app.py @@ -136,4 +136,5 @@ def configure(app, settings): if __name__ == '__main__': app = get_app(settings) + debug = settings.get('debug') app.run(debug=debug, host='0.0.0.0', port=8000,threaded = False) diff --git a/quantifiedcode/plugins/git/backend/api/v1/resources/git_snapshots.py b/quantifiedcode/plugins/git/backend/api/v1/resources/git_snapshots.py index 8b1c276..c6d23eb 100644 --- a/quantifiedcode/plugins/git/backend/api/v1/resources/git_snapshots.py +++ b/quantifiedcode/plugins/git/backend/api/v1/resources/git_snapshots.py @@ -10,6 +10,7 @@ from __future__ import print_function import logging +import subprocess from datetime import datetime from flask import request diff --git a/quantifiedcode/plugins/git/backend/api/v1/resources/project.py b/quantifiedcode/plugins/git/backend/api/v1/resources/project.py index 199114c..f9b6eb4 100644 --- a/quantifiedcode/plugins/git/backend/api/v1/resources/project.py +++ b/quantifiedcode/plugins/git/backend/api/v1/resources/project.py @@ -79,11 +79,13 @@ def post(self): 'url' : form.url.data } - try: - project = create_project(project_data, git_data, user) - except DuplicateProject: - return ({'message': 'A project with the same name already exists for your account.'}, - 403) + project = create_project(project_data, git_data, user) + # FIXME: handle duplicate project exception + # try: + # pass + # except DuplicateProject: + # return ({'message': 'A project with the same name already exists for your account.'}, + # 403) return ({'message': 'success!', 'project': self.export(project)}, 200) diff --git a/quantifiedcode/settings/default.yml b/quantifiedcode/settings/default.yml index cb7aa70..ff04851 100755 --- a/quantifiedcode/settings/default.yml +++ b/quantifiedcode/settings/default.yml @@ -2,10 +2,10 @@ backend: url: /api # by default, we use an in-memory instance of SQLite for testing. - db_url: 'postgresql://qc:password@db/quantified' + db_url: 'postgresql+psycopg2://qc:password@db/quantified' db_url_test: 'sqlite:///qc_test.sqlite' paths: - repositories: "data/repositories" + git_repositories: "data/repositories" tasks: "data/tasks" celery: config: @@ -15,8 +15,8 @@ backend: timezone: Europe/Oslo enable_utc: true worker_hijack_root_logger: false - broker_url: 'amqp://guest:guest@rabbitmq:5672/' - result_backend: 'amqp://guest:guest@rabbitmq:5672/' + broker_url: 'redis://redis:6379/' + result_backend: 'redis://redis:6379/' task_default_queue: tasks task_queues: - {name: tasks, routing_key: 'task'}