Add upload CSV results files to the front page

innogames · Jan 17, 2018 · 7844dc5 · 7844dc5
1 parent 8808bec
commit 7844dc5
Show file tree

Hide file tree

Showing 10 changed files with 1,227 additions and 22 deletions.
diff --git a/README.md b/README.md
@@ -15,6 +15,11 @@ Consist of several modules:
 4. Administrator - configure different parameters
 
 ## [DASHBOARD] 
+(!) New feature
+Upload CSV files with your test results data and get immediately report and compare
+it with previous tests:
+![alt tag](https://github.com/v0devil/jltom/blob/master/pics/upload.png)
+
 Get tests overview
 ![alt tag](https://github.com/v0devil/jltom/blob/master/pics/dashboard.png)
 

diff --git a/analyzer/forms.py b/analyzer/forms.py
@@ -0,0 +1,7 @@
+from .models import TestResultFile
+from django import forms
+
+class TestResultFileUploadForm(forms.ModelForm):
+    class Meta:
+        model = TestResultFile
+        fields = ('file', )
diff --git a/analyzer/models.py b/analyzer/models.py
@@ -135,10 +135,10 @@ class Meta:
         ]
 
 class TestResultFile(models.Model):
-    project = models.ForeignKey(Project, on_delete=models.CASCADE, default=1)
-    test = models.ForeignKey(Test)
+    #project = models.ForeignKey(Project, on_delete=models.CASCADE, default=1)
+    #test = models.ForeignKey(Test)
     file = models.FileField(upload_to='test_result_files/')
-    uploaded_at = models.DateTimeField(auto_now_add=True)
+    #uploaded_at = models.DateTimeField(auto_now_add=True)
 
     class Meta:
         db_table = 'test_result_file'

diff --git a/analyzer/templates/upload/success.html b/analyzer/templates/upload/success.html
@@ -0,0 +1,6 @@
+{{ result }}
+Test <b>{{ test_name }}</b> was added for project {{ project_id }}
+
+Number of lines: {{ num_lines }}
+
+
diff --git a/analyzer/templates/upload/test_result_file.html b/analyzer/templates/upload/test_result_file.html
@@ -0,0 +1,66 @@
+<head>
+  <meta charset="UTF-8">
+  <title>Upload test result file</title>
+  {% load static %}
+  <link rel="stylesheet" type="text/css" href="{% static 'css/jquery-ui.min.css' %}">
+  <link rel="stylesheet" type="text/css" href="{% static 'css/bootstrap.min.css' %}">
+  <link rel="stylesheet" type="text/css" href="{% static 'css/bootstrap-select.min.css' %}">
+  <link rel="stylesheet" type="text/css" href="{% static 'css/theme.bootstrap_2.min.css' %}">
+  <link rel="stylesheet" type="text/css" href="{% static 'css/my.css' %}" />
+
+
+</head>
+
+<body>
+  <div class="col-xs-5">
+    <div class="panel panel-default">
+      <div class="panel-heading">Upload test result</div>
+      <div class="panel-body">
+        <p class="bg-warning">
+        Currently supports only CSV-result files. 
+        CSV result file mast have 5 mandatory fields:
+        timestamp, response_time, url, responseCode, success.
+        Example:
+        1515027139558,310,Action_name,200,true
+
+        If you file have different amount of columns or different order, place 
+        this data to field in a bottom. Timestamp is mandatory and must be always on the first place
+
+        </p>
+        <form method="POST" enctype="multipart/form-data" class="form-horizontal">
+          {% csrf_token %}
+          <div class="form-group">
+            <label for="name" class="control-label">File: </label>
+            <input type="file" name="csv_file" id="csv_file" required="True" class="form-control">
+          </div>
+          <div class="form-group">
+              <label for="csv_file_fields">CSV file fields:</label>
+              <input name="csv_file_fields" type="text" class="form-control" id="csv_file_fields" value="response_time,url,responseCode,success,threadName,failureMessage,grpThreads,allThreads">
+            </div>
+          <div class="form-group">
+            <label for="test_display_name">Test name:</label>
+            <input name="test_name" type="text" class="form-control" id="test_display_name" value="Enter test name">
+          </div>
+          <div class="form-group">
+            <label for="select_project">Select project:</label>
+            <select name="project_id" id="select_project" class="selectpicker">
+              {% for project in projects %}
+              <option value="{{ project.id }}">{{ project.project_name }}</option>
+              {% endfor %}
+              <option value="0">Create new project</option>
+            </select>
+          </div>
+          <div class="form-group">
+              <button class="btn btn-primary">
+                <span class="glyphicon glyphicon-upload"></span>Upload </button>
+          </div>
+      </div>
+    </div>
+  </div>
+  </form>
+</body>
+<script src="{% static 'js/jquery-3.1.1.min.js' %}"></script>
+<script src="{% static 'js/jquery-ui.min.js' %}"></script>
+<script src="{% static 'js/bootstrap.min.js' %}"></script>
+<script src="{% static 'js/bootstrap-select.min.js' %}"></script>
+<script src="{% static 'js/main_script.js' %}"></script>
diff --git a/analyzer/urls.py b/analyzer/urls.py
@@ -49,5 +49,5 @@
         views.action_graphs),
     url(r'^test/(?P<test_id_1>\d+)/(?P<test_id_2>\d+)/compare_aggregate_data/$',
         views.tests_compare_aggregate_new),   
-    url(r'^test-result-file-upload/$', views.TestResultFileUploadView.as_view(), name='test_result_file_upload'), 
+    url(r'^upload/test_result_file/$', views.upload_test_result_file, name='upload_test_result_file'), 
 ]
diff --git a/analyzer/views.py b/analyzer/views.py
@@ -3,16 +3,21 @@
 import logging
 import math
 import time
+import os
+
+import numpy as na
+from pylab import *
 from collections import OrderedDict
 from decimal import Decimal
 
 from django import template
 from django.db.models import Avg, FloatField, Max, Min, Sum
 from django.db.models.expressions import F, RawSQL
-from django.http import HttpResponse, JsonResponse
+from django.http import HttpResponse, JsonResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.views.generic import TemplateView
 from django.views import View
+from django.urls import reverse
 
 import pandas as pd
 from administrator.models import Configuration
@@ -22,11 +27,15 @@
 from models import (Action, Project, Server, ServerMonitoringData, Test,
                     TestActionAggregateData, TestActionData, TestData, Error,
                     TestError, TestResultFile)
+from forms import TestResultFileUploadForm
 from scipy import stats
 from django.db.models import Func
 from django.template.loader import render_to_string
+from django.core.files.storage import default_storage
+from django.core.files.base import ContentFile
 
 logger = logging.getLogger(__name__)
+dateconv = np.vectorize(datetime.datetime.fromtimestamp)
 
 
 class Round(Func):
@@ -63,24 +72,136 @@ def to_pivot(data, a, b, c):
     return df_pivot
 
 
-class TestResultFileUploadView(View):
-    def get(self, request):
-        test_result_file_list = TestResultFile.objects.all()
-        return render(self.request, 'test_result_upload/index.html',
-                      {'test_result_files': test_result_file_list})
-
-    def post(self, request):
-        form = TestResultFileUploadForm(self.request.POST, self.request.FILES)
-        if form.is_valid():
-            test_result_file = form.save()
+def upload_test_result_file(request):
+    if request.method == 'GET':
+        projects = Project.objects.values()
+        return render(request, 'upload/test_result_file.html',
+                      {'projects': projects})
+    csv_file = request.FILES["csv_file"]
+    csv_file_fields = request.POST.get('csv_file_fields', '1')
+    test_name = request.POST.get('test_name', '1')
+    project_id = int(request.POST.get('project_id', '0'))
+    # Create new project
+    if project_id == 0:
+        project = Project(project_name="New project",)
+        project.save()
+        project_id = project.id
+    test = Test(
+        project_id=project_id,
+        display_name=test_name,
+        show=True,
+        start_time=int(time.time() * 1000))
+    test.save()
+    test_id = test.id
+    path = default_storage.save('test_result_data.csv',
+                                ContentFile(csv_file.read()))
+    jmeter_results_file = path
+    if os.path.exists(jmeter_results_file):
+        df = pd.DataFrame()
+        if os.stat(jmeter_results_file).st_size > 1000007777:
+            logger.debug("Executing a parse for a huge file")
+            chunks = pd.read_table(
+                jmeter_results_file, sep=',', index_col=0, chunksize=3000000)
+            for chunk in chunks:
+                chunk.columns = [csv_file_fields.split(',')]
+                chunk = chunk[~chunk['URL'].str.contains('exclude_')]
+                df = df.append(chunk)
+        else:
+            df = pd.read_csv(
+                jmeter_results_file, index_col=0, low_memory=False)
+            df.columns = [csv_file_fields.split(',')]
+            df = df[~df['url'].str.contains('exclude_', na=False)]
+
+        df.columns = [csv_file_fields.split(',')]
+
+        df.index = pd.to_datetime(dateconv((df.index.values / 1000)))
+        num_lines = df['response_time'].count()
+        logger.debug('Number of lines in file: {}'.format(num_lines))
+        unique_urls = df['url'].unique()
+        for url in unique_urls:
+            url = str(url)
+            if not Action.objects.filter(
+                    url=url, project_id=project_id).exists():
+                logger.debug("Adding new action: " + str(url) + " project_id: "
+                             + str(project_id))
+                a = Action(url=url, project_id=project_id)
+                a.save()
+            a = Action.objects.get(url=url, project_id=project_id)
+            action_id = a.id
+            if not TestActionData.objects.filter(
+                    action_id=action_id, test_id=test_id).exists():
+                logger.debug("Adding action data: {}".format(url))
+                df_url = df[(df.url == url)]
+                url_data = pd.DataFrame()
+                df_url_gr_by_ts = df_url.groupby(pd.TimeGrouper(freq='1Min'))
+                url_data['avg'] = df_url_gr_by_ts.response_time.mean()
+                url_data['median'] = df_url_gr_by_ts.response_time.median()
+                url_data['count'] = df_url_gr_by_ts.success.count()
+                df_url_gr_by_ts_only_errors = df_url[(
+                    df_url.success == False
+                )].groupby(pd.TimeGrouper(freq='1Min'))
+                url_data[
+                    'errors'] = df_url_gr_by_ts_only_errors.success.count()
+                url_data['test_id'] = test_id
+                url_data['url'] = url
+                output_json = json.loads(
+                    url_data.to_json(orient='index', date_format='iso'),
+                    object_pairs_hook=OrderedDict)
+                for row in output_json:
+                    data = {
+                        'timestamp': row,
+                        'avg': output_json[row]['avg'],
+                        'median': output_json[row]['median'],
+                        'count': output_json[row]['count'],
+                        'url': output_json[row]['url'],
+                        'errors': output_json[row]['errors'],
+                        'test_id': output_json[row]['test_id'],
+                    }
+                    test_action_data = TestActionData(
+                        test_id=output_json[row]['test_id'],
+                        action_id=action_id,
+                        data=data)
+                    test_action_data.save()
+
+                url_agg_data = dict(
+                    json.loads(df_url['response_time'].describe().to_json()))
+                url_agg_data['99%'] = df_url['response_time'].quantile(.99)
+                url_agg_data['90%'] = df_url['response_time'].quantile(.90)
+                url_agg_data['weight'] = float(df_url['response_time'].sum())
+                url_agg_data['errors'] = df_url[(
+                    df_url['success'] == False)]['success'].count()
+                test_action_aggregate_data = TestActionAggregateData(
+                    test_id=test_id, action_id=action_id, data=url_agg_data)
+                test_action_aggregate_data.save()
+
+        # zip_results_file(jmeter_results_file)
+
+        test_overall_data = pd.DataFrame()
+        df_gr_by_ts = df.groupby(pd.TimeGrouper(freq='1Min'))
+        test_overall_data['avg'] = df_gr_by_ts.response_time.mean()
+        test_overall_data['median'] = df_gr_by_ts.response_time.median()
+        test_overall_data['count'] = df_gr_by_ts.response_time.count()
+        test_overall_data['test_id'] = test_id
+        output_json = json.loads(
+            test_overall_data.to_json(orient='index', date_format='iso'),
+            object_pairs_hook=OrderedDict)
+        for row in output_json:
             data = {
-                'is_valid': True,
-                'name': test_result_file.file.name,
-                'url': test_result_file.file.url
+                'timestamp': row,
+                'avg': output_json[row]['avg'],
+                'median': output_json[row]['median'],
+                'count': output_json[row]['count']
             }
-        else:
-            data = {'is_valid': False}
-        return JsonResponse(data)
+            test_data = TestData(
+                test_id=output_json[row]['test_id'], data=data)
+            test_data.save()
+
+    return render(request, "upload/success.html", {
+        'result': 'ok',
+        'test_name': test_name,
+        'project_id': project_id,
+        'num_lines': num_lines
+    })
 
 
 def configure_page(request, project_id):