Merge pull request #656 from yandex/release

Release 1.10.3
yandex · Oct 15, 2018 · b5ee8c7 · b5ee8c7
2 parents 5ec56e6 + fc096b4
commit b5ee8c7
Show file tree

Hide file tree

Showing 24 changed files with 320 additions and 495 deletions.
diff --git a/docs/conf.py b/docs/conf.py
@@ -54,7 +54,7 @@
 # The short X.Y version.
 version = '1.10'
 # The full version, including alpha/beta/rc tags.
-release = '1.10.2'
+release = '1.10.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

diff --git a/docs/config_reference.rst b/docs/config_reference.rst
@@ -988,44 +988,4 @@ Telegraf
  ``10s``
   10 seconds
  ``2m``
-  2 minutes
-
-YASM
-====
-
-``panels`` (dict)
------------------
-*\- (no description).* **Required.**
-
-:valueschema:
- :schema:
-  :default_signals:
-   :default:
-    True
-   :type:
-    boolean
-  :host:
-   :required:
-    True
-   :type:
-    string
-  :signals:
-   :required:
-    False
-   :type:
-    list
-  :tags:
-   :required:
-    True
-   :type:
-    string
- :type:
-  dict
-
-``timeout`` (integer)
----------------------
-*\- (no description). Default:* ``120``
-
-``verbose_logging`` (boolean)
------------------------------
-*\- (no description). Default:* ``False``
+  2 minutes
diff --git a/docs/core_and_modules.rst b/docs/core_and_modules.rst
@@ -756,72 +756,53 @@ Disable phantom first (unless you really want to keep it active alongside at you
 
   # pandora_config.yml
 
-      pools: [{
-        "name": "dummy pool",
-        "gun": {"type": "log"},
-        "ammo": {
-          "type": "dummy/log",
-          "AmmoLimit": 10000000
-          },
-        "result": {
-          "type": "phout",
-          "destination": "./phout.log"
-          },
-        "shared-limits": false,
-        "user-limiter": {"type": "unlimited"},
-        "startup-limiter": {
-          "type": "periodic",
-          "batch": 1,
-          "max": 5,
-          "period": "0.5s"
-          }
-        }
-      ]
+  pools:
+    - id: HTTP pool                    # pool name (for your choice)
+      gun:
+        type: http                     # gun type
+        target: example.com:80         # gun target
+      ammo:
+        type: uri                      # ammo format
+        file: ./ammo.uri               # ammo file path
+      result:
+        type: phout                    # report format (phout is compatible for Yandex.Tank)
+        destination: ./phout.log       # report file name
+
+      rps:                             # RPS scheduler - controls throughput over test
+        type: line                     # linear growth load
+        from: 1                        # from 1 responses per second
+        to: 5                          # to 5 responses per second
+        duration: 2s                   # for 2 seconds
+
+      startup:                         # startup scheduler - control the level of parallelism
+        type: once                     # start 5 instances
+        times: 5                       #
+
+
+You may specify pandora config contents in tank's config file via ```config_content``` option.
+This option has more priority over config_file option.
+
+Create ```ammo.uri``` file, put your ammo inside and start the test.
 
-Schedules
----------
-
-The first schedule type is ``periodic`` schedule. It is defined as ``periodic(<batch_size>, <period>, <limit>)``.
-Pandora will issue one batch of size ``batch_size``, once in ``period`` seconds, maximum of ``limit`` ticks. Those ticks may be
-used in different places, for example as a limiter for user startups or as a limiter for each user request rate.
-
-Example:
-::
-
-    startup_schedule = periodic(2, 0.1, 100)
-    user_schedule = periodic(10, 15, 100)
-    shared_schedule = 0
-
-Start 2 users every 0.1 seconds, 100 batches, maximum of 2 * 100 = 200 users. Each user will issue requests in batches of 10 requests, every 15 seconds, maximum
-of 100 requests. All users will read from one ammo source.
-
-Second schedule type is ``linear``. It is defined like this: ``linear(<start_rps>, <end_rps>, <time>)``.
-
-Example:
-::
+.. code-block:: yaml
 
-    user_schedule = linear(.1, 10, 10m)
-    shared_schedule = 1
+  # ammo.uri
 
-The load will raise from .1 RPS (1 request in 10 seconds) until 10 RPS during 10 minutes. Since
-``shared_schedule`` is 1, this defines the overall load.
+  /my/first/url
+  /my/second/url
 
-The last schedule type is ``unlimited``. It has no parameters and users will shoot as soon
-as possible. It is convenient to use this type of load to find out maximum performance of a
-service and its level of parallelism. You should limit the loop number if you want the test
-to stop eventually. 
+Schedules
+---------
+```Pandora``` has two main RPS scheduler types:
+  1. ``line`` - makes linear load, where ```from``` and ```to``` are start and end,
+  `duration` is a time for linear load increase from ```from``` to ```to```.
 
-Example:
-::
+  2. ``const`` - makes constant load, where ```ops``` is a load value for ```duration``` time.
 
-    loop = 1000000
-    startup_schedule = periodic(2, 10, 50)
-    user_schedule = unlimited()
-    shared_schedule = 0
 
-Start 2 users every 10 seconds. Every user will shoot without any limits (next request is sended
-as soon as the previous response have been received). This is analogous to phantom's instances
-schedule mode.
+Custom_guns
+-----------
+You can create you own Golang-based gun with `pandora`. Feel free to find examples at [pandora documentation](https://yandexpandora.readthedocs.io/en/develop/)
 
 
 ******************
@@ -830,7 +811,7 @@ Artifact uploaders
 
 .. note::
 
-  Graphite uploader, InfluxDB uploader and BlazeMeter Sense are not currently supported in the last Yandex.Tank version.
+  Graphite uploader and BlazeMeter Sense are not currently supported in the last Yandex.Tank version.
   If you want one of them, use 1.7 branch.
 
 Yandex.Overload
@@ -861,6 +842,49 @@ Example:
     job_name: test
     job_dsc: test description
 
+
+InfluxDB
+========
+
+InfluxDB uploader.
+
+Options
+-------
+
+:address:
+  (Optional) InfluxDB address. (Default: 'localhost')
+:port:
+  (Optional) InfluxDB port. (Default: 8086)
+:database:
+  (Optional) InfluxDB database. (Default: 'mydb')
+:username:
+  (Optional) InfluxDB user name. (Default: 'root')
+:password:
+  (Optional) InfluxDB password. (Default: 'root')
+:labeled:
+  (Optional) Send labels (ammo tags) to influx. (Default: false)
+:prefix_measurement:
+  (Optional) Add prefix to measurement name. (Default: '')
+:tank_tag:
+  (Optional) Tank tag. (Default: 'unknown')
+:grafana_root:
+  (Optional) Grafana root path. Used to generate link to grafana dashboard.
+:grafana_dashboard:
+  (Optional) Grafana dashboard name. Used to generate link to grafana dashboard.
+
+Example:
+
+.. code-block:: yaml
+
+  influx:
+    enabled: true
+    address: yourhost.tld
+    database: yourbase
+    tank_tag: 'mytank'
+    prefix_measurement: 'your_test_prefix_'
+    labeled: true
+
+
 ***********
 Handy tools
 ***********

diff --git a/docs/generator_tuning.rst b/docs/generator_tuning.rst
@@ -31,7 +31,7 @@ Otherwise if port is unreachable:
 Routing
 *******
 
-OK, service is reachable, next thing
+OK, the service is reachable, next thing
 you should know is how far Yandex.Tank is located from the service you'd
 like to test. Heavy load can make switch to be unresponsible or to
 reboot, or at least it may lead to network losses, so the test results
@@ -59,13 +59,13 @@ execution of ``tracepath`` command or it analogs
     6:  8.example.com (203.0.113.1)                 0.525ms asymm  5
     7:  no reply
 
-In second example you'd better find another closer located tank.
+In the second example you'd better find another closer located tank.
 
 ******
 Tuning
 ******
 
-To achieve the top most performance you should tune the source server
+To achieve top performance you should tune the source server
 system limits:
 
 .. code-block:: bash

diff --git a/docs/tutorial.rst b/docs/tutorial.rst
@@ -106,7 +106,7 @@ To specify external ammo file use ``ammofile`` option.
 
 .. note::
   You can specify URL to ammofile, http(s). Small ammofiles (~<100MB) will be downloaded as is,
-  to directory ``/tmp/<hash>``, large files will be readed from stream.
+  to directory ``/tmp/<hash>``, large files will be read from stream.
 
 .. note::
 
@@ -154,7 +154,7 @@ Parameter ``uris`` contains uri, which should be used for requests generation.
 
 .. note::
 
-  Pay attention to sample above, because whitespaces in multiline ``uris`` and ``headers`` options are important.
+  Pay attention to the sample above, because whitespaces in multiline ``uris`` and ``headers`` options are important.
 
 URI-style, URIs in file
 -----------------------
@@ -174,7 +174,7 @@ Create a file with declared requests: ``ammo.txt``
   [Cookie: test]
   /buy/?rt=0&station_to=7&station_from=9
 
-File consist of list of URIs and headers to be added to every request defined below.
+File consists of list of URIs and headers to be added to every request defined below.
 Every URI must begin from a new line, with leading ``/``.
 Each line that begins from ``[`` is considered as a header.
 Headers could be (re)defined in the middle of URIs, as in sample above. 
@@ -340,7 +340,7 @@ file ``phout.txt`` is being written, which could be analyzed later.
 If you need more human-readable report, you can try Report plugin,
 You can found it `here <https://github.com/yandex-load/yatank-online>`_
 
-If you need to upload results to external storage, such as Graphite or InfluxDB, you can use one of existing artifacts uploading modules :doc:`core_and_modules`
+If you need to upload results to an external storage, such as Graphite or InfluxDB, you can use one of existing artifacts uploading modules :doc:`core_and_modules`
 
 Tags
 ====

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 setup(
     name='yandextank',
-    version='1.10.2',
+    version='1.10.3',
     description='a performance measurement tool',
     longer_description='''
 Yandex.Tank is a performance measurement and load testing automatization tool.
@@ -20,7 +20,7 @@
         'psutil>=1.2.1', 'requests>=2.5.1', 'paramiko>=1.16.0',
         'pandas>=0.18.0', 'numpy>=1.12.1', 'future>=0.16.0',
         'pip>=8.1.2',
-        'pyyaml>=3.12', 'cerberus==1.2', 'influxdb>=5.0.0', 'netort==0.2.4',
+        'pyyaml>=3.12', 'cerberus==1.2', 'influxdb>=5.0.0', 'netort==0.2.6',
     ],
     setup_requires=[
     ],
@@ -63,7 +63,6 @@
         'yandextank.plugins.Influx': ['config/*'],
         'yandextank.plugins.JMeter': ['config/*'],
         'yandextank.plugins.JsonReport': ['config/*'],
-        'yandextank.plugins.YASM': ['config/*'],
         'yandextank.plugins.Pandora': ['config/*'],
         'yandextank.plugins.Phantom': ['config/*'],
         'yandextank.plugins.RCAssert': ['config/*'],

diff --git a/yandextank/aggregator/tank_aggregator.py b/yandextank/aggregator/tank_aggregator.py
@@ -117,12 +117,17 @@ def is_test_finished(self):
     def end_test(self, retcode):
         retcode = self.generator.end_test(retcode)
         if self.reader:
+            logger.debug('Closing gun reader')
             self.reader.close()
         if self.stats_reader:
+            logger.debug('Closing stats reader')
             self.stats_reader.close()
         if self.drain:
+            logger.debug('Waiting for gun drain to finish')
             self.drain.wait()
+            logger.debug('Waiting for stats drain to finish')
             self.stats_drain.wait()
+        logger.debug('Collecting remaining data')
         self._collect_data(end=True)
         if self.drain:
             self.drain.join()

diff --git a/yandextank/common/util.py b/yandextank/common/util.py
@@ -18,7 +18,7 @@
 
 
 class SecuredShell(object):
-    def __init__(self, host, port, username, timeout):
+    def __init__(self, host, port, username, timeout=10):
         self.host = host
         self.port = port
         self.username = username
@@ -62,7 +62,7 @@ def fixed_AES_new(key, *ls):
                 self.host,
                 port=self.port,
                 username=self.username,
-                timeout=self.timeout, )
+                timeout=self.timeout)
         return client
 
     def execute(self, cmd):
@@ -89,15 +89,22 @@ def send_file(self, local_path, remote_path):
                 local=local_path, host=self.host, remote=remote_path))
 
         with self.connect() as client, client.open_sftp() as sftp:
-            result = sftp.put(local_path, remote_path)
+            result = sftp.put(local_path, remote_path, self.get_progress_logger(local_path))
         return result
 
+    @staticmethod
+    def get_progress_logger(name):
+
+        def print_progress(done, total):
+            logger.info("Transferring {}: {}%".format(name, done * 100 / total))
+        return print_progress
+
     def get_file(self, remote_path, local_path):
         logger.info(
             "Receiving from {host}:[{remote}] to [{local}]".format(
                 local=local_path, host=self.host, remote=remote_path))
         with self.connect() as client, client.open_sftp() as sftp:
-            result = sftp.get(remote_path, local_path)
+            result = sftp.get(remote_path, local_path, self.get_progress_logger(remote_path))
         return result
 
     def async_session(self, cmd):