Merge pull request #8 from SurpSG/improve-cartesian-performance

Improve cartesian product Collection<Collection<T>> performance

README.md

@@ -24,7 +24,7 @@ To get even better throughput(see [Benchmarking](#benchmarking) section), comput
     <dependency>
       <groupId>com.sgnatiuk</groupId>
       <artifactId>kombi</artifactId>
-      <version>3.0.0</version>
+      <version>3.0.1</version>
     </dependency>
 </dependencies>
 ```
@@ -35,7 +35,7 @@ repositories {
 }
 
 dependencies {
-    compile 'com.sgnatiuk:kombi:3.0.0'
+    compile 'com.sgnatiuk:kombi:3.0.1'
 }
 ```
 ## Combinations
@@ -155,49 +155,61 @@ There is an overloaded builder method `CartesianBuilder.cartesianProductOf(...,
 ```
 
 ## Benchmarking
-Measured throughput of generation of combination/cartesian product item (generated items per second)
+Measured time of generation of combination/cartesian product items (microseconds to generate all items)
 
-Benchmark results:
+Feel free to run benchmarks by yourself:
+```
+./gradlew clean kombi-jmh:jmh
+```
+
+
+Benchmark results(less is better):
 ```
 Ubuntu 18.04.4 LTS
 Intel® Core™ i7-6500U CPU @ 2.50GHz × 4
-JMH version: 1.19
-VM version: JDK 1.8.0_242, VM 25.242-b08
-VM invoker: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
-VM options: -Xms4g -Xmx4g
-Warmup: 10 iterations, 1 s each
-Measurement: 10 iterations, 1 s each
-Timeout: 10 min per iteration
-Threads: 1 thread, will synchronize iterations
-Benchmark mode: Average time, time/op
+# JMH version: 1.22
+# VM version: JDK 1.8.0_242, OpenJDK 64-Bit Server VM, 25.242-b08
+# VM invoker: /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java
+# VM options: -Xms512m -Xmx1g
+# Warmup: 5 iterations, 10 s each
+# Measurement: 5 iterations, 10 s each
+# Timeout: 10 min per iteration
+# Threads: 1 thread, will synchronize iterations
+# Benchmark mode: Average time, time/op
 
 Benchmark                                                                         (itemsQuantity)  Mode  Cnt        Score        Error  Units
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                             3  avgt   10        0.416 ±      0.004  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                             5  avgt   10        8.983 ±      0.045  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                             7  avgt   10      525.439 ±      2.649  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                            11  avgt   10  6402100.731 ± 208391.654  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                              3  avgt   10        0.835 ±      0.010  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                              5  avgt   10       16.800 ±      0.203  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                              7  avgt   10      745.600 ±     13.021  us/op
-c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                             11  avgt   10  8956251.389 ±  54373.550  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                             3  avgt   10        0.525 ±      0.002  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                             5  avgt   10       10.777 ±      0.165  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                             7  avgt   10      535.627 ±     44.617  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                            11  avgt   10  5461936.892 ±  20583.561  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder                3  avgt   10        0.608 ±      0.007  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder                5  avgt   10       12.682 ±      1.236  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder                7  avgt   10      578.047 ±      3.468  us/op
-c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder               11  avgt   10  6385427.416 ± 271354.148  us/op
-c.s.b.cartesian.CartesianMapBenchmark.Kombi_cartesianProduct_Maps                               7  avgt   10     1019.071 ±     15.560  us/op
-c.s.b.cartesian.CartesianMapBenchmark.Kombi_cartesianProduct_Maps_keepingOrder                  7  avgt   10     1086.156 ±     10.034  us/op
-c.s.b.combination.CombinationBenchmark.Kombi_combinations_list                                 11  avgt   10      220.303 ±      1.957  us/op
-c.s.b.combination.CombinationBenchmark.Kombi_combinations_list                                 19  avgt   10    78645.589 ±   1509.309  us/op
-c.s.b.combination.CombinationBenchmark.Kombi_combinations_map                                  11  avgt   10      463.854 ±      1.873  us/op
-c.s.b.combination.CombinationBenchmark.Kombi_combinations_map                                  19  avgt   10   169522.011 ±    695.623  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                             3  avgt   10        0.396 ±      0.002  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                             5  avgt   10        8.592 ±      0.190  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                             7  avgt   10      507.613 ±      3.286  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Lists                            11  avgt   10  6047357.993 ±  11218.642  us/op
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                             3  avgt   10        0.363 ±      0.005  us/op
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                             5  avgt   10        6.838 ±      0.176  us/op      
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                             7  avgt   10      374.914 ±     69.084  us/op
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists                            11  avgt   10  3360446.209 ±  45311.037  us/op
 
-```
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                              3  avgt   10        0.815 ±      0.066  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                              5  avgt   10       15.611 ±      0.578  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                              7  avgt   10      645.309 ±     43.153  us/op
+c.s.b.cartesian.CartesianListBenchmark.Guava_cartesianProduct_Sets                             11  avgt   10  7492806.803 ± 137744.113  us/op
+
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder                3  avgt   10        0.449 ±      0.059  us/op
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder                5  avgt   10        8.432 ±      0.260  us/op
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder                7  avgt   10      407.532 ±      1.454  us/op
+c.s.b.cartesian.CartesianListBenchmark.Kombi_cartesianProduct_Lists_keepingOrder               11  avgt   10  4061078.368 ±  42057.603  us/op
+
+c.s.b.cartesian.CartesianMapBenchmark.Kombi_cartesianProduct_Maps                               5  avgt   10       18.971 ±      1.902  us/op
+c.s.b.cartesian.CartesianMapBenchmark.Kombi_cartesianProduct_Maps                               7  avgt   10     1050.295 ±     20.054  us/op
+c.s.b.cartesian.CartesianMapBenchmark.Kombi_cartesianProduct_Maps_keepingOrder                  5  avgt   10       19.619 ±      2.603  us/op
+c.s.b.cartesian.CartesianMapBenchmark.Kombi_cartesianProduct_Maps_keepingOrder                  7  avgt   10     1212.077 ±    139.650  us/op
+
+c.s.b.combination.CombinationBenchmark.Kombi_combinations_list                                 11  avgt   10      216.704 ±      3.633  us/op
+c.s.b.combination.CombinationBenchmark.Kombi_combinations_list                                 19  avgt   10    77641.630 ±   1561.054  us/op
+
+c.s.b.combination.CombinationBenchmark.Kombi_combinations_map                                  11  avgt   10      467.513 ±      2.014  us/op
+c.s.b.combination.CombinationBenchmark.Kombi_combinations_map                                  19  avgt   10   170390.506 ±   3108.922  us/op
 
-Feel free to run benchmarks by yourself:
-```
-./gradlew clean kombi-jmh:jmh
 ```
+Comparing performance with Guava(microseconds per generation, less is better):
+
+![](kombi-jmh/charts/items_39916800.jpg)
+

RELEASENOTES.md

@@ -0,0 +1,12 @@
+v3.0.1
+* 'Cartesian product' from Collection<Collection<T>> performance tuning.(~40% speed-up)
+
+v3.0.0
+* The library is fully migrated from Kotlin to Java to avoid adding of Kotlin runtime dependency to Java-only projects.
+* Small performance fixes in the generation of the cartesian product.
+
+v2.2
+* Provided stream support(java.util.stream.Stream)
+
+v2.1
+* Provided split functionality allowing to split the cartesian product or combinations generation into equals chunks, so cartesian product can be generated in few threads independently.  

kombi-jmh/build.gradle

@@ -5,6 +5,7 @@ plugins {
 description = ''
 dependencies {
     jmh project(':kombi-lib')
+    jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.22'
     compile "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version"
     compile group: 'com.google.guava', name: 'guava', version: '28.2-jre'
 
@@ -19,33 +20,12 @@ compileTestKotlin {
 
 jmhJar.archiveFileName = 'benchmarks.jar'
 jmh {
-    jmhVersion = '1.21' // Specifies JMH version
-
-    include = ['.*'] // include pattern (regular expression) for benchmarks to be executed
-    jvmArgs = ['-Xms4g', '-Xmx4g']
-
-
-    benchmarkMode = ['avgt'] // Benchmark mode. Available modes are: [Throughput/thrpt, AverageTime/avgt, SampleTime/sample, SingleShotTime/ss, All/all]
-    timeUnit = 'us'// Output time unit. Available time units are: [m, s, ms, us, ns].
-    verbosity = 'NORMAL' // Verbosity mode. Available modes are: [SILENT, NORMAL, EXTRA]
-    forceGC = true // Should JMH force GC between iterations?
-    failOnError = false // Should JMH fail immediately if any benchmark had experienced the unrecoverable error?
-
-    operationsPerInvocation = 1 // Operations per invocation.
-    batchSize = 1 // Batch size: number of benchmark method calls per operation. (some benchmark modes can ignore this setting)
-    warmupBatchSize = 1 // Warmup batch size: number of benchmark method calls per operation.
-    fork = 1 // How many times to forks a single benchmark. Use 0 to disable forking altogether
-    warmupForks = 0 // How many warmup forks to make for a single benchmark. 0 to disable warmup forks.
-    threads = 1 // Number of worker threads to run with.
-
-    iterations = 5 // Number of measurement iterations to do.
-    warmupIterations = 5 // Number of warmup iterations to do.
+    jmhVersion = '1.22'
 
+    jvmArgs = ['-Xms512m', '-Xmx1g']
     humanOutputFile = project.file("${project.buildDir}/reports/jmh/human.txt") // human-readable output file
     resultsFile = project.file("${project.buildDir}/reports/jmh/results.csv") // results file
-    resultFormat = 'CSV' // Result format type (one of CSV, JSON, NONE, SCSV, TEXT)
-
-    //more options by the link https://github.com/melix/jmh-gradle-plugin
+    resultFormat = 'TEXT' // Result format type (one of CSV, JSON, NONE, SCSV, TEXT)
 }
 
 

kombi-jmh/src/jmh/kotlin/com/sgnatiuk/benchmark/cartesian/CartesianListBenchmark.kt

@@ -5,8 +5,14 @@ import com.google.common.collect.Sets
 import com.sgnatiuk.cartesian.CartesianBuilder.cartesianProductOf
 import org.openjdk.jmh.annotations.*
 import org.openjdk.jmh.infra.Blackhole
+import java.util.concurrent.TimeUnit
 
 @State(Scope.Benchmark)
+@Fork(2)
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
 open class CartesianListBenchmark {
 
     @Param("3", "5", "7", "11")
@@ -18,44 +24,49 @@ open class CartesianListBenchmark {
     @Setup(Level.Trial)
     fun doSetup() {
         listOfLists = List(itemsQuantity) { i ->
-            List(i + 1) { it }
+            List(i + 1) { it + 1 }
         }
         listOfSets = listOfLists.map { it.toSet() }
+        println("\n=================================================")
+        println("combinationsQuantity=${cartesianProductOf(listOfLists).combinationsCount()}")
+        println("Data:")
+        listOfLists.forEach {
+            println(it)
+        }
+        println("=================================================\n")
     }
 
     @Benchmark
     fun Kombi_cartesianProduct_Lists(blackhole: Blackhole) {
         for (combination in cartesianProductOf(listOfLists, false)) {
-            for (combinationItem in combination) {
-                blackhole.consume(combinationItem)
-            }
+            iterateWithIterator(combination, blackhole)
         }
     }
 
     @Benchmark
     fun Kombi_cartesianProduct_Lists_keepingOrder(blackhole: Blackhole) {
         for (combination in cartesianProductOf(listOfSets, true)) {
-            for (combinationItem in combination) {
-                blackhole.consume(combinationItem)
-            }
+            iterateWithIterator(combination, blackhole)
         }
     }
 
     @Benchmark
     fun Guava_cartesianProduct_Sets(blackhole: Blackhole) {
         for (combination in Sets.cartesianProduct(listOfSets)) {
-            for (combinationItem in combination) {
-                blackhole.consume(combinationItem)
-            }
+            iterateWithIterator(combination, blackhole)
         }
     }
 
     @Benchmark
     fun Guava_cartesianProduct_Lists(blackhole: Blackhole) {
         for (combination in Lists.cartesianProduct(listOfLists)) {
-            for (combinationItem in combination) {
-                blackhole.consume(combinationItem)
-            }
+            iterateWithIterator(combination, blackhole)
+        }
+    }
+
+    private fun iterateWithIterator(combination: MutableList<Int>, blackhole: Blackhole) {
+        for (combinationItem in combination) {
+            blackhole.consume(combinationItem)
         }
     }
 }

kombi-jmh/src/jmh/kotlin/com/sgnatiuk/benchmark/cartesian/CartesianMapBenchmark.kt

@@ -3,11 +3,17 @@ package com.sgnatiuk.benchmark.cartesian
 import com.sgnatiuk.cartesian.CartesianBuilder.cartesianProductOf
 import org.openjdk.jmh.annotations.*
 import org.openjdk.jmh.infra.Blackhole
+import java.util.concurrent.TimeUnit
 
 @State(Scope.Benchmark)
+@Fork(2)
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
 open class CartesianMapBenchmark {
 
-    @Param("7")
+    @Param("5", "7")
     var itemsQuantity: Int = 0
 
     lateinit var mapOf: Map<Int, List<Int>>

kombi-jmh/src/jmh/kotlin/com/sgnatiuk/benchmark/combination/CombinationBenchmark.kt

@@ -3,9 +3,14 @@ package com.sgnatiuk.benchmark.combination
 import com.sgnatiuk.combination.CombinationsBuilder.combinationsOf
 import org.openjdk.jmh.annotations.*
 import org.openjdk.jmh.infra.Blackhole
-
+import java.util.concurrent.TimeUnit
 
 @State(Scope.Benchmark)
+@Fork(2)
+@Warmup(iterations = 5)
+@Measurement(iterations = 5)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
 open class CombinationBenchmark {
 
     @Param("11", "19")
@@ -18,6 +23,10 @@ open class CombinationBenchmark {
     fun doSetup() {
         list = List(itemsQuantity){ it }
         map = (1..itemsQuantity).map { it to it.toString() }.toMap()
+        println("\n=================================================")
+        println("itemsQuantity=$itemsQuantity")
+        println("combinationsQuantity=${combinationsOf(list).combinationsNumber()}")
+        println("=================================================\n")
     }
 
     @Benchmark

kombi-lib/build.gradle

@@ -13,7 +13,7 @@ plugins {
 }
 
 ext{
-    libVersion = '3.0.0'
+    libVersion = '3.0.1'
     libPackage = 'com.sgnatiuk'
     libName = 'kombi'
 }