Cbas systests

build.sh

@@ -7,7 +7,8 @@ docker pull clue/httpie
 # base template
 docker build -t ubuntu_gcc -f containers/templates/Dockerfile-ubuntu-gcc  containers/templates/
 docker build -t ubuntu_python -f containers/templates/Dockerfile-ubuntu-python  containers/templates/
-docker build -t ubuntu_vbr -f containers/templates/Dockerfile-ubuntu-vbr  containers/templates/
+#docker build -t ubuntu_vbr -f containers/templates/Dockerfile-ubuntu-vbr  containers/templates/
+#docker build -t ubuntu_cbas -f containers/templates/Dockerfile-ubuntu-cbas  containers/templates/
 
 # framework containers
 docker build -t sequoiatools/perfrunner containers/perfrunner
@@ -17,7 +18,9 @@ docker build -t sequoiatools/testrunner containers/testrunner
 docker build -t sequoiatools/tpcc containers/tpcc
 docker build -t sequoiatools/pillowfight containers/pillowfight
 docker build -t sequoiatools/gideon containers/gideon  # depends on pillowfight
-docker build -t sequoiatools/vbr containers/vbr
+docker build -t sequoiatools/curloop containers/curloop
+#docker build -t sequoiatools/vbr containers/vbr
+#docker build -t sequoiatools/cbas containers/cbas
 #docker build -t ycsb containers/ycsb
 #docker build -t mysql containers/mysql
 #docker build -t elasticsearch containers/elasticsearch

containers/cbas/Dockerfile

@@ -0,0 +1,8 @@
+FROM ubuntu_cbas
+RUN ls -la
+WORKDIR /root/cbas/cbas
+RUN sed -i '$ d' ./samples/local/bin/start-sample-cluster.sh
+RUN echo "ifconfig" >> ./samples/local/bin/start-sample-cluster.sh
+RUN echo "sleep 100000" >> ./samples/local/bin/start-sample-cluster.sh
+ENV JAVA_HOME /usr/lib/jvm/java-8-oracle
+ENTRYPOINT ["./samples/local/bin/start-sample-cluster.sh"]

containers/curloop/Dockerfile

@@ -1,3 +1,5 @@
 FROM appropriate/curl
 COPY n1ql.sh /bin/n1ql.sh 
-COPY view.sh /bin/view.sh 
+COPY view.sh /bin/view.sh
+COPY cbas.sh /bin/cbas.sh
+COPY cbasloop.sh /bin/cbasloop.sh

containers/curloop/cbas.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+#ID=`docker ps -a | grep "Up" | grep "sequoiatools/cbas" | awk '{print $1}'`
+#HOST=`docker inspect --format '{{ .NetworkSettings.IPAddress }}' $ID`
+HOST="172.17.0.5"  # we need run 'docker rm -fv $(docker ps -qa)' before test
+STATEMENT=$@
+STATEMENT=${STATEMENT//\\/""}  # remove \ slashes
+STATEMENT=${STATEMENT//\'NUM/""}  # to use nums in expression
+echo "curl -s --data pretty=true --data-urlencode \"statement=$STATEMENT\" http://$HOST:8095/analytics/service -v"
+curl -s --data pretty=true --data format=CLEAN_JSON --data-urlencode "statement=$STATEMENT" http://$HOST:8095/analytics/service #> log

containers/curloop/cbasloop.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+#ID=`docker ps -a | grep "Up" | grep "sequoiatools/cbas" | awk '{print $1}'`
+#HOST=`docker inspect --format '{{ .NetworkSettings.IPAddress }}' $ID`
+HOST="172.17.0.5"
+STATEMENT=$@
+STATEMENT=${STATEMENT//\\/""} # remove \ slashes
+STATEMENT=${STATEMENT//\'NUM/""}  # to use nums in expression
+while [ 1 ] ; do
+    curl -s --data pretty=true --data format=CLEAN_JSON --data-urlencode "statement=$STATEMENT" http://$HOST:8095/analytics/service > log
+done

containers/templates/Dockerfile-ubuntu-cbas

@@ -0,0 +1,29 @@
+FROM ubuntu:14.04
+RUN apt-get update
+RUN apt-get install -y python-software-properties software-properties-common
+RUN add-apt-repository ppa:webupd8team/java
+RUN apt-get update
+RUN apt-get install -y git curl wget zip unzip
+RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | sudo debconf-set-selections
+RUN apt-get install -y oracle-java8-installer
+WORKDIR /tmp
+RUN wget http://mirror.fibergrid.in/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+RUN tar -zxvf apache-maven-3.3.9-bin.tar.gz
+RUN mv apache-maven-3.3.9 /usr/local/.
+RUN sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/bin/mvn
+RUN mvn --version
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "andreibaranouski"
+RUN git config --global color.ui false
+RUN mkdir -p /root/.ssh/
+RUN ssh-keyscan github.com  >> ~/.ssh/known_hosts
+RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/bin/repo
+RUN chmod a+x /usr/bin/repo
+# please replace 'id_rsa' file with private key when you pull private projects from github
+ADD id_rsa /root/.ssh/id_rsa
+RUN chmod 400 /root/.ssh/id_rsa
+RUN mkdir -p /root/cbas
+RUN cd /root/cbas && repo init -u git://github.com/couchbase/manifest -m analytics/master.xml && repo sync --jobs=6
+RUN cd /root/cbas/asterixdb && mvn clean install -DskipTests
+RUN cd /root/cbas && unzip ./asterixdb/asterixdb/asterix-opt/installer/target/couchbase-analytics-1.0.0-SNAPSHOT-generic.zip
+

containers/templates/id_rsa

@@ -0,0 +1,2 @@
+                     !!!!used for Dockerfile-ubuntu-cbas!!!!
+!!!!please this file with private key when you pull private projects from github!!!!

tests/cbas/scope_baseQueries.yml

@@ -0,0 +1,19 @@
+---
+buckets:
+  -
+      name: default
+      ram: 80%
+      eviction: fullEviction
+
+
+servers:
+  -
+      name: local
+      count: 8
+      ram: 80%
+      rest_username: Administrator
+      rest_password: password
+      #data_path: "/data"
+      #index_path: "/data"
+      init_nodes: 6   # ONLY INIT 6 NODES
+      buckets: default

tests/cbas/test_cbas.yml

@@ -0,0 +1,151 @@
+---
+-
+  include: tests/templates/rebalance.yml, tests/templates/fts.yml
+
+# start json and binary data loading
+-
+   image: sequoiatools/pillowfight
+   command: "-U  {{.Orchestrator}} -B {{.Scale 100}} -I {{.Scale 1000000}} -t 1"
+-
+   image: sequoiatools/gideon
+   command: "kv --ops {{.Scale 1000}} --create 10 --get 90 --expire 20 --ttl 600  --hosts {{.Orchestrator}} --bucket {{.Bucket}} --sizes 1600"
+-
+   command: "kv --ops {{.Scale 1000}} --create 30 --get 70 --delete 10   --hosts {{.Orchestrator}} --bucket {{.Bucket}}"
+-
+   image: sequoiatools/cbas
+   command: "> LOGS"
+   wait: false
+-
+   image: sequoiatools/cmd
+   entrypoint: sleep
+   command: "{{.Scale 10}}"
+   wait: true
+
+# connect bucket with CBAS
+-
+  image: sequoiatools/curloop
+  command: "cbas.sh create bucket defaultBucket with { \''bucket\': \''default\', \''nodes\': \''172.23.108.8\' };"
+-
+  command: "cbas.sh create shadow dataset shadowBucket on defaultBucket;"
+-
+  command: "cbas.sh connect bucket defaultBucket with { \''password': \''' };"
+-
+   image: sequoiatools/cmd
+   entrypoint: sleep
+   command: "{{.Scale 10}}"
+   wait: true
+
+#debug: run once to check
+-
+  image: sequoiatools/curloop
+  command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''25\' and \\`result\\` = \''SUCCESS\' group by \\`city\\` ;"
+-
+  command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''29\' and \\`component\\` = \''0A\' ;"
+-
+  command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''79\' and \\`component\\` != \''D8\' ;"
+-
+  command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''B1\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` > \''NUM7000\'NUM and \\`totalCount\\`> \''NUM80\'NUM group by \\`component\\` ;"
+-
+  command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''7B\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` < \''NUM7000\'NUM and \\`totalCount\\`< \''NUM80\'NUM group by \\`component\\` ;"
+
+-
+   image: sequoiatools/cmd
+   entrypoint: sleep
+   command: "{{.Scale 20}}"
+   wait: true
+
+# query on cbas side. # TODO add more queries
+-
+  image: sequoiatools/curloop
+  command: "cbasloop.sh select value t from shadowBucket t where \\`state\\` = \''25\' and \\`result\\` = \''SUCCESS\' group by \\`city\\` ;"
+-
+  command: "cbasloop.sh select count(*) from shadowBucket where \\`state\\` = \''29\' and \\`component\\` = \''0A\' ;"
+-
+  command: "cbasloop.sh select count(*) from shadowBucket where \\`state\\` = \''79\' and \\`component\\` != \''D8\' ;"
+-
+  command: "cbasloop.sh select value t from shadowBucket t where \\`state\\` = \''B1\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` > \''NUM7000\'NUM and \\`totalCount\\`> \''NUM80\'NUM group by \\`component\\` ;"
+-
+  command: "cbasloop.sh select value t from shadowBucket t where \\`state\\` = \''7B\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` < \''NUM7000\'NUM and \\`totalCount\\`< \''NUM80\'NUM group by \\`component\\` ;"
+
+-
+   image: sequoiatools/cmd
+   entrypoint: sleep
+   command: "{{.Scale 20}}"
+   wait: true
+
+# swap rebalance
+-
+  template: add_node
+  args: "{{.NthDataNode 6}}, data"
+-
+  template: rebalance_out
+  args: "{{.NthDataNode 2}}"
+-
+  template: wait_for_rebalance
+  wait: true
+
+# scale back down
+-
+  template: rebalance_out
+  args: "{{.NthDataNode 3}}"
+-
+  template: wait_for_rebalance
+  wait: true
+
+# failover a node
+-
+  template: failover_node_forced
+  args: "{{.NthDataNode 4}}"
+  wait: true
+-
+  template: rebalance
+  wait: true
+-
+  template: wait_for_rebalance
+  wait: true
+
+# add one node and then stop rebalance
+-
+  template: add_node
+  args: "{{.NthDataNode 7}}, data"
+-
+  template: rebalance
+  wait: false
+-
+   image: sequoiatools/cmd
+   entrypoint: sleep
+   command: "{{.Scale 100}}"
+   wait: true
+-
+  template: rebalance_stop
+
+#debug: run once again to check
+-
+  image: sequoiatools/curloop
+  command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''25\' and \\`result\\` = \''SUCCESS\' group by \\`city\\` ;"
+-
+  command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''29\' and \\`component\\` = \''0A\' ;"
+-
+  command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''79\' and \\`component\\` != \''D8\' ;"
+-
+  command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''B1\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` > \''NUM7000\'NUM and \\`totalCount\\`> \''NUM80\'NUM group by \\`component\\` ;"
+-
+  command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''7B\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` < \''NUM7000\'NUM and \\`totalCount\\`< \''NUM80\'NUM group by \\`component\\` ;"
+
+
+# failover node with delta recovery
+-
+  template: failover_and_recover
+  args: "{{.NthDataNode 6}}, delta"
+  wait: true
+-
+  template: wait_for_rebalance
+  wait: true
+
+# rebalance out one node
+-
+  template: rebalance_out
+  args: "{{.NthDataNode 4}}"
+-
+  template: wait_for_rebalance
+  wait: true

tests/vbr/test_vbr.yml

@@ -67,12 +67,12 @@
 -
   template: rebalance_by_ip
   args: "{{.Nodes | .Cluster 1 | net 0}}"
--
-  template: wait_for_rebalance
-  wait: true
 -
   template: wait_for_rebalance_by_ip
   args: "{{.Nodes | .Cluster 1 | net 0}}"
+-
+  template: wait_for_rebalance
+  wait: true
 
 #============ 4 failover with delta recovery at cluster A============
 -
@@ -123,12 +123,12 @@
 -
   template: rebalance_by_ip
   args: "{{.Nodes | .Cluster 1 | net 0}}"
--
-  template: wait_for_rebalance
-  wait: true
 -
   template: wait_for_rebalance_by_ip
   args: "{{.Nodes | .Cluster 1 | net 0}}"
+-
+  template: wait_for_rebalance
+  wait: true
 
 
 #============ 8 rebalance in 2 nodes on cluster A ============
@@ -150,4 +150,4 @@
   template: rebalance
 -
   template: wait_for_rebalance
-  wait: true
+  wait: true