diff --git a/build.sh b/build.sh index 514a622e..441d8161 100755 --- a/build.sh +++ b/build.sh @@ -7,7 +7,8 @@ docker pull clue/httpie # base template docker build -t ubuntu_gcc -f containers/templates/Dockerfile-ubuntu-gcc containers/templates/ docker build -t ubuntu_python -f containers/templates/Dockerfile-ubuntu-python containers/templates/ -docker build -t ubuntu_vbr -f containers/templates/Dockerfile-ubuntu-vbr containers/templates/ +#docker build -t ubuntu_vbr -f containers/templates/Dockerfile-ubuntu-vbr containers/templates/ +#docker build -t ubuntu_cbas -f containers/templates/Dockerfile-ubuntu-cbas containers/templates/ # framework containers docker build -t sequoiatools/perfrunner containers/perfrunner @@ -17,7 +18,9 @@ docker build -t sequoiatools/testrunner containers/testrunner docker build -t sequoiatools/tpcc containers/tpcc docker build -t sequoiatools/pillowfight containers/pillowfight docker build -t sequoiatools/gideon containers/gideon # depends on pillowfight -docker build -t sequoiatools/vbr containers/vbr +docker build -t sequoiatools/curloop containers/curloop +#docker build -t sequoiatools/vbr containers/vbr +#docker build -t sequoiatools/cbas containers/cbas #docker build -t ycsb containers/ycsb #docker build -t mysql containers/mysql #docker build -t elasticsearch containers/elasticsearch diff --git a/containers/cbas/Dockerfile b/containers/cbas/Dockerfile new file mode 100644 index 00000000..00aa0453 --- /dev/null +++ b/containers/cbas/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu_cbas +RUN ls -la +WORKDIR /root/cbas/cbas +RUN sed -i '$ d' ./samples/local/bin/start-sample-cluster.sh +RUN echo "ifconfig" >> ./samples/local/bin/start-sample-cluster.sh +RUN echo "sleep 100000" >> ./samples/local/bin/start-sample-cluster.sh +ENV JAVA_HOME /usr/lib/jvm/java-8-oracle +ENTRYPOINT ["./samples/local/bin/start-sample-cluster.sh"] \ No newline at end of file diff --git a/containers/curloop/Dockerfile b/containers/curloop/Dockerfile index 352358c8..9d9df8b1 100644 --- a/containers/curloop/Dockerfile +++ b/containers/curloop/Dockerfile @@ -1,3 +1,5 @@ FROM appropriate/curl COPY n1ql.sh /bin/n1ql.sh -COPY view.sh /bin/view.sh +COPY view.sh /bin/view.sh +COPY cbas.sh /bin/cbas.sh +COPY cbasloop.sh /bin/cbasloop.sh diff --git a/containers/curloop/cbas.sh b/containers/curloop/cbas.sh new file mode 100755 index 00000000..109c3665 --- /dev/null +++ b/containers/curloop/cbas.sh @@ -0,0 +1,9 @@ +#!/bin/sh +#ID=`docker ps -a | grep "Up" | grep "sequoiatools/cbas" | awk '{print $1}'` +#HOST=`docker inspect --format '{{ .NetworkSettings.IPAddress }}' $ID` +HOST="172.17.0.5" # we need run 'docker rm -fv $(docker ps -qa)' before test +STATEMENT=$@ +STATEMENT=${STATEMENT//\\/""} # remove \ slashes +STATEMENT=${STATEMENT//\'NUM/""} # to use nums in expression +echo "curl -s --data pretty=true --data-urlencode \"statement=$STATEMENT\" http://$HOST:8095/analytics/service -v" +curl -s --data pretty=true --data format=CLEAN_JSON --data-urlencode "statement=$STATEMENT" http://$HOST:8095/analytics/service #> log \ No newline at end of file diff --git a/containers/curloop/cbasloop.sh b/containers/curloop/cbasloop.sh new file mode 100755 index 00000000..75cfb44c --- /dev/null +++ b/containers/curloop/cbasloop.sh @@ -0,0 +1,10 @@ +#!/bin/sh +#ID=`docker ps -a | grep "Up" | grep "sequoiatools/cbas" | awk '{print $1}'` +#HOST=`docker inspect --format '{{ .NetworkSettings.IPAddress }}' $ID` +HOST="172.17.0.5" +STATEMENT=$@ +STATEMENT=${STATEMENT//\\/""} # remove \ slashes +STATEMENT=${STATEMENT//\'NUM/""} # to use nums in expression +while [ 1 ] ; do + curl -s --data pretty=true --data format=CLEAN_JSON --data-urlencode "statement=$STATEMENT" http://$HOST:8095/analytics/service > log +done diff --git a/containers/templates/Dockerfile-ubuntu-cbas b/containers/templates/Dockerfile-ubuntu-cbas new file mode 100644 index 00000000..0667dc58 --- /dev/null +++ b/containers/templates/Dockerfile-ubuntu-cbas @@ -0,0 +1,29 @@ +FROM ubuntu:14.04 +RUN apt-get update +RUN apt-get install -y python-software-properties software-properties-common +RUN add-apt-repository ppa:webupd8team/java +RUN apt-get update +RUN apt-get install -y git curl wget zip unzip +RUN echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | sudo debconf-set-selections +RUN apt-get install -y oracle-java8-installer +WORKDIR /tmp +RUN wget http://mirror.fibergrid.in/apache/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz +RUN tar -zxvf apache-maven-3.3.9-bin.tar.gz +RUN mv apache-maven-3.3.9 /usr/local/. +RUN sudo ln -s /usr/local/apache-maven-3.3.9/bin/mvn /usr/bin/mvn +RUN mvn --version +RUN git config --global user.email "andrei.baranouski@gmail.com" +RUN git config --global user.name "andreibaranouski" +RUN git config --global color.ui false +RUN mkdir -p /root/.ssh/ +RUN ssh-keyscan github.com >> ~/.ssh/known_hosts +RUN curl https://storage.googleapis.com/git-repo-downloads/repo > /usr/bin/repo +RUN chmod a+x /usr/bin/repo +# please replace 'id_rsa' file with private key when you pull private projects from github +ADD id_rsa /root/.ssh/id_rsa +RUN chmod 400 /root/.ssh/id_rsa +RUN mkdir -p /root/cbas +RUN cd /root/cbas && repo init -u git://github.com/couchbase/manifest -m analytics/master.xml && repo sync --jobs=6 +RUN cd /root/cbas/asterixdb && mvn clean install -DskipTests +RUN cd /root/cbas && unzip ./asterixdb/asterixdb/asterix-opt/installer/target/couchbase-analytics-1.0.0-SNAPSHOT-generic.zip + diff --git a/containers/templates/id_rsa b/containers/templates/id_rsa new file mode 100644 index 00000000..389b5351 --- /dev/null +++ b/containers/templates/id_rsa @@ -0,0 +1,2 @@ + !!!!used for Dockerfile-ubuntu-cbas!!!! +!!!!please this file with private key when you pull private projects from github!!!! \ No newline at end of file diff --git a/tests/cbas/scope_baseQueries.yml b/tests/cbas/scope_baseQueries.yml new file mode 100644 index 00000000..d7d97ec8 --- /dev/null +++ b/tests/cbas/scope_baseQueries.yml @@ -0,0 +1,19 @@ +--- +buckets: + - + name: default + ram: 80% + eviction: fullEviction + + +servers: + - + name: local + count: 8 + ram: 80% + rest_username: Administrator + rest_password: password + #data_path: "/data" + #index_path: "/data" + init_nodes: 6 # ONLY INIT 6 NODES + buckets: default diff --git a/tests/cbas/test_cbas.yml b/tests/cbas/test_cbas.yml new file mode 100644 index 00000000..3e097960 --- /dev/null +++ b/tests/cbas/test_cbas.yml @@ -0,0 +1,151 @@ +--- +- + include: tests/templates/rebalance.yml, tests/templates/fts.yml + +# start json and binary data loading +- + image: sequoiatools/pillowfight + command: "-U {{.Orchestrator}} -B {{.Scale 100}} -I {{.Scale 1000000}} -t 1" +- + image: sequoiatools/gideon + command: "kv --ops {{.Scale 1000}} --create 10 --get 90 --expire 20 --ttl 600 --hosts {{.Orchestrator}} --bucket {{.Bucket}} --sizes 1600" +- + command: "kv --ops {{.Scale 1000}} --create 30 --get 70 --delete 10 --hosts {{.Orchestrator}} --bucket {{.Bucket}}" +- + image: sequoiatools/cbas + command: "> LOGS" + wait: false +- + image: sequoiatools/cmd + entrypoint: sleep + command: "{{.Scale 10}}" + wait: true + +# connect bucket with CBAS +- + image: sequoiatools/curloop + command: "cbas.sh create bucket defaultBucket with { \''bucket\': \''default\', \''nodes\': \''172.23.108.8\' };" +- + command: "cbas.sh create shadow dataset shadowBucket on defaultBucket;" +- + command: "cbas.sh connect bucket defaultBucket with { \''password': \''' };" +- + image: sequoiatools/cmd + entrypoint: sleep + command: "{{.Scale 10}}" + wait: true + +#debug: run once to check +- + image: sequoiatools/curloop + command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''25\' and \\`result\\` = \''SUCCESS\' group by \\`city\\` ;" +- + command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''29\' and \\`component\\` = \''0A\' ;" +- + command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''79\' and \\`component\\` != \''D8\' ;" +- + command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''B1\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` > \''NUM7000\'NUM and \\`totalCount\\`> \''NUM80\'NUM group by \\`component\\` ;" +- + command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''7B\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` < \''NUM7000\'NUM and \\`totalCount\\`< \''NUM80\'NUM group by \\`component\\` ;" + +- + image: sequoiatools/cmd + entrypoint: sleep + command: "{{.Scale 20}}" + wait: true + +# query on cbas side. # TODO add more queries +- + image: sequoiatools/curloop + command: "cbasloop.sh select value t from shadowBucket t where \\`state\\` = \''25\' and \\`result\\` = \''SUCCESS\' group by \\`city\\` ;" +- + command: "cbasloop.sh select count(*) from shadowBucket where \\`state\\` = \''29\' and \\`component\\` = \''0A\' ;" +- + command: "cbasloop.sh select count(*) from shadowBucket where \\`state\\` = \''79\' and \\`component\\` != \''D8\' ;" +- + command: "cbasloop.sh select value t from shadowBucket t where \\`state\\` = \''B1\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` > \''NUM7000\'NUM and \\`totalCount\\`> \''NUM80\'NUM group by \\`component\\` ;" +- + command: "cbasloop.sh select value t from shadowBucket t where \\`state\\` = \''7B\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` < \''NUM7000\'NUM and \\`totalCount\\`< \''NUM80\'NUM group by \\`component\\` ;" + +- + image: sequoiatools/cmd + entrypoint: sleep + command: "{{.Scale 20}}" + wait: true + +# swap rebalance +- + template: add_node + args: "{{.NthDataNode 6}}, data" +- + template: rebalance_out + args: "{{.NthDataNode 2}}" +- + template: wait_for_rebalance + wait: true + +# scale back down +- + template: rebalance_out + args: "{{.NthDataNode 3}}" +- + template: wait_for_rebalance + wait: true + +# failover a node +- + template: failover_node_forced + args: "{{.NthDataNode 4}}" + wait: true +- + template: rebalance + wait: true +- + template: wait_for_rebalance + wait: true + +# add one node and then stop rebalance +- + template: add_node + args: "{{.NthDataNode 7}}, data" +- + template: rebalance + wait: false +- + image: sequoiatools/cmd + entrypoint: sleep + command: "{{.Scale 100}}" + wait: true +- + template: rebalance_stop + +#debug: run once again to check +- + image: sequoiatools/curloop + command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''25\' and \\`result\\` = \''SUCCESS\' group by \\`city\\` ;" +- + command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''29\' and \\`component\\` = \''0A\' ;" +- + command: "cbas.sh select count(*) from shadowBucket where \\`state\\` = \''79\' and \\`component\\` != \''D8\' ;" +- + command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''B1\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` > \''NUM7000\'NUM and \\`totalCount\\`> \''NUM80\'NUM group by \\`component\\` ;" +- + command: "cbas.sh select value t from shadowBucket t where \\`state\\` = \''7B\' and \\`rating\\`> \''NUM100\'NUM and \\`duration\\` < \''NUM7000\'NUM and \\`totalCount\\`< \''NUM80\'NUM group by \\`component\\` ;" + + +# failover node with delta recovery +- + template: failover_and_recover + args: "{{.NthDataNode 6}}, delta" + wait: true +- + template: wait_for_rebalance + wait: true + +# rebalance out one node +- + template: rebalance_out + args: "{{.NthDataNode 4}}" +- + template: wait_for_rebalance + wait: true \ No newline at end of file diff --git a/tests/vbr/test_vbr.yml b/tests/vbr/test_vbr.yml index 80867bc0..d748aa6d 100644 --- a/tests/vbr/test_vbr.yml +++ b/tests/vbr/test_vbr.yml @@ -67,12 +67,12 @@ - template: rebalance_by_ip args: "{{.Nodes | .Cluster 1 | net 0}}" -- - template: wait_for_rebalance - wait: true - template: wait_for_rebalance_by_ip args: "{{.Nodes | .Cluster 1 | net 0}}" +- + template: wait_for_rebalance + wait: true #============ 4 failover with delta recovery at cluster A============ - @@ -123,12 +123,12 @@ - template: rebalance_by_ip args: "{{.Nodes | .Cluster 1 | net 0}}" -- - template: wait_for_rebalance - wait: true - template: wait_for_rebalance_by_ip args: "{{.Nodes | .Cluster 1 | net 0}}" +- + template: wait_for_rebalance + wait: true #============ 8 rebalance in 2 nodes on cluster A ============ @@ -150,4 +150,4 @@ template: rebalance - template: wait_for_rebalance - wait: true + wait: true \ No newline at end of file