From cfb2ef8b05466be3a0e747ce783be65cf0d1d9dc Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 13:42:55 +0100 Subject: [PATCH 1/7] Updating to work with luamongo 0.5.0 --- .travis.yml | 12 ++++++------ mapreduce/cnn.lua | 9 ++++++++- mapreduce/examples/APRIL-ANN/common.lua | 4 ++-- mapreduce/server.lua | 4 +++- mapreduce/utils.lua | 2 +- test.sh | 8 ++++---- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/.travis.yml b/.travis.yml index 900dae0..e876d86 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,12 @@ env: LUA_PATH="./?/init.lua;./?.lua" before_install: - sudo apt-get update -qq - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libssl-dev - - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-0.9.0.tar.gz - - tar zxvf legacy-0.9.0.tar.gz - - cd mongo-cxx-driver-legacy-0.9.0 && sudo scons --prefix=/usr install-mongoclient && cd - - - wget https://github.com/moai/luamongo/archive/v0.4.3.tar.gz - - tar zxvf v0.4.3.tar.gz - - cd luamongo-0.4.3 && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - + - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-1.1.0.tar.gz + - tar zxvf legacy-1.1.0.tar.gz + - cd mongo-cxx-driver-legacy-1.1.0 && sudo scons --sharedclient=yes --prefix=/usr install && cd - + - wget https://github.com/moai/luamongo/archive/v0.5.0.tar.gz + - tar zxvf v0.5.0.tar.gz + - cd luamongo-0.5.0 && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - - ssh-keygen -b 2048 -f /home/travis/.ssh/id_rsa -t rsa -q -N "" - ssh-keyscan -t rsa localhost > ~/.ssh/known_hosts - ssh-keyscan -t rsa $(hostname) >> ~/.ssh/known_hosts diff --git a/mapreduce/cnn.lua b/mapreduce/cnn.lua index cb2b1de..2ddec9c 100644 --- a/mapreduce/cnn.lua +++ b/mapreduce/cnn.lua @@ -44,8 +44,15 @@ function cnn:gridfs() return gridfs end +-- this weak table allow to index gridfs objects using weak grid_file_builder +-- keys +local gridfs_instances = setmetatable({}, { __mode="k" }) + function cnn:grid_file_builder() - return mongo.GridFileBuilder.New(self:connect(), self.gridfs_dbname) + local gridfs = self:gridfs() + local grid_file_builder = mongo.GridFileBuilder.New(gridfs) + gridfs_instances[grid_file_builder] = gridfs + return grid_file_builder end function cnn:get_dbname() diff --git a/mapreduce/examples/APRIL-ANN/common.lua b/mapreduce/examples/APRIL-ANN/common.lua index fa3cc67..d583ba9 100644 --- a/mapreduce/examples/APRIL-ANN/common.lua +++ b/mapreduce/examples/APRIL-ANN/common.lua @@ -23,13 +23,13 @@ util.omp_set_num_threads(1) local serialize_to_gridfs = function(gridfs, filename, obj) gridfs:remove_file(filename) - local builder = mongo.GridFileBuilder.New(db, dbname) + local builder = mongo.GridFileBuilder.New(gridfs) builder:append(util.serialize(obj)) builder:build(filename) end local deserialize_from_gridfs = function(gridfs, filename) - local file = assert( gridfs:find_file(filename) ) + local file = assert( gridfs:find_file_by_name(filename) ) local str_tbl = {} for i=1,file:num_chunks() do local chunk = file:chunk(i-1) diff --git a/mapreduce/server.lua b/mapreduce/server.lua index 41b0cf6..df4274b 100644 --- a/mapreduce/server.lua +++ b/mapreduce/server.lua @@ -333,7 +333,9 @@ local function server_drop_collections(self) local dbname = self.cnn:get_dbname() -- drop all the collections for _,name in ipairs(db:get_collections(dbname)) do - db:drop_collection(name) + if name ~= "system_indexes" then + db:drop_collection(dbname .. "." .. name) + end end local gridfs = self.cnn:gridfs() local list = gridfs:list() diff --git a/mapreduce/utils.lua b/mapreduce/utils.lua index 386da63..3b59db2 100644 --- a/mapreduce/utils.lua +++ b/mapreduce/utils.lua @@ -131,7 +131,7 @@ end -- and last chunks where the line is contained, and the first and last position -- inside the corresponding chunks local function gridfs_lines_iterator(gridfs, filename) - local gridfile = gridfs:find_file(filename) + local gridfile = assert( gridfs:find_file_by_name(filename) ) local size = #gridfile local current_chunk = 0 local current_pos = 1 diff --git a/test.sh b/test.sh index 80b9364..c7863b9 100755 --- a/test.sh +++ b/test.sh @@ -6,7 +6,7 @@ if [[ $? -ne 0 ]]; then exit 1 fi for storage in gridfs shared sshfs; do - ## COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER + echo COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER screen -d -m ./execute_example_worker.sh diff <(./execute_example_server.sh $storage | awk '{ print $1,$2 }' | sort) \ <(cat mapreduce/server.lua \ @@ -17,7 +17,7 @@ for storage in gridfs shared sshfs; do echo "ERROR" exit 1 fi - ## NO COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER + echo NO COMBINER + ASSOCIATIVE COMMUTATIVE IDEMPOTENT REDUCER screen -d -m ./execute_example_worker.sh diff <(lua execute_server.lua localhost wordcount \ mapreduce.examples.WordCount.taskfn \ @@ -34,7 +34,7 @@ for storage in gridfs shared sshfs; do echo "ERROR" exit 1 fi - ## NO COMBINER + GENERAL REDUCER + echo NO COMBINER + GENERAL REDUCER screen -d -m ./execute_example_worker.sh diff <(lua execute_server.lua localhost wordcount \ mapreduce.examples.WordCount.taskfn \ @@ -51,7 +51,7 @@ for storage in gridfs shared sshfs; do echo "ERROR" exit 1 fi - ## INIT SCRIPT + echo INIT SCRIPT screen -d -m ./execute_example_worker.sh diff <(lua execute_server.lua localhost wordcount \ mapreduce.examples.WordCount \ From aae139a6a89696966b1daf0b78bc1c6dcc7b9db5 Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 13:49:32 +0100 Subject: [PATCH 2/7] Fixed problem with libraries installation in Travis CI --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e876d86..94376e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ services: env: LUA_PATH="./?/init.lua;./?.lua" before_install: - sudo apt-get update -qq - - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libssl-dev + - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libssl-dev - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-1.1.0.tar.gz - tar zxvf legacy-1.1.0.tar.gz - cd mongo-cxx-driver-legacy-1.1.0 && sudo scons --sharedclient=yes --prefix=/usr install && cd - From 0fa772376ea1d1f60603ac42de52f66edbdb90ac Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 16:51:50 +0100 Subject: [PATCH 3/7] Updated version check --- mapreduce/utils.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapreduce/utils.lua b/mapreduce/utils.lua index 3b59db2..08f2f3f 100644 --- a/mapreduce/utils.lua +++ b/mapreduce/utils.lua @@ -19,7 +19,7 @@ local mongo = require "mongo" local heap = require "mapreduce.heap" -assert(mongo._VERSION == "0.4" or tonumber(mongo._VERSION > 0.4)) +assert(mongo._VERSION == "0.5" or tonumber(mongo._VERSION) > 0.5) local utils = { _VERSION = "0.3", From 7421b954a51fe2d9e6606f9489fad8a8f5c60d47 Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 17:06:27 +0100 Subject: [PATCH 4/7] Added luamongo as dependency --- .gitmodules | 3 +++ .travis.yml | 9 +++------ external/luamongo | 1 + 3 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 .gitmodules create mode 160000 external/luamongo diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..b3f971d --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "external/luamongo"] + path = external/luamongo + url = https://github.com/moai/luamongo.git diff --git a/.travis.yml b/.travis.yml index 94376e8..a080df4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,12 +4,9 @@ env: LUA_PATH="./?/init.lua;./?.lua" before_install: - sudo apt-get update -qq - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libssl-dev - - wget https://github.com/mongodb/mongo-cxx-driver/archive/legacy-1.1.0.tar.gz - - tar zxvf legacy-1.1.0.tar.gz - - cd mongo-cxx-driver-legacy-1.1.0 && sudo scons --sharedclient=yes --prefix=/usr install && cd - - - wget https://github.com/moai/luamongo/archive/v0.5.0.tar.gz - - tar zxvf v0.5.0.tar.gz - - cd luamongo-0.5.0 && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - + - git submodule update --init --recursive + - cd external/luamongo/external/mongo-cxx-driver/ && sudo scons --sharedclient=yes --prefix=/usr install && cd - + - cd external/luamongo/ && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - - ssh-keygen -b 2048 -f /home/travis/.ssh/id_rsa -t rsa -q -N "" - ssh-keyscan -t rsa localhost > ~/.ssh/known_hosts - ssh-keyscan -t rsa $(hostname) >> ~/.ssh/known_hosts diff --git a/external/luamongo b/external/luamongo new file mode 160000 index 0000000..0e4e7ab --- /dev/null +++ b/external/luamongo @@ -0,0 +1 @@ +Subproject commit 0e4e7ab9c7098f11b35644f102ee9528a3fb5e87 From 308f78d4898c334fd83d672cda8664c64202e284 Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 17:12:24 +0100 Subject: [PATCH 5/7] Added 15 seconds sleep for Travis CI --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index a080df4..57abb96 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,5 +11,7 @@ before_install: - ssh-keyscan -t rsa localhost > ~/.ssh/known_hosts - ssh-keyscan -t rsa $(hostname) >> ~/.ssh/known_hosts - cp ~/.ssh/id_rsa.pub ~/.ssh/authorized_keys +before_script: + - sleep 15 language: lua script: ./test.sh From b45070c876c9277686d55a72e367af988169973b Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 17:15:15 +0100 Subject: [PATCH 6/7] Forcing mongodb installation --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 57abb96..1f4f3b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ services: env: LUA_PATH="./?/init.lua;./?.lua" before_install: - sudo apt-get update -qq - - sudo apt-get install -y screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libssl-dev + - sudo apt-get install -y mongodb mongodb-server screen lua5.2 liblua5.2-dev libboost-filesystem-dev libboost-filesystem-dev libboost-thread-dev libboost-regex-dev libssl-dev - git submodule update --init --recursive - cd external/luamongo/external/mongo-cxx-driver/ && sudo scons --sharedclient=yes --prefix=/usr install && cd - - cd external/luamongo/ && make && sudo mkdir -p /usr/lib/lua/5.2 && sudo cp mongo.so /usr/lib/lua/5.2 && cd - From d26308dfac42806bacf900b0664423160ed91797 Mon Sep 17 00:00:00 2001 From: Paco Zamora Martinez Date: Tue, 22 Dec 2015 17:23:14 +0100 Subject: [PATCH 7/7] Version 0.4.0 --- mapreduce/init.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapreduce/init.lua b/mapreduce/init.lua index de67ca7..fece2f3 100644 --- a/mapreduce/init.lua +++ b/mapreduce/init.lua @@ -23,7 +23,7 @@ local tuple = require "mapreduce.tuple" local persistent_table = require "mapreduce.persistent_table" local mapreduce = { - _VERSION = "0.3.7", + _VERSION = "0.4.0", _NAME = "mapreduce", worker = worker, server = server,