parially download dataset

.gitignore

@@ -7,3 +7,7 @@ vk_interaction_obj_rec/.env
 vk_interaction_obj_rec/codebase/build/
 vk_interaction_obj_rec/codebase/node_modules/
 vk_interaction_obj_rec/codebase/package-lock.json
+
+## environments
+docker-compose/
+env4spb/

docker-compose.yml

@@ -1,59 +1,57 @@
 version: '2.2'
 services:
-  api-python:
-    build:
-      context: ./ranking_system
-      dockerfile: Dockerfile
-    command: python3 /codebase/api_server.py
-    privileged: true
-    depends_on:
-    - mongo-db
-    healthcheck:
-      test: ["CMD", "bash", "./healtcheck.sh", "http://api-python:5000/"]
-      interval: 30s
-      timeout: 10s
-      retries: 10
-    volumes:
-      - ./ranking_system/dataset:/dataset
-      - ./ranking_system/tmpdata:/tmpdata
-      - ./ranking_system/cluster:/cluster
-      - ./ranking_system/weights:/root/.keras/models/
-    ports:
-    - 5000:5000
-    networks:
-      - lostpet
-  api-js:
-    build:
-      context: ./vk_interaction_obj_rec
-      dockerfile: Dockerfile
-    command: node VkWall.js
-    depends_on:
-      api-python:
-        condition: service_healthy
-    networks:
-      - lostpet
-  scheduler-py:
-    build:
-      context: ./scheduler
-      dockerfile: Dockerfile
-    command: python3 /codebase/main.py
-    depends_on:
-      api-python:
-        condition: service_healthy
-    networks:
-      - lostpet
+#  api-python:
+#    build:
+#      context: ./ranking_system
+#      dockerfile: Dockerfile
+#    command: python3 /codebase/api_server.py
+#    privileged: true
+#    depends_on:
+#    - mongo-db
+#    healthcheck:
+#      test: ["CMD", "bash", "./healtcheck.sh", "http://api-python:5000/"]
+#      interval: 30s
+#      timeout: 10s
+#      retries: 10
+#    volumes:
+#      - ./ranking_system/dataset:/dataset
+#      - ./ranking_system/tmpdata:/tmpdata
+#      - ./ranking_system/cluster:/cluster
+#      - ./ranking_system/weights:/root/.keras/models/
+#    ports:
+#    - 5000:5000
+#    networks:
+#      - default
+#  api-js:
+#    build:
+#      context: ./vk_interaction_obj_rec
+#      dockerfile: Dockerfile
+#    command: node VkWall.js
+#    networks:
+#      - bridge
+#    depends_on:
+#      api-python:
+#        condition: service_healthy
+#    networks:
+#      - default
+#  scheduler-py:
+#    build:
+#      context: ./scheduler
+#      dockerfile: Dockerfile
+#    command: python3 /codebase/main.py
+#    depends_on:
+#      api-python:
+#        condition: service_healthy
+#    networks:
+#      - default
   mongo-db:
     image: mongo
     environment:
       - MONGO_INITDB_ROOT_USERNAME=root
       - MONGO_INITDB_ROOT_PASSWORD=rootPassXXX
-      - API_URL="{{ vk_token }}"
     ports:
     - 27017:27017
-    networks:
-      - lostpet
-
 
-networks:
-  lostpet:
-    driver: bridge
+#networks:
+#  lostpet:
+#    driver: bridge

ranking_system/codebase/api_server.py

@@ -11,113 +11,109 @@
 import yaml
 
 # external modules
-import dbHelper as dbHelper
-import findHelper as findHelper
-import main as core
+from . import dbHelper as dbHelper
+from . import findHelper as findHelper
+from .import main as core
+
+
+cfg = yaml.safe_load(open("config.yaml"))
+mongourl = cfg["mongourl"]
+database = cfg["database"]
+collection = cfg["collection"]
+collection_new = cfg["collection_new"]
+db = dbHelper.Db(mongourl, database, collection)
+db_new = dbHelper.Db(mongourl, database, collection_new)
+find = findHelper.Find()
+app = FlaskAPI(__name__)
+
+def search_latest():
+    """
+    get request, search in db for latest record
+    """
+    response = db.search_latest_record()
+    return json_util.dumps(response["date"])
 
 
-class Api:
+def search():
     """
-    class for rest interaction
+    get request, search in db for data
     """
+    data = request.get_json()
+    response = db.search_records(data)
+    return json_util.dumps(response)
 
-    def __init__(self):
-        cfg = yaml.safe_load(open("config.yaml"))
-        mongourl = cfg["mongourl"]
-        database = cfg["database"]
-        collection = cfg["collection"]
-        collection_new = cfg["collection_new"]
-        self.db = dbHelper.Db(mongourl, database, collection)
-        self.db_new = dbHelper.Db(mongourl, database, collection_new)
-        self.find = findHelper.Find()
-        self.app = FlaskAPI(__name__)
-        self.app.add_url_rule("/search/", "search", self.search, methods=["POST"])
-        self.app.add_url_rule("/populate/", "populate", self.insert, methods=["POST"])
-        self.app.add_url_rule(
-            "/search_trans/", "search_trans", self.search_trans, methods=["POST"]
-        )
-        self.app.add_url_rule(
-            "/populate_trans/", "populate_trans", self.insert_trans, methods=["POST"]
-        )
-        self.app.add_url_rule(
-            "/find_image/", "find_image", self.find_image, methods=["POST"]
-        )
-        self.app.add_url_rule(
-            "/get_latest/", "search_latest", self.search_latest, methods=["GET"]
-        )
-        self.app.add_url_rule(
-            "/update_cluster/", "update_cluster", self.update_cluster, methods=["GET"]
-        )
-        logging.debug("Api has been initialized")
-
-    def search_latest(self):
-        """
-        get request, search in db for latest record
-        """
-        response = self.db.search_latest_record()
-        return json_util.dumps(response["date"])
-
-
-    def search(self):
-        """
-        get request, search in db for data
-        """
-        data = request.get_json()
-        response = self.db.search_records(data)
-        return json_util.dumps(response)
+def insert():
+    """
+    Get POST json. populate db
+    """
+    data = request.get_json()
+    logging.debug(data)
+    res = db.write_record(data)
+    return str(res)
 
-    def insert(self):
-        """
-        Get POST json. populate db
-        """
-        data = request.get_json()
-        logging.debug(data)
-        res = self.db.write_record(data)
-        return str(res)
+def search_trans():
+    """
+    get request, search in db for data
+    """
+    data = request.get_json()
+    response = db_new.search_records(data)
+    return json_util.dumps(response)
 
-    def search_trans(self):
-        """
-        get request, search in db for data
-        """
+def insert_trans():
+    """
+    Get POST json. populate db
+    """
+    data = request.get_json()
+    res = db_new.write_record(data)
+    return str(res)
+
+def find_image():
+    """
+    get json with 'photo' : %photo_name%
+    :return: search results
+    https://pp.userapi.com/c851216/v851216826/efbc4/pnz7eaWD3b8.jpg
+    db_new.search_record({'photos_name':{'$elemMatch':{'$in':['pnz7eaWD3b8.jpg']}}})
+    """
+    try:
         data = request.get_json()
-        response = self.db_new.search_records(data)
+        logging.debug(data)
+        response = find.main(data["photo"])
         return json_util.dumps(response)
+    except Exception:
+        logging.error(traceback.format_exc())
+
+def update_cluster():
+    """
+    invoce by get
+    :return: status
+    """
+    lastdate = db.search_latest_record()
+    core.update(lastdate)
+    return 'done'
+
+app.add_url_rule("/search/", "search", search, methods=["POST"])
+app.add_url_rule("/populate/", "populate", insert, methods=["POST"])
+app.add_url_rule(
+    "/search_trans/", "search_trans", search_trans, methods=["POST"]
+)
+app.add_url_rule(
+    "/populate_trans/", "populate_trans", insert_trans, methods=["POST"]
+)
+app.add_url_rule(
+    "/find_image/", "find_image", find_image, methods=["POST"]
+)
+app.add_url_rule(
+    "/get_latest/", "search_latest", search_latest, methods=["GET"]
+)
+app.add_url_rule(
+    "/update_cluster/", "update_cluster", update_cluster, methods=["GET"]
+)
+logging.debug("Api has been initialized")
 
-    def insert_trans(self):
-        """
-        Get POST json. populate db
-        """
-        data = request.get_json()
-        res = self.db_new.write_record(data)
-        return str(res)
-
-    def find_image(self):
-        """
-        get json with 'photo' : %photo_name%
-        :return: search results
-        https://pp.userapi.com/c851216/v851216826/efbc4/pnz7eaWD3b8.jpg
-        db_new.search_record({'photos_name':{'$elemMatch':{'$in':['pnz7eaWD3b8.jpg']}}})
-        """
-        try:
-            data = request.get_json()
-            logging.debug(data)
-            response = self.find.main(data["photo"])
-            return json_util.dumps(response)
-        except Exception:
-            logging.error(traceback.format_exc())
-
-    def update_cluster(self):
-        """
-        invoce by get
-        :return: status
-        """
-        lastdate = self.db.search_latest_record()
-        core.update(lastdate)
-        return 'done'
 
 
 if __name__ == "__main__":
     logging.getLogger().setLevel(logging.DEBUG)
-    Api().app.run(host="0.0.0.0", debug=False, threaded=False)
+    app.run(host="127.0.0.1", debug=False, threaded=False)
 
 

ranking_system/codebase/config.yaml

@@ -1,13 +1,16 @@
-mongourl: 'mongodb://root:rootPassXXX@mongo-db:27017/admin'
+mongourl: 'mongodb://root:rootPassXXX@localhost:27017/admin'
 database: 'lostpets'
 collection: 'dataset'
 collection_new: 'datastore'
 image_url: 'https://sun9-44.userapi.com/c635104/v635104532/56199/xUZvuTiE0pA.jpg'
 cluster_path: '../cluster/knn.joblib.pkl'
 dataset_path: '../dataset/'
 temp_path: '../tmpdata/'
-weights_file: '/root/.keras/models/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
+
+weights_file: '../ranking_system/weights/models/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
+#weights_file: '/root/.keras/models/vgg19_weights_tf_dim_ordering_tf_kernels.h5'
+weight_docker_folder: './ranking_system/weights'
 
 test:
-  api_url: 'http://35.228.24.125:5000'
-  mongodb: 'mongodb://root:rootPassXXX@35.228.24.125:27017/admin'
+  api_url: 'http://127.0.0.1:5000'
+  mongodb: 'mongodb://root:rootPassXXX@127.0.0.1:27017/admin'

ranking_system/codebase/dbTransform.py

@@ -1,4 +1,4 @@
-import dbHelper as dbHelper
+from . import dbHelper as dbHelper
 
 # logging
 import logging

ranking_system/codebase/findHelper.py

@@ -1,7 +1,7 @@
-import vectorize as vectorize
-import imageHelper as imageHelper
-import cluster as cluster
-import dbHelper as dbHelper
+from . import vectorize as vectorize
+from . import imageHelper as imageHelper
+from . import cluster as cluster
+from . import dbHelper as dbHelper
 import logging
 import numpy
 

ranking_system/codebase/main.py

@@ -1,7 +1,7 @@
-import vectorize as vectorize
-import cluster as cluster
-import imageHelper as imageHelper
-import dbTransform as trans
+from . import vectorize as vectorize
+from . import cluster as cluster
+from . import imageHelper as imageHelper
+from . import dbTransform as trans
 
 # get config
 import yaml

requirements.txt

@@ -0,0 +1 @@
+docker-compose