Merge pull request #322 from steemit/redis_cache

Redis cache layer

Dockerfile.dev

@@ -0,0 +1,46 @@
+FROM phusion/baseimage:0.11
+
+
+ENV ENVIRONMENT DEV
+ENV LOG_LEVEL INFO
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV PIPENV_VENV_IN_PROJECT 1
+ARG SOURCE_COMMIT
+ENV SOURCE_COMMIT ${SOURCE_COMMIT}
+ARG SCHEMA_HASH
+ENV SCHEMA_HASH adb5cd9b
+#ENV SCHEMA_HASH ${SCHEMA_HASH}
+ARG DOCKER_TAG
+ENV DOCKER_TAG ${DOCKER_TAG}
+
+ENV APP_ROOT /app
+ENV WSGI_APP ${APP_ROOT}/hive/server/serve.py
+ENV HTTP_SERVER_PORT 8080
+
+RUN \
+    apt-get update && \
+    apt-get install -y \
+        awscli \
+        build-essential \
+        daemontools \
+        libffi-dev \
+        libmysqlclient-dev \
+        libssl-dev \
+        make \
+        liblz4-tool \
+        postgresql \
+        postgresql-contrib \
+        python3 \
+        python3-dev \
+        python3-pip \
+        libxml2-dev \
+        libxslt-dev \
+        runit \
+        s3cmd \
+        libpcre3 \
+        libpcre3-dev
+
+RUN \
+    pip3 install --upgrade pip setuptools
+

dev_env.sh

@@ -0,0 +1,58 @@
+#/bin/bash
+
+container_name=dev-hivemind-env
+
+run() {
+    tag=$1
+    network=$2
+
+    if [ "${tag}" = "" ]; then
+        tag=dev-env
+    fi
+
+    if [ -n ${network} ]; then
+        network_str="--network ${network}"
+    fi
+
+    docker run -it --rm \
+        --name ${container_name} \
+        -v $(pwd):/project \
+        --env-file $(pwd)/.env \
+        --workdir /project \
+        ${network_str} \
+        steemit/hivemind:${tag} \
+        /bin/bash
+}
+
+cli() {
+    docker exec -it ${container_name} /bin/bash
+}
+
+stop() {
+    docker stop ${container_name}
+}
+
+main_func() {
+    op=$1
+    tag=$2
+    network=$3
+
+    case ${op} in
+        run)
+            run $tag $network
+            ;;
+        cli)
+            cli
+            ;;
+        stop)
+            stop
+            ;;
+        *)
+            echo "Unknown Command"
+            exit 1
+            ;;
+    esac
+}
+
+main_func $1 $2 $3
+

hive/conf.py

@@ -31,8 +31,9 @@ def init_argparse(cls, strict=True, **kwargs):
         add('mode', nargs='*', default=['sync'])
 
         # common
-        add('--database-url', env_var='DATABASE_URL', required=False, help='database connection url', default='')
-        add('--steemd-url', env_var='STEEMD_URL', required=False, help='steemd/jussi endpoint', default='https://api.steemit.com')
+        add('--database-url', env_var='DATABASE_URL', required=True, help='database connection url', default='')
+        add('--steemd-url', env_var='STEEMD_URL', required=True, help='steemd/jussi endpoint', default='https://api.steemit.com')
+        add('--redis-url', env_var='REDIS_URL', required=True, help='redis connection url', default='')
         add('--muted-accounts-url', env_var='MUTED_ACCOUNTS_URL', required=False, help='url to flat list of muted accounts', default='')
 
         # server

hive/db/db_state.py

@@ -173,7 +173,7 @@ def status():
         row = DbState.db().query_row(sql)
         return dict(db_head_block=row['num'],
                     db_head_time=str(row['created_at']),
-                    db_head_age=int(time.time() - row['ts']))
+                    db_head_age=int(time.time() - float(row['ts'])))
 
     @classmethod
     def _is_schema_loaded(cls):

hive/server/bridge_api/cursor.py

@@ -428,18 +428,30 @@ async def pids_by_replies(db, start_author: str, start_permlink: str = '',
         parent_account = start_author
 
     sql = """
-       SELECT id FROM hive_posts
-        WHERE parent_id IN (SELECT id FROM hive_posts
-                             WHERE author = :parent
-                               AND is_deleted = '0'
-                          ORDER BY id DESC
-                             LIMIT 10000) %s
-          AND is_deleted = '0'
-     ORDER BY id DESC
-        LIMIT :limit
-    """ % seek
+    SELECT id FROM hive_posts
+    WHERE author = :parent
+    AND is_deleted = '0'
+    ORDER BY id DESC
+    LIMIT 10000
+    """
 
-    return await db.query_col(sql, parent=parent_account, start_id=start_id, limit=limit)
+    cache_key = "hive_posts-" + parent_account + "-is_deleted_0"
+    id_res = await db.query_all(sql, cache_key=cache_key, parent=parent_account)
+    if id_res == None or len(id_res) == 0:
+        return None
+    tmp_ids = []
+    for el in id_res:
+        tmp_ids.append(str(el[0]))
+    ids = ",".join(tmp_ids)
+
+    sql = """
+    SELECT id FROM hive_posts
+    WHERE parent_id IN (%s) %s
+    AND is_deleted = '0'
+    ORDER BY id DESC
+    LIMIT :limit
+    """ % (ids, seek)
+    return await db.query_col(sql, limit=limit)
 
 async def pids_by_payout(db, account: str, start_author: str = '',
                          start_permlink: str = '', limit: int = 20):

hive/server/common/helpers.py

@@ -20,6 +20,7 @@ async def wrapper(*args, **kwargs):
         try:
             return await function(*args, **kwargs)
         except (ApiError, AssertionError, TypeError, Exception) as e:
+            log.error("ERR-DEBUG: args: {%s}, kwargs: {%s}", args, kwargs)
             if isinstance(e, KeyError):
                 #TODO: KeyError overloaded for method not found. Any KeyErrors
                 #      captured in this decorater are likely irrelevant to
@@ -39,7 +40,6 @@ async def wrapper(*args, **kwargs):
             if isinstance(e, AssertionError):
                 log.error("ERR2: %s\n%s", repr(e), traceback.format_exc())
                 raise e
-            log.error("ERR0: %s\n%s", repr(e), traceback.format_exc())
             raise e
             #return {
             #    "error": {

hive/server/condenser_api/cursor.py

@@ -4,9 +4,14 @@
 from dateutil.relativedelta import relativedelta
 
 from hive.utils.normalize import rep_to_raw
+import re
 
 # pylint: disable=too-many-lines
 
+def to_string_without_special_char(v):
+    cleaned_string = re.sub('[^A-Za-z0-9]+', '', str(v))
+    return cleaned_string
+
 def last_month():
     """Get the date 1 month ago."""
     return datetime.now() + relativedelta(months=-1)
@@ -15,7 +20,8 @@ async def get_post_id(db, author, permlink):
     """Given an author/permlink, retrieve the id from db."""
     sql = ("SELECT id FROM hive_posts WHERE author = :a "
            "AND permlink = :p AND is_deleted = '0' LIMIT 1")
-    return await db.query_one(sql, a=author, p=permlink)
+    cache_key = "get_post_id_" + author + "_" + permlink
+    return await db.query_one(sql, a=author, p=permlink, cache_key=cache_key)
 
 async def get_child_ids(db, post_id):
     """Given a parent post id, retrieve all child ids."""
@@ -25,12 +31,14 @@ async def get_child_ids(db, post_id):
 async def _get_post_id(db, author, permlink):
     """Get post_id from hive db."""
     sql = "SELECT id FROM hive_posts WHERE author = :a AND permlink = :p"
-    return await db.query_one(sql, a=author, p=permlink)
+    cache_key = "_get_post_id_" + author + "_" + permlink
+    return await db.query_one(sql, a=author, p=permlink, cache_key=cache_key)
 
 async def _get_account_id(db, name):
     """Get account id from hive db."""
     assert name, 'no account name specified'
-    _id = await db.query_one("SELECT id FROM hive_accounts WHERE name = :n", n=name)
+    cache_key = "_get_account_id_" + name
+    _id = await db.query_one("SELECT id FROM hive_accounts WHERE name = :n", n=name, cache_key=cache_key)
     assert _id, "account not found: `%s`" % name
     return _id
 
@@ -57,8 +65,13 @@ async def get_followers(db, account: str, start: str, follow_type: str, limit: i
          LIMIT :limit
     """ % seek
 
+    cache_key = "get_followers_"
+    cache_key = cache_key + to_string_without_special_char(account_id) + "_"
+    cache_key = cache_key + to_string_without_special_char(start_id) + "_"
+    cache_key = cache_key + to_string_without_special_char(state)
+
     return await db.query_all(sql, account_id=account_id, start_id=start_id,
-                              state=state, limit=limit)
+                              state=state, limit=limit, cache_key=cache_key)
 
 
 async def get_followers_by_page(db, account: str, page: int, page_size: int, follow_type: str):
@@ -75,8 +88,14 @@ async def get_followers_by_page(db, account: str, page: int, page_size: int, fol
          LIMIT :limit OFFSET :offset
     """
 
+    cache_key = "get_followers_by_page_"
+    cache_key = cache_key + to_string_without_special_char(account_id) + "_"
+    cache_key = cache_key + to_string_without_special_char(state) + "_"
+    cache_key = cache_key + to_string_without_special_char(page*page_size)
+
     return await db.query_all(sql, account_id=account_id,
-                              state=state, limit=page_size, offset=page*page_size)
+                              state=state, limit=page_size, offset=page*page_size,
+                              cache_key=cache_key)
 
 async def get_following(db, account: str, start: str, follow_type: str, limit: int):
     """Get a list of accounts followed by a given account."""
@@ -100,8 +119,13 @@ async def get_following(db, account: str, start: str, follow_type: str, limit: i
          LIMIT :limit
     """ % seek
 
+    cache_key = "get_following_"
+    cache_key = cache_key + to_string_without_special_char(account_id) + "_"
+    cache_key = cache_key + to_string_without_special_char(start_id) + "_"
+    cache_key = cache_key + to_string_without_special_char(state)
+
     return await db.query_all(sql, account_id=account_id, start_id=start_id,
-                              state=state, limit=limit)
+                              state=state, limit=limit, cache_key=cache_key)
 
 
 async def get_following_by_page(db, account: str, page: int, page_size: int, follow_type: str):
@@ -118,8 +142,14 @@ async def get_following_by_page(db, account: str, page: int, page_size: int, fol
          LIMIT :limit OFFSET :offset
     """
 
+    cache_key = "get_following_by_page_"
+    cache_key = cache_key + to_string_without_special_char(account_id) + "_"
+    cache_key = cache_key + to_string_without_special_char(state) + "_"
+    cache_key = cache_key + to_string_without_special_char(page*page_size)
+
     return await db.query_all(sql, account_id=account_id,
-                              state=state, limit=page_size, offset=page*page_size)
+                              state=state, limit=page_size, offset=page*page_size,
+                              cache_key=cache_key)
 
 
 async def get_follow_counts(db, account: str):
@@ -395,17 +425,30 @@ async def pids_by_replies_to_account(db, start_author: str, start_permlink: str
         seek = "AND id <= :start_id"
     else:
         parent_account = start_author
-
+    
     sql = """
-       SELECT id FROM hive_posts
-        WHERE parent_id IN (SELECT id FROM hive_posts
-                             WHERE author = :parent
-                               AND is_deleted = '0'
-                          ORDER BY id DESC
-                             LIMIT 10000) %s
-          AND is_deleted = '0'
-     ORDER BY id DESC
-        LIMIT :limit
-    """ % seek
+    SELECT id FROM hive_posts
+    WHERE author = :parent
+    AND is_deleted = '0'
+    ORDER BY id DESC
+    LIMIT 10000
+    """
 
-    return await db.query_col(sql, parent=parent_account, start_id=start_id, limit=limit)
+    cache_key = "hive_posts-" + parent_account + "-is_deleted_0"
+    id_res = await db.query_all(sql, parent=parent_account, cache_key=cache_key)
+    if id_res == None or len(id_res) == 0:
+        return None
+    tmp_ids = []
+    for el in id_res:
+        tmp_ids.append(str(el[0]))
+    ids = ",".join(tmp_ids)
+
+    sql = """
+    SELECT id FROM hive_posts
+    WHERE parent_id IN (%s) %s
+    AND is_deleted = '0'
+    ORDER BY id DESC
+    LIMIT :limit
+    """ % (ids, seek)
+
+    return await db.query_col(sql, limit=limit)

hive/server/condenser_api/get_state.py

@@ -4,7 +4,6 @@
 import logging
 from collections import OrderedDict
 import ujson as json
-from aiocache import cached
 
 from hive.utils.normalize import legacy_amount
 from hive.server.common.mutes import Mutes
@@ -278,16 +277,24 @@ async def _load_discussion(db, author, permlink):
     # return all nodes keyed by ref
     return {refs[pid]: post for pid, post in posts.items()}
 
-@cached(ttl=1800, timeout=1200)
 async def _get_feed_price(db):
     """Get a steemd-style ratio object representing feed price."""
-    price = await db.query_one("SELECT usd_per_steem FROM hive_state")
+    price = await db.query_one(
+        "SELECT usd_per_steem FROM hive_state",
+        cache_key="_get_feed_price",
+        cache_ttl=1800)
     return {"base": "%.3f SBD" % price, "quote": "1.000 STEEM"}
 
-@cached(ttl=1800, timeout=1200)
 async def _get_props_lite(db):
     """Return a minimal version of get_dynamic_global_properties data."""
-    raw = json.loads(await db.query_one("SELECT dgpo FROM hive_state"))
+    tmp = await db.query_one(
+        "SELECT dgpo FROM hive_state",
+        cache_key="_hive_state_dgpo",
+        cache_ttl=300)
+    if tmp is None or tmp == '':
+        return dict()
+
+    raw = json.loads(tmp)
 
     # convert NAI amounts to legacy
     nais = ['virtual_supply', 'current_supply', 'current_sbd_supply',