Feature/56 redesign replication (#57)

verida · Feb 15, 2023 · 0c6a785 · 0c6a785
1 parent 4bc1416
commit 0c6a785
Show file tree

Hide file tree

Showing 14 changed files with 504 additions and 280 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,8 @@
+2023-02-15 (2.1.0)
+--------------------
+
+- Redesign how replication works (initiated when a user connects and remains active for 20 minutes, instead of always replicate everything)
+
 2023-01-13 (2.0.0)
 --------------------
 

diff --git a/package.json b/package.json
@@ -11,8 +11,9 @@
     "dev": "nodemon --exec babel-node src/server.js",
     "build": "yarn clean && yarn babel src -d dist --extensions .js",
     "serve": "node --trace-warnings dist/server.js",
-    "build-docker-multiplatform": "yarn build && docker buildx build --platform linux/amd64,linux/arm64 --push -t verida/storage-node:latest .",
-    "build-docker-amd64": "yarn build && docker buildx build --platform linux/amd64 --push -t verida/storage-node:latest ."
+    "build-docker-multiplatform-dev": "yarn build && docker buildx build --platform linux/amd64,linux/arm64 --push -t verida/storage-node:dev .",
+    "build-docker-multiplatform-prod": "yarn build && docker buildx build --platform linux/amd64,linux/arm64 --push -t verida/storage-node:latest .",
+    "build-docker-amd64-prod": "yarn build && docker buildx build --platform linux/amd64 --push -t verida/storage-node:latest ."
   },
   "files": [
     "**/*.ts",
@@ -43,9 +44,9 @@
   "homepage": "https://github.com/verida/storage-node/README.md",
   "dependencies": {
     "@babel/runtime": "^7.16.7",
-    "@verida/did-client": "^2.0.0-rc4",
-    "@verida/did-document": "^2.0.0-rc4",
-    "@verida/encryption-utils": "^2.0.0-rc5",
+    "@verida/did-client": "^2.0.5",
+    "@verida/did-document": "^2.0.5",
+    "@verida/encryption-utils": "^2.0.4",
     "aws-serverless-express": "^3.4.0",
     "axios": "^1.2.1",
     "cors": "^2.8.5",

diff --git a/sample.env b/sample.env
@@ -27,8 +27,8 @@ ENDPOINT_URI="http://localhost:5000"
 
 DB_REJECT_UNAUTHORIZED_SSL=true
 ACCESS_JWT_SIGN_PK=insert-random-access-symmetric-key
-# 5 Minutes
-ACCESS_TOKEN_EXPIRY=300
+# 10 Minutes
+ACCESS_TOKEN_EXPIRY=600
 REFRESH_JWT_SIGN_PK=insert-random-refresh-symmetric-key
 # 30 Days
 REFRESH_TOKEN_EXPIRY=2592000
@@ -41,6 +41,9 @@ VDA_PRIVATE_KEY=
 DEFAULT_USER_CONTEXT_LIMIT_MB=10
 # Maximum number of users supported by this node
 MAX_USERS=10000
+# How many minutes before the replication expires on an open database
+# Should be 2x ACCESS_TOKEN_EXPIRY
+REPLICATION_EXPIRY_MINUTES=20
 
 # Alpha numeric only
 DB_PUBLIC_USER=784c2n780c9cn0789

diff --git a/src/components/authManager.js b/src/components/authManager.js
@@ -193,8 +193,6 @@ class AuthManager {
 
         await tokenDb.insert(tokenRow);
 
-        this.gc()
-
         return token
     }
 
@@ -378,7 +376,7 @@ class AuthManager {
             })
         } catch (err) {
             // Handle invalid JWT by rejecting verification
-            if (err.name == "JsonWebTokenError") {
+            if (err.name == "JsonWebTokenError" || err.name == "TokenExpiredError") {
                 return false
             }
 
@@ -446,18 +444,21 @@ class AuthManager {
             }
         }
 
+        const expiryIndex = {
+            index: { fields: ['expiry'] },
+            name: 'expiry'
+        };
+
+        const replicatorDb = couch.db.use('_replicator');
+        await replicatorDb.createIndex(expiryIndex);
+
         const tokenDb = couch.db.use(process.env.DB_REFRESH_TOKENS);
 
         const deviceIndex = {
             index: { fields: ['deviceHash'] },
             name: 'deviceHash'
         };
 
-        const expiryIndex = {
-            index: { fields: ['expiry'] },
-            name: 'expiry'
-        };
-
         await tokenDb.createIndex(deviceIndex);
         await tokenDb.createIndex(expiryIndex);
     }
@@ -533,15 +534,7 @@ class AuthManager {
     }
 
     // Garbage collection of refresh tokens
-    async gc() {
-        const GC_PERCENT = process.env.GC_PERCENT
-        const random = Math.random()
-
-        if (random >= GC_PERCENT) {
-            // Skip running GC
-            return
-        }
-
+    async clearExpired() {
         // Delete all expired refresh tokens
         const now = parseInt((new Date()).getTime() / 1000.0)
         const query = {
@@ -563,7 +556,7 @@ class AuthManager {
                     await tokenDb.destroy(doc._id, doc._rev)
                 } catch (err) {
                     if (err.error != 'not_found' && err.error != 'conflict') {
-                        console.log(`Unknown error in garbage collection: ${err.message}`)
+                        console.error(`Unknown error in garbage collection: ${err.message}`)
                     }
                 }
             }

diff --git a/src/components/dbManager.js b/src/components/dbManager.js
@@ -97,13 +97,18 @@ class DbManager {
         }
     }
 
-    async getUserDatabase(did, contextName, databaseName) {
+    async getUserDatabase(did, contextName, databaseName, isHash=false) {
         const couch = Db.getCouch()
         const didContextHash = Utils.generateDidContextHash(did, contextName)
         const didContextDbName = `c${didContextHash}`
         const db = couch.db.use(didContextDbName)
 
-        const id = Utils.generateDatabaseName(did, contextName, databaseName)
+        let id
+        if (isHash) {
+            id = databaseName
+        } else {
+            id = Utils.generateDatabaseName(did, contextName, databaseName)
+        }
 
         try {
             const doc = await db.get(id)
@@ -220,7 +225,6 @@ class DbManager {
 
     async configurePermissions(did, db, username, contextName, permissions) {
         try {
-            console.log(`configurePermissions() START`)
             permissions = permissions ? permissions : {};
 
             let owner = username;
@@ -310,7 +314,6 @@ class DbManager {
                 }
             }
 
-            console.log(`configurePermissions() END`)
             return true;
         } catch (err) {
             console.log(err)