diff --git a/src/PolykeyAgent.ts b/src/PolykeyAgent.ts index 7815302907..e3f033c710 100644 --- a/src/PolykeyAgent.ts +++ b/src/PolykeyAgent.ts @@ -690,6 +690,7 @@ class PolykeyAgent { this.logger.info(`Started ${this.constructor.name}`); } catch (e) { this.logger.warn(`Failed Starting ${this.constructor.name}`); + this.events.removeAllListeners(); await this.status?.beginStop({ pid: process.pid }); await this.sessionManager?.stop(); await this.notificationsManager?.stop(); @@ -706,7 +707,6 @@ class PolykeyAgent { await this.keyManager?.stop(); await this.schema?.stop(); await this.status?.stop({}); - this.events.removeAllListeners(); throw e; } } @@ -716,6 +716,7 @@ class PolykeyAgent { */ public async stop() { this.logger.info(`Stopping ${this.constructor.name}`); + this.events.removeAllListeners(); await this.status.beginStop({ pid: process.pid }); await this.sessionManager.stop(); await this.notificationsManager.stop(); @@ -736,7 +737,6 @@ class PolykeyAgent { await this.keyManager.stop(); await this.schema.stop(); await this.status.stop({}); - this.events.removeAllListeners(); this.logger.info(`Stopped ${this.constructor.name}`); } diff --git a/src/nodes/NodeGraph.ts b/src/nodes/NodeGraph.ts index afa0a81569..a5b8da3323 100644 --- a/src/nodes/NodeGraph.ts +++ b/src/nodes/NodeGraph.ts @@ -18,7 +18,6 @@ import { IdInternal } from '@matrixai/id'; import * as nodesUtils from './utils'; import * as nodesErrors from './errors'; import { getUnixtime, never } from '../utils'; -import { bucketKey } from './utils'; /** * NodeGraph is an implementation of Kademlia for maintaining peer to peer information @@ -298,10 +297,7 @@ class NodeGraph { const bucketPath = [...this.nodeGraphBucketsDbPath, bucketKey]; const lastUpdatedPath = [...this.nodeGraphLastUpdatedDbPath, bucketKey]; const nodeIdKey = nodesUtils.bucketDbKey(nodeId); - const nodeData = await tran.get([ - ...bucketPath, - nodeIdKey, - ]); + const nodeData = await tran.get([...bucketPath, nodeIdKey]); if (nodeData != null) { this.logger.debug( `Removing node ${nodesUtils.encodeNodeId( @@ -514,8 +510,10 @@ class NodeGraph { ); } + const logger = this.logger.getChild('resetBuckets'); // Setup new space const spaceNew = this.space === '0' ? '1' : '0'; + logger.debug('new space: ' + spaceNew); const nodeGraphMetaDbPathNew = [...this.nodeGraphDbPath, 'meta' + spaceNew]; const nodeGraphBucketsDbPathNew = [ ...this.nodeGraphDbPath, @@ -536,11 +534,16 @@ class NodeGraph { this.nodeGraphBucketsDbPath, )) { // The key is a combined bucket key and node ID - const { nodeId } = nodesUtils.parseBucketsDbKey(key as Array); + const { bucketIndex: bucketIndexOld, nodeId } = + nodesUtils.parseBucketsDbKey(key as Array); + const nodeIdEncoded = nodesUtils.encodeNodeId(nodeId); const nodeIdKey = nodesUtils.bucketDbKey(nodeId); // If the new own node ID is one of the existing node IDs, it is just dropped // We only map to the new bucket if it isn't one of the existing node IDs if (nodeId.equals(nodeIdOwn)) { + logger.debug( + `nodeId ${nodeIdEncoded} from bucket ${bucketIndexOld} was identical to new NodeId and was dropped.`, + ); continue; } const bucketIndexNew = nodesUtils.bucketIndex(nodeIdOwn, nodeId); @@ -552,7 +555,7 @@ class NodeGraph { if (countNew < this.nodeBucketLimit) { await tran.put([...metaPathNew, 'count'], countNew + 1); } else { - let oldestIndexKey: Buffer | undefined = undefined; + let oldestIndexKey: Array | undefined = undefined; let oldestNodeId: NodeId | undefined = undefined; for await (const [key] of tran.iterator( { @@ -560,7 +563,7 @@ class NodeGraph { }, indexPathNew, )) { - oldestIndexKey = key as unknown as Buffer; + oldestIndexKey = key as Array; ({ nodeId: oldestNodeId } = nodesUtils.parseLastUpdatedBucketDbKey( key as Array, )); @@ -569,12 +572,16 @@ class NodeGraph { ...bucketPathNew, nodesUtils.bucketDbKey(oldestNodeId!), ]); - await tran.del([...indexPathNew, oldestIndexKey!]); + await tran.del([...indexPathNew, ...oldestIndexKey!]); } - await tran.put( - [...bucketPathNew, nodeIdKey], - nodeData, - ); + if (bucketIndexOld !== bucketIndexNew) { + logger.debug( + `nodeId ${nodeIdEncoded} moved ${bucketIndexOld}=>${bucketIndexNew}`, + ); + } else { + logger.debug(`nodeId ${nodeIdEncoded} unchanged ${bucketIndexOld}`); + } + await tran.put([...bucketPathNew, nodeIdKey], nodeData); const lastUpdatedKey = nodesUtils.lastUpdatedKey(nodeData.lastUpdated); await tran.put( [...indexPathNew, lastUpdatedKey, nodeIdKey], diff --git a/src/nodes/NodeManager.ts b/src/nodes/NodeManager.ts index 0774986742..23832dbb7f 100644 --- a/src/nodes/NodeManager.ts +++ b/src/nodes/NodeManager.ts @@ -429,7 +429,6 @@ class NodeManager { const nodeData = await this.nodeGraph.getNode(nodeId, tran); // If this is a new entry, check the bucket limit const [bucketIndex] = this.nodeGraph.bucketIndex(nodeId); - console.trace(1); const count = await this.nodeGraph.getBucketMetaProp( bucketIndex, 'count', @@ -525,7 +524,6 @@ class NodeManager { } } // Check if we now have room and add the new node - console.trace(1); const count = await this.nodeGraph.getBucketMetaProp(bucketIndex, 'count'); if (count < this.nodeGraph.nodeBucketLimit) { this.logger.debug(`Bucket ${bucketIndex} now has room, adding new node`); diff --git a/src/nodes/utils.ts b/src/nodes/utils.ts index 457ad12dea..449fc407bd 100644 --- a/src/nodes/utils.ts +++ b/src/nodes/utils.ts @@ -136,7 +136,7 @@ function lastUpdatedBucketDbKey(lastUpdated: number, nodeId: NodeId): Buffer { } function lastUpdatedKey(lastUpdated: number): Buffer { - return Buffer.from(lexi.pack(lastUpdated, 'hex')) + return Buffer.from(lexi.pack(lastUpdated, 'hex')); } /** @@ -192,8 +192,9 @@ function parseLastUpdatedBucketsDbKey(keyBufferArray: Array): { if (bucketIndex == null) { throw new TypeError('Buffer is not an NodeGraph index key'); } - const { lastUpdated, nodeId } = - parseLastUpdatedBucketDbKey(lastUpdatedBufferArray); + const { lastUpdated, nodeId } = parseLastUpdatedBucketDbKey( + lastUpdatedBufferArray, + ); return { bucketIndex, bucketKey, diff --git a/tests/client/service/agentLockAll.test.ts b/tests/client/service/agentLockAll.test.ts index d44eef0933..fe56a0d7d6 100644 --- a/tests/client/service/agentLockAll.test.ts +++ b/tests/client/service/agentLockAll.test.ts @@ -14,8 +14,8 @@ import { ClientServiceService } from '@/proto/js/polykey/v1/client_service_grpc_ import * as utilsPB from '@/proto/js/polykey/v1/utils/utils_pb'; import * as keysUtils from '@/keys/utils'; import * as clientUtils from '@/client/utils/utils'; -import * as testUtils from '../../utils'; import { timerStart } from '@/utils/index'; +import * as testUtils from '../../utils'; describe('agentLockall', () => { const logger = new Logger('agentLockall test', LogLevel.WARN, [ diff --git a/tests/gestalts/GestaltGraph.test.ts b/tests/gestalts/GestaltGraph.test.ts index 0953a2b4a4..e24a08e00a 100644 --- a/tests/gestalts/GestaltGraph.test.ts +++ b/tests/gestalts/GestaltGraph.test.ts @@ -1248,8 +1248,8 @@ describe('GestaltGraph', () => { // its just that node 1 is eliminated nodePerms = await acl.getNodePerms(); expect(Object.keys(nodePerms)).toHaveLength(1); - expect(nodePerms[0]).not.toHaveProperty(nodeIdABC.toString()); - expect(nodePerms[0]).toHaveProperty(nodeIdDEE.toString()); + expect(nodePerms[0][nodeIdABC.toString()]).toBeUndefined(); + expect(nodePerms[0][nodeIdDEE.toString()]).toBeDefined(); await gestaltGraph.unsetNode(nodeIdDEE); nodePerms = await acl.getNodePerms(); expect(Object.keys(nodePerms)).toHaveLength(0); diff --git a/tests/nodes/NodeConnection.test.ts b/tests/nodes/NodeConnection.test.ts index 541094b1b2..dbd95397e9 100644 --- a/tests/nodes/NodeConnection.test.ts +++ b/tests/nodes/NodeConnection.test.ts @@ -63,7 +63,7 @@ const mockedGenerateDeterministicKeyPair = jest.spyOn( 'generateDeterministicKeyPair', ); -describe('${NodeConnection.name} test', () => { +describe(`${NodeConnection.name} test`, () => { const logger = new Logger(`${NodeConnection.name} test`, LogLevel.WARN, [ new StreamHandler(), ]); @@ -342,7 +342,6 @@ describe('${NodeConnection.name} test', () => { }, global.polykeyStartupTimeout * 2); afterEach(async () => { - console.log('b'); await clientProxy.stop(); await clientKeyManager.stop(); await clientKeyManager.destroy(); @@ -350,41 +349,32 @@ describe('${NodeConnection.name} test', () => { force: true, recursive: true, }); - console.log('b'); await serverACL.stop(); await serverACL.destroy(); - console.log('b'); await serverSigchain.stop(); await serverSigchain.destroy(); await serverGestaltGraph.stop(); await serverGestaltGraph.destroy(); - console.log('b'); await serverVaultManager.stop(); await serverVaultManager.destroy(); await serverNodeGraph.stop(); - console.log('b'); await serverNodeGraph.destroy(); await serverNodeConnectionManager.stop(); await serverNodeManager.stop(); - console.log('b'); await serverQueue.stop(); await serverNotificationsManager.stop(); await serverNotificationsManager.destroy(); - console.log('b'); await agentTestUtils.closeTestAgentServer(agentServer); await serverProxy.stop(); await serverKeyManager.stop(); - console.log('b'); await serverKeyManager.destroy(); await serverDb.stop(); await serverDb.destroy(); - console.log('b'); await fs.promises.rm(serverDataDir, { force: true, recursive: true, }); - console.log('b'); }); test('session readiness', async () => { @@ -491,7 +481,6 @@ describe('${NodeConnection.name} test', () => { let nodeConnection: NodeConnection | undefined; let polykeyAgent: PolykeyAgent | undefined; try { - console.log('a'); polykeyAgent = await PolykeyAgent.createPolykeyAgent({ password, nodePath: path.join(dataDir, 'PolykeyAgent3'), @@ -514,29 +503,23 @@ describe('${NodeConnection.name} test', () => { targetPort: polykeyAgent.proxy.getProxyPort(), clientFactory: (args) => GRPCClientAgent.createGRPCClientAgent(args), }); - console.log('a'); // Resolves if the shutdownCallback was called await polykeyAgent.stop(); await polykeyAgent.destroy(); - console.log('a'); const client = nodeConnection.getClient(); const echoMessage = new utilsPB.EchoMessage().setChallenge( 'Hello world!', ); - console.log('a'); await expect(async () => client.echo(echoMessage)).rejects.toThrow( agentErrors.ErrorAgentClientDestroyed, ); - console.log('a'); } finally { - console.log('ending'); await polykeyAgent?.stop(); await polykeyAgent?.destroy(); await nodeConnection?.destroy(); } - console.log('asdasdasd') }); test('fails to connect to target (times out)', async () => { await expect( diff --git a/tests/nodes/NodeGraph.test.ts b/tests/nodes/NodeGraph.test.ts index 7174de01ca..4ad0c70639 100644 --- a/tests/nodes/NodeGraph.test.ts +++ b/tests/nodes/NodeGraph.test.ts @@ -23,7 +23,7 @@ import * as testUtils from '../utils'; describe(`${NodeGraph.name} test`, () => { const password = 'password'; - const logger = new Logger(`${NodeGraph.name} test`, LogLevel.WARN, [ + const logger = new Logger(`${NodeGraph.name} test`, LogLevel.DEBUG, [ new StreamHandler(), ]); let mockedGenerateKeyPair: jest.SpyInstance; @@ -628,7 +628,6 @@ describe(`${NodeGraph.name} test`, () => { // Resetting to the same NodeId results in the same bucket structure await nodeGraph.resetBuckets(nodeIdNew2); const buckets3 = await utils.asyncIterableArray(nodeGraph.getBuckets()); - console.log(buckets3.map(([a, b]) => [a, b.map(b => b[0].toMultibase('base32hex'))] )); expect(buckets3).toStrictEqual(buckets2); // Resetting to an existing NodeId const nodeIdExisting = buckets3[0][1][0][0]; diff --git a/tests/nodes/utils.test.ts b/tests/nodes/utils.test.ts index c9a3a6d480..0d962f963e 100644 --- a/tests/nodes/utils.test.ts +++ b/tests/nodes/utils.test.ts @@ -111,7 +111,10 @@ describe('nodes/utils', () => { nodeId, key: Buffer.concat([Buffer.from(bucketKey), nodeId]), }); - await db.put(['buckets', bucketKey, nodesUtils.bucketDbKey(nodeId)], null); + await db.put( + ['buckets', bucketKey, nodesUtils.bucketDbKey(nodeId)], + null, + ); } // LevelDB will store keys in lexicographic order // Use the key property as a concatenated buffer of the bucket key and node ID @@ -142,18 +145,16 @@ describe('nodes/utils', () => { const bucketKey = lexi.pack(bucketIndex, 'hex'); const lastUpdated = utils.getUnixtime(); const nodeId = testNodesUtils.generateRandomNodeId(); - const lastUpdatedKey = nodesUtils.lastUpdatedBucketDbKey( - lastUpdated, - nodeId, - ); + const nodeIdKey = nodesUtils.bucketDbKey(nodeId); + const lastUpdatedKey = nodesUtils.lastUpdatedKey(lastUpdated); data.push({ bucketIndex, bucketKey, lastUpdated, nodeId, - key: Buffer.concat([Buffer.from(bucketKey), lastUpdatedKey]), + key: Buffer.concat([Buffer.from(bucketKey), lastUpdatedKey, nodeIdKey]), }); - await db.put(['lastUpdated', bucketKey, lastUpdatedKey], null); + await db.put(['lastUpdated', bucketKey, lastUpdatedKey, nodeIdKey], null); } // LevelDB will store keys in lexicographic order // Use the key property as a concatenated buffer of diff --git a/tests/vaults/VaultInternal.test.ts b/tests/vaults/VaultInternal.test.ts index ab817a538b..d95ae1c2ca 100644 --- a/tests/vaults/VaultInternal.test.ts +++ b/tests/vaults/VaultInternal.test.ts @@ -678,60 +678,64 @@ describe('VaultInternal', () => { expect(log).toHaveLength(2); expect(log[0].commitId).toStrictEqual(commit); }); - test('garbage collection', async () => { - await vault.writeF(async (efs) => { - await efs.writeFile(secret1.name, secret1.content); - }); - await vault.writeF(async (efs) => { - await efs.writeFile(secret2.name, secret2.content); - }); - await vault.writeF(async (efs) => { - await efs.writeFile(secret3.name, secret3.content); - }); - // @ts-ignore: kidnap efs - const vaultEfs = vault.efs; - // @ts-ignore: kidnap efs - const vaultEfsData = vault.efsVault; - const quickCommit = async (ref: string, secret: string) => { - await vaultEfsData.writeFile(secret, secret); - await git.add({ - fs: vaultEfs, - dir: vault.vaultDataDir, - gitdir: vault.vaultGitDir, - filepath: secret, + test( + 'garbage collection', + async () => { + await vault.writeF(async (efs) => { + await efs.writeFile(secret1.name, secret1.content); }); - return await git.commit({ - fs: vaultEfs, - dir: vault.vaultDataDir, - gitdir: vault.vaultGitDir, - author: { - name: 'test', - email: 'test', - }, - message: 'test', - ref: ref, + await vault.writeF(async (efs) => { + await efs.writeFile(secret2.name, secret2.content); }); - }; - const log = await vault.log(); - let num = 5; - const refs: string[] = []; - for (const logElement of log) { - refs.push(await quickCommit(logElement.commitId, `secret-${num++}`)); - } - // @ts-ignore - await vault.garbageCollectGitObjects(); - - for (const ref of refs) { - await expect( - git.checkout({ + await vault.writeF(async (efs) => { + await efs.writeFile(secret3.name, secret3.content); + }); + // @ts-ignore: kidnap efs + const vaultEfs = vault.efs; + // @ts-ignore: kidnap efs + const vaultEfsData = vault.efsVault; + const quickCommit = async (ref: string, secret: string) => { + await vaultEfsData.writeFile(secret, secret); + await git.add({ fs: vaultEfs, dir: vault.vaultDataDir, gitdir: vault.vaultGitDir, - ref, - }), - ).rejects.toThrow(git.Errors.CommitNotFetchedError); - } - }); + filepath: secret, + }); + return await git.commit({ + fs: vaultEfs, + dir: vault.vaultDataDir, + gitdir: vault.vaultGitDir, + author: { + name: 'test', + email: 'test', + }, + message: 'test', + ref: ref, + }); + }; + const log = await vault.log(); + let num = 5; + const refs: string[] = []; + for (const logElement of log) { + refs.push(await quickCommit(logElement.commitId, `secret-${num++}`)); + } + // @ts-ignore + await vault.garbageCollectGitObjects(); + + for (const ref of refs) { + await expect( + git.checkout({ + fs: vaultEfs, + dir: vault.vaultDataDir, + gitdir: vault.vaultGitDir, + ref, + }), + ).rejects.toThrow(git.Errors.CommitNotFetchedError); + } + }, + global.defaultTimeout * 2, + ); // Locking tests const waitDelay = 200; test('writeF respects read and write locking', async () => {