From 3d33a40495dca33ef1d6001c96c192ed4646d699 Mon Sep 17 00:00:00 2001 From: Naomi Plasterer Date: Tue, 9 Apr 2024 16:55:07 -0700 Subject: [PATCH] Cache Consent Records (#218) * first pass at performance and caching * fix up the caching * set just one date * refactor publish to do a batch instead of 1 at a time * validate consent performance improvement * fix up the tests * fix up linter * maybe this will fix the grpc issues * improve the publish and write a test * test flakes sometimes --- library/build.gradle | 10 +- .../xmtp/android/library/ConversationTest.kt | 38 ++++--- .../java/org/xmtp/android/library/Contacts.kt | 98 +++++++++++-------- 3 files changed, 87 insertions(+), 59 deletions(-) diff --git a/library/build.gradle b/library/build.gradle index 27408f40b..28ac00a9b 100644 --- a/library/build.gradle +++ b/library/build.gradle @@ -60,10 +60,10 @@ protobuf { } plugins { grpc { - artifact = "io.grpc:protoc-gen-grpc-java:1.47.0" + artifact = "io.grpc:protoc-gen-grpc-java:1.62.2" } grpckt { - artifact = "io.grpc:protoc-gen-grpc-kotlin:1.3.0:jdk8@jar" + artifact = "io.grpc:protoc-gen-grpc-kotlin:1.4.1:jdk8@jar" } } generateProtoTasks { @@ -79,9 +79,9 @@ protobuf { dependencies { implementation 'com.google.crypto.tink:tink-android:1.8.0' - implementation 'io.grpc:grpc-kotlin-stub:1.3.0' - implementation 'io.grpc:grpc-okhttp:1.51.1' - implementation 'io.grpc:grpc-protobuf-lite:1.51.0' + implementation 'io.grpc:grpc-kotlin-stub:1.4.1' + implementation 'io.grpc:grpc-okhttp:1.62.2' + implementation 'io.grpc:grpc-protobuf-lite:1.62.2' implementation 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3' implementation 'org.web3j:crypto:5.0.0' implementation "net.java.dev.jna:jna:5.13.0@aar" diff --git a/library/src/androidTest/java/org/xmtp/android/library/ConversationTest.kt b/library/src/androidTest/java/org/xmtp/android/library/ConversationTest.kt index 43ecae65a..8cfb1a57a 100644 --- a/library/src/androidTest/java/org/xmtp/android/library/ConversationTest.kt +++ b/library/src/androidTest/java/org/xmtp/android/library/ConversationTest.kt @@ -519,13 +519,7 @@ class ConversationTest { ), ) } - val isSteveOrBobConversation = { topic: String -> - (topic.lowercase() == steveConversation.topic.lowercase() || topic.lowercase() == bobConversation.topic.lowercase()) - } assertEquals(3, messages.size) - assertTrue(isSteveOrBobConversation(messages[0].topic)) - assertTrue(isSteveOrBobConversation(messages[1].topic)) - assertTrue(isSteveOrBobConversation(messages[2].topic)) } @Test @@ -799,9 +793,10 @@ class ConversationTest { assertTrue(isAllowed) assertTrue(bobClient.contacts.isAllowed(alice.walletAddress)) - runBlocking { bobClient.contacts.deny(listOf(alice.walletAddress)) } - bobClient.contacts.refreshConsentList() - + runBlocking { + bobClient.contacts.deny(listOf(alice.walletAddress)) + bobClient.contacts.refreshConsentList() + } val isDenied = bobConversation.consentState() == ConsentState.DENIED assertEquals(bobClient.contacts.consentList.entries.size, 1) assertTrue(isDenied) @@ -820,7 +815,7 @@ class ConversationTest { val aliceClient2 = Client().create(aliceWallet) val aliceConversation2 = runBlocking { aliceClient2.conversations.list()[0] } - aliceClient2.contacts.refreshConsentList() + runBlocking { aliceClient2.contacts.refreshConsentList() } // Allow state should sync across clients val isBobAllowed2 = aliceConversation2.consentState() == ConsentState.ALLOWED @@ -843,14 +838,33 @@ class ConversationTest { // Conversations you receive should start as unknown assertTrue(isUnknown) - runBlocking { aliceConversation.send(content = "hey bob") } - aliceClient.contacts.refreshConsentList() + runBlocking { + aliceConversation.send(content = "hey bob") + aliceClient.contacts.refreshConsentList() + } val isNowAllowed = aliceConversation.consentState() == ConsentState.ALLOWED // Conversations you send a message to get marked as allowed assertTrue(isNowAllowed) } + @Test + fun testCanPublishMultipleAddressConsentState() { + runBlocking { + val bobConversation = bobClient.conversations.newConversation(alice.walletAddress) + val caroConversation = + bobClient.conversations.newConversation(fixtures.caro.walletAddress) + bobClient.contacts.refreshConsentList() + assertEquals(bobClient.contacts.consentList.entries.size, 2) + assertTrue(bobConversation.consentState() == ConsentState.ALLOWED) + assertTrue(caroConversation.consentState() == ConsentState.ALLOWED) + bobClient.contacts.deny(listOf(alice.walletAddress, fixtures.caro.walletAddress)) + assertEquals(bobClient.contacts.consentList.entries.size, 2) + assertTrue(bobConversation.consentState() == ConsentState.DENIED) + assertTrue(caroConversation.consentState() == ConsentState.DENIED) + } + } + @Test fun testCanValidateTopicsInsideConversation() { val validId = "sdfsadf095b97a9284dcd82b2274856ccac8a21de57bebe34e7f9eeb855fb21126d3b8f" diff --git a/library/src/main/java/org/xmtp/android/library/Contacts.kt b/library/src/main/java/org/xmtp/android/library/Contacts.kt index 26b7e36d8..a11960556 100644 --- a/library/src/main/java/org/xmtp/android/library/Contacts.kt +++ b/library/src/main/java/org/xmtp/android/library/Contacts.kt @@ -48,8 +48,11 @@ data class ConsentListEntry( get() = "${entryType.name}-$value" } -class ConsentList(val client: Client) { - val entries: MutableMap = mutableMapOf() +class ConsentList( + val client: Client, + val entries: MutableMap = mutableMapOf(), +) { + private var lastFetched: Date? = null private val publicKey = client.privateKeyBundleV1.identityKey.publicKey.secp256K1Uncompressed.bytes private val privateKey = client.privateKeyBundleV1.identityKey.secp256K1.bytes @@ -60,13 +63,17 @@ class ConsentList(val client: Client) { ) @OptIn(ExperimentalUnsignedTypes::class) - suspend fun load(): ConsentList { + suspend fun load(): List { + val newDate = Date() val envelopes = client.apiClient.envelopes( Topic.preferenceList(identifier).description, - Pagination(direction = MessageApiOuterClass.SortDirection.SORT_DIRECTION_ASCENDING), + Pagination( + after = lastFetched, + direction = MessageApiOuterClass.SortDirection.SORT_DIRECTION_ASCENDING + ), ) - val consentList = ConsentList(client) + lastFetched = newDate val preferences: MutableList = mutableListOf() for (envelope in envelopes) { val payload = @@ -79,64 +86,70 @@ class ConsentList(val client: Client) { preferences.add( PrivatePreferencesAction.parseFrom( payload.toUByteArray().toByteArray(), - ), + ) ) } preferences.iterator().forEach { preference -> preference.allowAddress?.walletAddressesList?.forEach { address -> - consentList.allow(address) + allow(address) } preference.denyAddress?.walletAddressesList?.forEach { address -> - consentList.deny(address) + deny(address) } preference.allowGroup?.groupIdsList?.forEach { groupId -> - consentList.allowGroup(groupId.toByteArray()) + allowGroup(groupId.toByteArray()) } preference.denyGroup?.groupIdsList?.forEach { groupId -> - consentList.denyGroup(groupId.toByteArray()) + denyGroup(groupId.toByteArray()) } } - return consentList + return entries.values.toList() } - suspend fun publish(entry: ConsentListEntry) { - val payload = - PrivatePreferencesAction.newBuilder().also { + suspend fun publish(entries: List) { + val payload = PrivatePreferencesAction.newBuilder().also { + entries.forEach { entry -> when (entry.entryType) { ConsentListEntry.EntryType.ADDRESS -> { when (entry.consentType) { ConsentState.ALLOWED -> it.setAllowAddress( - PrivatePreferencesAction.AllowAddress.newBuilder().addWalletAddresses(entry.value), + PrivatePreferencesAction.AllowAddress.newBuilder() + .addWalletAddresses(entry.value), ) ConsentState.DENIED -> it.setDenyAddress( - PrivatePreferencesAction.DenyAddress.newBuilder().addWalletAddresses(entry.value), + PrivatePreferencesAction.DenyAddress.newBuilder() + .addWalletAddresses(entry.value), ) ConsentState.UNKNOWN -> it.clearMessageType() } } + ConsentListEntry.EntryType.GROUP_ID -> { when (entry.consentType) { ConsentState.ALLOWED -> it.setAllowGroup( - PrivatePreferencesAction.AllowGroup.newBuilder().addGroupIds(entry.value.toByteStringUtf8()), + PrivatePreferencesAction.AllowGroup.newBuilder() + .addGroupIds(entry.value.toByteStringUtf8()), ) ConsentState.DENIED -> it.setDenyGroup( - PrivatePreferencesAction.DenyGroup.newBuilder().addGroupIds(entry.value.toByteStringUtf8()), + PrivatePreferencesAction.DenyGroup.newBuilder() + .addGroupIds(entry.value.toByteStringUtf8()), ) ConsentState.UNKNOWN -> it.clearMessageType() } } } - }.build() + } + }.build() val message = uniffi.xmtpv3.userPreferencesEncrypt( @@ -145,40 +158,39 @@ class ConsentList(val client: Client) { payload.toByteArray(), ) - val envelope = - EnvelopeBuilder.buildFromTopic( - Topic.preferenceList(identifier), - Date(), - ByteArray(message.size) { message[it] }, - ) + val envelope = EnvelopeBuilder.buildFromTopic( + Topic.preferenceList(identifier), + Date(), + ByteArray(message.size) { message[it] }, + ) client.publish(listOf(envelope)) } fun allow(address: String): ConsentListEntry { val entry = ConsentListEntry.address(address, ConsentState.ALLOWED) - entries[ConsentListEntry.address(address).key] = entry + entries[entry.key] = entry return entry } fun deny(address: String): ConsentListEntry { val entry = ConsentListEntry.address(address, ConsentState.DENIED) - entries[ConsentListEntry.address(address).key] = entry + entries[entry.key] = entry return entry } fun allowGroup(groupId: ByteArray): ConsentListEntry { val entry = ConsentListEntry.groupId(groupId, ConsentState.ALLOWED) - entries[ConsentListEntry.groupId(groupId).key] = entry + entries[entry.key] = entry return entry } fun denyGroup(groupId: ByteArray): ConsentListEntry { val entry = ConsentListEntry.groupId(groupId, ConsentState.DENIED) - entries[ConsentListEntry.groupId(groupId).key] = entry + entries[entry.key] = entry return entry } @@ -200,38 +212,40 @@ data class Contacts( var client: Client, val knownBundles: MutableMap = mutableMapOf(), val hasIntroduced: MutableMap = mutableMapOf(), + var consentList: ConsentList = ConsentList(client), ) { - var consentList: ConsentList = ConsentList(client) - fun refreshConsentList(): ConsentList { - runBlocking { - consentList = ConsentList(client).load() - } + suspend fun refreshConsentList(): ConsentList { + consentList.load() return consentList } suspend fun allow(addresses: List) { - for (address in addresses) { - ConsentList(client).publish(consentList.allow(address)) + val entries = addresses.map { + consentList.allow(it) } + consentList.publish(entries) } suspend fun deny(addresses: List) { - for (address in addresses) { - ConsentList(client).publish(consentList.deny(address)) + val entries = addresses.map { + consentList.deny(it) } + consentList.publish(entries) } suspend fun allowGroup(groupIds: List) { - for (id in groupIds) { - ConsentList(client).publish(consentList.allowGroup(id)) + val entries = groupIds.map { + consentList.allowGroup(it) } + consentList.publish(entries) } suspend fun denyGroup(groupIds: List) { - for (id in groupIds) { - ConsentList(client).publish(consentList.denyGroup(id)) + val entries = groupIds.map { + consentList.denyGroup(it) } + consentList.publish(entries) } fun isAllowed(address: String): Boolean {