Add Groups to Conversations (#160)

* first pass at all the pieces needed for threading * a few more places * make signing key extend inboxOwner * get it decoding messages * dump the latest v3 code * write a test for creating a v3 client * use created At * write test for creating libxmtp client and confirm it works * move these change to a different branch * dont pass a conversation * fix linter * point to local not dev * feature flag the client creating of libxmtp while in alpha * change to local * fix up the test helper * feat: fix up the example app * fix up the 22 compat issue * Revert "move these change to a different branch" This reverts commit 8998d13. * try and get some tests running * setup local database * have it create correctly * write tests for functionality * test sending * send encoded content * add updates to the v3 bindings * add updates to the v3 bindings * store in a keystore * move to preferences * fix lint * Fix build issues * new libxmtp updates * dump the latest schema * update to the latest client creation flow * get the create working again * use the keystore because its more secure * fix up linter compat again * flaky test * get the tests all passing * get the example working with groups * create a group with two addresses * more tweaks to the example app to get groups working * add streaming messages to groups * a few example UI tweaks * fix the lowercasing issue in the example app * dump the schema again * implement all the conversation functionality * add new codec for membership changes * write tests for it * fix up the tests a bit' * add more tests and group streaming * get the new codec working as expected * add pagination to messages * fix up the library linting issues * fix up flaky test * fix up min sdk version issue again * update the example app * remove the saved wallet stuff from the demo * get groups working again with signer improvements and membership changes * fix linter * remove syncs so the client will need to manage * add pagination to group listing * dont return self for peers and add erroring to new group creation * update the syncing in the tests * remove all the streams work and move to another PR * remove need to lowercase * try some update ci stuff * try the warp runners again * add test on can message * another scenario to lowercase * try removing mls validation * forgot to remove it in one place * fix path * add back validation * try without warp * try warp again * why do I need colima again * try not on mac runner * dump schema and try to fix tests * try ubuntu latest * no lowercasing needed * typo * try warp build again * try the other linux one * try lower version * add linus runners * add one last try for the jni --------- Co-authored-by: Nicholas Molnar <[email protected]>
xmtp · Feb 1, 2024 · e6876ca · e6876ca
1 parent 125cc65
commit e6876ca
Show file tree

Hide file tree

Showing 36 changed files with 1,212 additions and 105 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -24,37 +24,35 @@ jobs:
         uses: gradle/gradle-build-action@v2
       - name: Validate Gradle Wrapper
         uses: gradle/wrapper-validation-action@v1
-      - name: Start local test server
-        run: docker-compose -p xmtp -f dev/local/docker-compose.yml up -d
+      - name: Start Docker containers
+        run: dev/up
       - name: Gradle Run Unit Tests
         run: ./gradlew library:testDebug
-      - name: Stop local test server
-        run: docker-compose -p xmtp -f dev/local/docker-compose.yml down
   library-integration:
     name: Library (Integration Tests)
-    runs-on: macos-latest
+    runs-on: ubuntu-latest
     steps:
-      - name: Checkout project sources
-        uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
+      - name: Checkout
+        uses: actions/checkout@v4
         with:
-          distribution: 'adopt'
-          java-version: '11'
-      - name: Setup Gradle
+          fetch-depth: 0
+      - name: Configure JDK
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: 11
+      - name: Enable KVM group perms
+        run: |
+          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+          sudo udevadm control --reload-rules
+          sudo udevadm trigger --name-match=kvm
+      - name: Gradle cache
         uses: gradle/gradle-build-action@v2
-      - name: Validate Gradle Wrapper
-        uses: gradle/wrapper-validation-action@v1
-      - name: Set up Docker
-        run: brew install docker docker-compose
-      - name: Start Colima
-        run: colima start
-      - name: Start local test server
-        run: docker-compose -p xmtp -f dev/local/docker-compose.yml up -d
+      - name: Start Docker containers
+        run: dev/up
       - name: Gradle Run Integration Tests
         uses: reactivecircus/android-emulator-runner@v2
         with:
           api-level: 29
           script: ./gradlew connectedCheck
-      - name: Stop local test server
-        run: docker-compose -p xmtp -f dev/local/docker-compose.yml down
 
diff --git a/dev/local/compose b/dev/local/compose
@@ -0,0 +1,4 @@
+#!/bin/bash
+set -eou pipefail
+
+docker-compose -f dev/local/docker-compose.yml -p "xmtp-android" "$@"
diff --git a/dev/local/docker-compose.yml b/dev/local/docker-compose.yml
@@ -1,5 +1,5 @@
 services:
-  waku-node:
+  node:
     image: xmtp/node-go:latest
     platform: linux/amd64
     environment:
@@ -8,14 +8,27 @@ services:
       - --store.enable
       - --store.db-connection-string=postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
       - --store.reader-db-connection-string=postgres://postgres:xmtp@db:5432/postgres?sslmode=disable
+      - --mls-store.db-connection-string=postgres://postgres:xmtp@mlsdb:5432/postgres?sslmode=disable
+      - --mls-validation.grpc-address=validation:50051
+      - --api.enable-mls
       - --wait-for-db=30s
       - --api.authn.enable
     ports:
-      - 9001:9001
       - 5555:5555
+      - 5556:5556
     depends_on:
       - db
+
+  validation:
+    image: xmtp/mls-validation-service:latest
+    platform: linux/amd64
+
   db:
     image: postgres:13
     environment:
-      POSTGRES_PASSWORD: xmtp
+      POSTGRES_PASSWORD: xmtp
+
+  mlsdb:
+    image: postgres:13
+    environment:
+      POSTGRES_PASSWORD: xmtp
diff --git a/dev/local/up b/dev/local/up
@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eou pipefail
+script_dir="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+"${script_dir}"/compose pull
+"${script_dir}"/compose up -d --build
diff --git a/dev/up b/dev/up
@@ -0,0 +1,18 @@
+#!/bin/bash
+set -eou pipefail
+
+if [[ "${OSTYPE}" == "darwin"* ]]; then
+    if ! which buf &>/dev/null; then brew install buf; fi
+    if ! which shellcheck &>/dev/null; then brew install shellcheck; fi
+    if ! which markdownlint &>/dev/null; then brew install markdownlint-cli; fi
+    if ! java -version &>/dev/null; then
+        brew install java
+        sudo ln -sfn /opt/homebrew/opt/openjdk/libexec/openjdk.jdk \
+            /Library/Java/JavaVirtualMachines/
+    fi
+    if ! kotlinc -version &>/dev/null; then brew install kotlin; fi
+fi
+
+rustup update
+
+dev/local/up
diff --git a/example/src/main/java/org/xmtp/android/example/ClientManager.kt b/example/src/main/java/org/xmtp/android/example/ClientManager.kt
@@ -1,5 +1,6 @@
 package org.xmtp.android.example
 
+import android.content.Context
 import androidx.annotation.UiThread
 import kotlinx.coroutines.Dispatchers
 import kotlinx.coroutines.GlobalScope
@@ -10,10 +11,21 @@ import org.xmtp.android.library.Client
 import org.xmtp.android.library.ClientOptions
 import org.xmtp.android.library.XMTPEnvironment
 import org.xmtp.android.library.messages.PrivateKeyBundleV1Builder
+import uniffi.xmtpv3.org.xmtp.android.library.codecs.GroupMembershipChangeCodec
 
 object ClientManager {
 
-    val CLIENT_OPTIONS = ClientOptions(api = ClientOptions.Api(XMTPEnvironment.DEV, appVersion = "XMTPAndroidExample/v1.0.0"))
+    fun clientOptions(appContext: Context?): ClientOptions {
+        return ClientOptions(
+            api = ClientOptions.Api(
+                XMTPEnvironment.LOCAL,
+                appVersion = "XMTPAndroidExample/v1.0.0",
+                isSecure = false
+            ),
+            enableAlphaMls = true,
+            appContext = appContext
+        )
+    }
 
     private val _clientState = MutableStateFlow<ClientState>(ClientState.Unknown)
     val clientState: StateFlow<ClientState> = _clientState
@@ -28,13 +40,14 @@ object ClientManager {
         }
 
     @UiThread
-    fun createClient(encodedPrivateKeyData: String) {
+    fun createClient(encodedPrivateKeyData: String, appContext: Context) {
         if (clientState.value is ClientState.Ready) return
         GlobalScope.launch(Dispatchers.IO) {
             try {
                 val v1Bundle =
                     PrivateKeyBundleV1Builder.fromEncodedData(data = encodedPrivateKeyData)
-                _client = Client().buildFrom(v1Bundle, CLIENT_OPTIONS)
+                _client = Client().buildFrom(v1Bundle, clientOptions(appContext))
+                Client.register(codec = GroupMembershipChangeCodec())
                 _clientState.value = ClientState.Ready
             } catch (e: Exception) {
                 _clientState.value = ClientState.Error(e.localizedMessage.orEmpty())

diff --git a/example/src/main/java/org/xmtp/android/example/MainActivity.kt b/example/src/main/java/org/xmtp/android/example/MainActivity.kt
@@ -22,6 +22,7 @@ import org.xmtp.android.example.conversation.ConversationDetailActivity
 import org.xmtp.android.example.conversation.ConversationsAdapter
 import org.xmtp.android.example.conversation.ConversationsClickListener
 import org.xmtp.android.example.conversation.NewConversationBottomSheet
+import org.xmtp.android.example.conversation.NewGroupBottomSheet
 import org.xmtp.android.example.databinding.ActivityMainBinding
 import org.xmtp.android.example.pushnotifications.PushNotificationTokenManager
 import org.xmtp.android.example.utils.KeyUtil
@@ -35,6 +36,7 @@ class MainActivity : AppCompatActivity(),
     private lateinit var accountManager: AccountManager
     private lateinit var adapter: ConversationsAdapter
     private var bottomSheet: NewConversationBottomSheet? = null
+    private var groupBottomSheet: NewGroupBottomSheet? = null
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -48,7 +50,7 @@ class MainActivity : AppCompatActivity(),
             return
         }
 
-        ClientManager.createClient(keys)
+        ClientManager.createClient(keys, this)
 
         binding = ActivityMainBinding.inflate(layoutInflater)
         setContentView(binding.root)
@@ -67,6 +69,10 @@ class MainActivity : AppCompatActivity(),
             openConversationDetail()
         }
 
+        binding.groupFab.setOnClickListener {
+            openGroupDetail()
+        }
+
         lifecycleScope.launch {
             repeatOnLifecycle(Lifecycle.State.STARTED) {
                 ClientManager.clientState.collect(::ensureClientState)
@@ -86,6 +92,7 @@ class MainActivity : AppCompatActivity(),
 
     override fun onDestroy() {
         bottomSheet?.dismiss()
+        groupBottomSheet?.dismiss()
         super.onDestroy()
     }
 
@@ -127,6 +134,7 @@ class MainActivity : AppCompatActivity(),
             is ClientManager.ClientState.Ready -> {
                 viewModel.fetchConversations()
                 binding.fab.visibility = View.VISIBLE
+                binding.groupFab.visibility = View.VISIBLE
             }
             is ClientManager.ClientState.Error -> showError(clientState.message)
             is ClientManager.ClientState.Unknown -> Unit
@@ -193,4 +201,11 @@ class MainActivity : AppCompatActivity(),
             NewConversationBottomSheet.TAG
         )
     }
+    private fun openGroupDetail() {
+        groupBottomSheet = NewGroupBottomSheet.newInstance()
+        groupBottomSheet?.show(
+            supportFragmentManager,
+            NewGroupBottomSheet.TAG
+        )
+    }
 }
diff --git a/example/src/main/java/org/xmtp/android/example/MainViewModel.kt b/example/src/main/java/org/xmtp/android/example/MainViewModel.kt
@@ -42,7 +42,7 @@ class MainViewModel : ViewModel() {
         viewModelScope.launch(Dispatchers.IO) {
             val listItems = mutableListOf<MainListItem>()
             try {
-                val conversations = ClientManager.client.conversations.list()
+                val conversations = ClientManager.client.conversations.list(includeGroups = true)
                 PushNotificationTokenManager.xmtpPush.subscribe(conversations.map { it.topic })
                 listItems.addAll(
                     conversations.map { conversation ->

diff --git a/example/src/main/java/org/xmtp/android/example/connect/ConnectWalletViewModel.kt b/example/src/main/java/org/xmtp/android/example/connect/ConnectWalletViewModel.kt
@@ -1,8 +1,9 @@
 package org.xmtp.android.example.connect
 
+import android.app.Application
 import android.net.Uri
 import androidx.annotation.UiThread
-import androidx.lifecycle.ViewModel
+import androidx.lifecycle.AndroidViewModel
 import androidx.lifecycle.viewModelScope
 import com.walletconnect.wcmodal.client.Modal
 import kotlinx.coroutines.Dispatchers
@@ -20,8 +21,9 @@ import org.xmtp.android.library.Client
 import org.xmtp.android.library.XMTPException
 import org.xmtp.android.library.messages.PrivateKeyBuilder
 import org.xmtp.android.library.messages.PrivateKeyBundleV1Builder
+import uniffi.xmtpv3.org.xmtp.android.library.codecs.GroupMembershipChangeCodec
 
-class ConnectWalletViewModel : ViewModel() {
+class ConnectWalletViewModel(application: Application) : AndroidViewModel(application) {
 
     private val chains: List<ChainSelectionUi> =
         Chains.values().map { it.toChainUiState() }
@@ -84,7 +86,8 @@ class ConnectWalletViewModel : ViewModel() {
             _uiState.value = ConnectUiState.Loading
             try {
                 val wallet = PrivateKeyBuilder()
-                val client = Client().create(wallet, ClientManager.CLIENT_OPTIONS)
+                val client = Client().create(wallet, ClientManager.clientOptions(getApplication()))
+                Client.register(codec = GroupMembershipChangeCodec())
                 _uiState.value = ConnectUiState.Success(
                     wallet.address,
                     PrivateKeyBundleV1Builder.encodeData(client.privateKeyBundleV1)
@@ -108,7 +111,8 @@ class ConnectWalletViewModel : ViewModel() {
                         it.copy(showWallet = true, uri = uri)
                     }
                 }
-                val client = Client().create(wallet, ClientManager.CLIENT_OPTIONS)
+                val client = Client().create(wallet, ClientManager.clientOptions(getApplication()))
+                Client.register(codec = GroupMembershipChangeCodec())
                 _uiState.value = ConnectUiState.Success(
                     wallet.address,
                     PrivateKeyBundleV1Builder.encodeData(client.privateKeyBundleV1)

diff --git a/example/src/main/java/org/xmtp/android/example/conversation/ConversationDetailActivity.kt b/example/src/main/java/org/xmtp/android/example/conversation/ConversationDetailActivity.kt
@@ -54,7 +54,14 @@ class ConversationDetailActivity : AppCompatActivity() {
         setContentView(binding.root)
         setSupportActionBar(binding.toolbar)
         supportActionBar?.setDisplayHomeAsUpEnabled(true)
-        supportActionBar?.subtitle = peerAddress?.truncatedAddress()
+        supportActionBar?.subtitle = if (peerAddress != null && peerAddress!!.contains(",")) {
+            val addresses = peerAddress?.split(",")?.toMutableList()
+            addresses?.joinToString(" & ") {
+                it.truncatedAddress()
+            }
+        } else {
+            peerAddress?.truncatedAddress()
+        }
 
         adapter = MessageAdapter()
         binding.list.layoutManager =
@@ -106,10 +113,12 @@ class ConversationDetailActivity : AppCompatActivity() {
                 finish()
                 true
             }
+
             R.id.copy_address -> {
                 copyWalletAddress()
                 true
             }
+
             else -> super.onOptionsItemSelected(item)
         }
     }
@@ -130,10 +139,12 @@ class ConversationDetailActivity : AppCompatActivity() {
                     adapter.setData(uiState.listItems)
                 }
             }
+
             is ConversationDetailViewModel.UiState.Success -> {
                 binding.refresh.isRefreshing = false
                 adapter.setData(uiState.listItems)
             }
+
             is ConversationDetailViewModel.UiState.Error -> {
                 binding.refresh.isRefreshing = false
                 showError(uiState.message)
@@ -146,10 +157,12 @@ class ConversationDetailActivity : AppCompatActivity() {
             is ConversationDetailViewModel.SendMessageState.Error -> {
                 showError(sendState.message)
             }
+
             ConversationDetailViewModel.SendMessageState.Loading -> {
                 binding.sendButton.isEnabled = false
                 binding.messageEditText.isEnabled = false
             }
+
             ConversationDetailViewModel.SendMessageState.Success -> {
                 binding.messageEditText.text.clear()
                 binding.messageEditText.isEnabled = true

diff --git a/example/src/main/java/org/xmtp/android/example/conversation/ConversationDetailViewModel.kt b/example/src/main/java/org/xmtp/android/example/conversation/ConversationDetailViewModel.kt
@@ -20,6 +20,7 @@ import org.xmtp.android.example.extension.flowWhileShared
 import org.xmtp.android.example.extension.stateFlow
 import org.xmtp.android.library.Conversation
 import org.xmtp.android.library.DecodedMessage
+import org.xmtp.android.library.Group
 
 class ConversationDetailViewModel(private val savedStateHandle: SavedStateHandle) : ViewModel() {
 
@@ -49,9 +50,15 @@ class ConversationDetailViewModel(private val savedStateHandle: SavedStateHandle
             val listItems = mutableListOf<MessageListItem>()
             try {
                 if (conversation == null) {
-                    conversation = ClientManager.client.fetchConversation(conversationTopic)
+                    conversation = ClientManager.client.fetchConversation(
+                        conversationTopic,
+                        includeGroups = true
+                    )
                 }
                 conversation?.let {
+                    if (conversation is Conversation.Group) {
+                        (conversation as Conversation.Group).group.sync()
+                    }
                     listItems.addAll(
                         it.messages().map { message ->
                             MessageListItem.Message(message.id, message)
@@ -69,7 +76,8 @@ class ConversationDetailViewModel(private val savedStateHandle: SavedStateHandle
     val streamMessages: StateFlow<MessageListItem?> =
         stateFlow(viewModelScope, null) { subscriptionCount ->
             if (conversation == null) {
-                conversation = ClientManager.client.fetchConversation(conversationTopic)
+                conversation =
+                    ClientManager.client.fetchConversation(conversationTopic, includeGroups = false)
             }
             if (conversation != null) {
                 conversation!!.streamMessages()