Merge branch 'release/0.16.0' into main

.github/workflows/main.yml

@@ -55,17 +55,10 @@ jobs:
         env:
           ACTIONS_ALLOW_UNSECURE_COMMANDS: 'true'
       - name: Upload airy binary to S3
-        if: startsWith(github.ref, 'refs/heads/release') || startsWith(github.ref, 'refs/heads/main')
+        if: ${{ github.ref == 'refs/heads/develop' || startsWith(github.ref, 'refs/heads/release') || github.ref == 'refs/heads/main' }}
         run: |
           ./scripts/upload-cli-binaries.sh
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-      - name: Upload airy binary to S3 [develop]
-        if: startsWith(github.ref, 'refs/heads/develop')
-        run: |
-          aws s3 cp bazel-bin/cli/airy_linux_bin s3://airy-core-binaries/develop/linux/amd64/airy
-          aws s3 cp bazel-bin/cli/airy_darwin_bin s3://airy-core-binaries/develop/darwin/amd64/airy
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          GITHUB_BRANCH: ${{ github.ref }}

.gitignore

@@ -5,6 +5,8 @@
 .vscode/
 .ijwb/
 
+# config dir for creating a local dev instance
+airy_dev
 
 dist/
 bazel-*

.prettierignore

@@ -1,4 +1,4 @@
-**/*/cli/reference.md
+**/*/cli/usage.md
 **/*/changelog.md
 build/
 bazel-*

README.md

@@ -1,7 +1,7 @@
 <p align="center">
   <img src="https://global-uploads.webflow.com/5e9d5014fb5d85233d05fa23/5ea6ab4327484b79bdb4cea4_airy_primary_rgb.svg" alt="Airy-logo" width="240">
   <div align="center">The open source, fully-featured, production ready</div>
-  <div align="center">Messaging platform</div>
+  <div align="center">Conversational Platform</div>
 </p>
 
 # Airy Core
@@ -13,15 +13,20 @@
 [![License](https://img.shields.io/github/license/airyhq/airy)](https://github.com/airyhq/airy/blob/develop/LICENSE)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](https://github.com/airyhq/airy/projects)
 
-Airy Core is an open source, fully-featured, production ready messaging
+
+---
+
+![Airy_Explainer_Highlevel_Readme](https://user-images.githubusercontent.com/124274/113720584-18a8d500-96ef-11eb-97c3-362eebd6253d.jpeg)
+
+Airy Core is an open source, fully-featured, production ready conversational
 platform. With Airy you can process conversational data from a variety of
 sources:
 
 - **Facebook**
 - **WhatsApp**
 - **Google's Business Messages**
 - **SMS**
-- **Website Chat Plugins**
+- **Website Chat Plugins, like our own open source Live Chat**
 - **Twilio**
 - **Your own conversational channels**
 
@@ -37,16 +42,14 @@ Since Airy's infrastructure is built around Apache Kafka, it can process a large
 amount of conversations and messages simultaneously and stream the relevant
 conversational data to wherever you need it.
 
-Learn more about what we open-sourced in the
-[announcement blog post](https://airy.co/blog/what-we-open-sourced).
-
+---
 ## About Airy
 
 - **What does Airy do? 🚀**
   [Learn more on our Website](https://airy.co/developers)
 
 - **I'm new to Airy 😄**
-  [Get Started with Airy](https://docs.airy.co/)
+  [Get Started with Airy](https://airy.co/docs/core/)
 
 - **I'd like to read the detailed docs 📖**
   [Read The Docs](https://airy.co/docs/core/)
@@ -60,42 +63,49 @@ Learn more about what we open-sourced in the
 - **I have a question ❓**
   [The Airy Community will help](https://airy.co/community)
 
+---
 ## Components
 
+![Airy_Explainer_Components_Readme (1)](https://user-images.githubusercontent.com/12533283/112460661-6de3fe80-8d5f-11eb-8274-8446fbfcf5c8.png)
+
 Airy Core contains the following components:
 
-### 💬 Connectors for all [conversational sources](https://airy.co/docs/core/sources/introduction)
+- 💬 Connectors for all [conversational sources](https://airy.co/docs/core/sources/introduction)
 
 Connect anything from our free open-source [live chat
-plugin](https://airy.co/docs/core/sources/chat-plugin), [Facebook
-Messenger](https://airy.co/docs/core/sources/facebook), [Google's Business
-Messages](https://airy.co/docs/core/sources/google) to your Airy Core. This is
+plugin](https://airy.co/docs/core/sources/chat-plugin) to Facebook
+Messenger & Google's Business Messages to your Airy Core. This is
 all possible through an ingestion platform that heavily relies on [Apache
 Kafka](https://kafka.apache.org) to process incoming webhook data from different
 sources. We make sense of the data and reshape it into source independent
 contacts, conversations, and messages.
 
-### ⚡[APIs](https://airy.co/docs/core/api/introduction) to access your data
+
+- ⚡[APIs](https://airy.co/docs/core/api/introduction) to access your data
 
 An [API](https://airy.co/docs/core/api/introduction) to access conversational
 data with blazing fast HTTP endpoints.
 
-### 🔌[WebSockets](https://airy.co/docs/core/api/websocket) to power real-time applications
+
+- 🔌[WebSockets](https://airy.co/docs/core/api/websocket) to power real-time applications
 
 A [WebSocket server](https://airy.co/docs/core/api/websocket) that allows
 clients to receive near real-time updates about data flowing through the system.
 
-### 🎣[Webhook](https://airy.co/docs/core/api/webhook) to listen to events and participate programmatically in conversations
+
+- 🎣[Webhook](https://airy.co/docs/core/api/webhook) to listen to events and participate programmatically in conversations
 
 A webhook integration server that allows its users to programmatically
 participate in conversations by sending messages (the webhook integration
 exposes events users can "listen" to and react programmatically.)
 
-### 💎[UI: From an inbox to dashboards](https://airy.co/docs/core/apps/ui/introduction)
+
+- 💎[UI: From an inbox to dashboards](https://airy.co/docs/core/apps/ui/introduction)
 
 Not every message can be handled by code, this is why Airy comes with different
 UIs ready for you and your teams to use.
 
+
 ## How to contribute
 
 We welcome (and love) every form of contribution! Good entry points to the

VERSION

@@ -1 +1 @@
-0.15.1
+0.16.0

WORKSPACE

@@ -9,9 +9,9 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
 # Airy Bazel tools
 git_repository(
     name = "com_github_airyhq_bazel_tools",
-    commit = "4141e06b0fab583f98486dbd9eff843cc58edf9f",
+    commit = "777db5e4d099dc960291b1187b0d41e8e444ae77",
     remote = "https://github.com/airyhq/bazel-tools.git",
-    shallow_since = "1616595376 +0100",
+    shallow_since = "1617185101 +0200",
 )
 
 load("@com_github_airyhq_bazel_tools//:repositories.bzl", "airy_bazel_tools_dependencies", "airy_jvm_deps")

backend/api/communication/src/main/java/co/airy/core/api/communication/ConversationsController.java

@@ -5,6 +5,7 @@
 import co.airy.core.api.communication.dto.Conversation;
 import co.airy.core.api.communication.dto.ConversationIndex;
 import co.airy.core.api.communication.dto.LuceneQueryResult;
+import co.airy.core.api.communication.lucene.AiryAnalyzer;
 import co.airy.core.api.communication.lucene.ExtendedQueryParser;
 import co.airy.core.api.communication.lucene.ReadOnlyLuceneStore;
 import co.airy.core.api.communication.payload.ConversationByIdRequestPayload;
@@ -22,7 +23,7 @@
 import co.airy.spring.web.payload.RequestErrorResponsePayload;
 import org.apache.kafka.streams.state.KeyValueIterator;
 import org.apache.kafka.streams.state.ReadOnlyKeyValueStore;
-import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
 import org.apache.lucene.queryparser.classic.ParseException;
 import org.apache.lucene.search.Query;
 import org.springframework.http.HttpStatus;
@@ -32,6 +33,7 @@
 import org.springframework.web.bind.annotation.RestController;
 
 import javax.validation.Valid;
+import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -47,12 +49,12 @@ public class ConversationsController {
     private final Stores stores;
     private final ExtendedQueryParser queryParser;
 
-    ConversationsController(Stores stores) {
+    ConversationsController(Stores stores) throws IOException {
         this.stores = stores;
         this.queryParser = new ExtendedQueryParser(Set.of("unread_count"),
                 Set.of("created_at"),
                 "id",
-                new WhitespaceAnalyzer());
+                AiryAnalyzer.build());
         this.queryParser.setAllowLeadingWildcard(true);
     }
 

backend/api/communication/src/main/java/co/airy/core/api/communication/dto/ConversationIndex.java

@@ -1,12 +1,17 @@
 package co.airy.core.api.communication.dto;
 
+import co.airy.model.metadata.dto.MetadataMap;
+import co.airy.model.metadata.dto.MetadataNode;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.NoArgsConstructor;
+import org.apache.avro.generic.GenericData;
 
 import java.io.Serializable;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 @Data
 @Builder
@@ -21,12 +26,20 @@ public class ConversationIndex implements Serializable {
     private Integer unreadMessageCount;
     private List<String> tagIds;
 
+    @Builder.Default
+    private List<MetadataNode> metadata = new ArrayList<>();
+
     public static ConversationIndex fromConversation(Conversation conversation) {
+        final List<MetadataNode> metadataNodes = conversation.getMetadataMap().values().stream()
+                .map((record) -> new MetadataNode(record.getKey(), record.getValue()))
+                .collect(Collectors.toList());
+
         return ConversationIndex.builder()
                 .id(conversation.getId())
                 .channelId(conversation.getChannelId())
                 .source(conversation.getChannelContainer().getChannel().getSource())
                 .displayName(conversation.getDisplayNameOrDefault())
+                .metadata(metadataNodes)
                 .createdAt(conversation.getCreatedAt())
                 .tagIds(conversation.getTagIds())
                 .unreadMessageCount(conversation.getUnreadMessageCount())

backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/AiryAnalyzer.java

@@ -0,0 +1,19 @@
+package co.airy.core.api.communication.lucene;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.core.LowerCaseFilterFactory;
+import org.apache.lucene.analysis.core.WhitespaceTokenizerFactory;
+import org.apache.lucene.analysis.custom.CustomAnalyzer;
+
+import java.io.IOException;
+
+public class AiryAnalyzer {
+
+    public static Analyzer build() throws IOException {
+        return CustomAnalyzer
+                .builder()
+                .withTokenizer(WhitespaceTokenizerFactory.class)
+                .addTokenFilter(LowerCaseFilterFactory.class)
+                .build();
+    }
+}

backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/DocumentMapper.java

@@ -1,6 +1,7 @@
 package co.airy.core.api.communication.lucene;
 
 import co.airy.core.api.communication.dto.ConversationIndex;
+import co.airy.model.metadata.dto.MetadataNode;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.IntPoint;
@@ -33,6 +34,12 @@ public Document fromConversationIndex(ConversationIndex conversation) {
             document.add(new TextField("tag_ids", tagId, Field.Store.YES));
         }
 
+        for (MetadataNode node : conversation.getMetadata()) {
+            final String key = String.format("metadata.%s", node.getKey());
+            // Index but don't store metadata
+            document.add(new TextField(key, node.getValue(), Field.Store.NO));
+        }
+
         return document;
     }
 

backend/api/communication/src/main/java/co/airy/core/api/communication/lucene/LuceneProvider.java

@@ -33,7 +33,8 @@ public class LuceneProvider implements LuceneStore {
     public LuceneProvider() throws IOException {
         boolean testMode = System.getenv("TEST_TARGET") != null;
         FSDirectory dir = FSDirectory.open(Paths.get(testMode ? System.getenv("TEST_TMPDIR") : "/tmp/lucene"));
-        IndexWriterConfig config = new IndexWriterConfig(new WhitespaceAnalyzer());
+        IndexWriterConfig config = new IndexWriterConfig(AiryAnalyzer.build());
+
         writer = new IndexWriter(dir, config);
         reader = DirectoryReader.open(writer, true, true);
         documentMapper = new DocumentMapper();

backend/api/communication/src/test/java/co/airy/core/api/communication/ConversationsListTest.java

@@ -81,7 +81,7 @@ class ConversationsListTest {
                     Map.of(MetadataKeys.ConversationKeys.TAGS + "." + tagId, "", MetadataKeys.ConversationKeys.TAGS + "." + anotherTagId, ""),
                     1),
             TestConversation.from(conversationIdToFind, defaultChannel, Map.of(MetadataKeys.ConversationKeys.TAGS + "." + tagId, ""), 1),
-            TestConversation.from(UUID.randomUUID().toString(), defaultChannel, 2),
+            TestConversation.from(UUID.randomUUID().toString(), defaultChannel, Map.of("user_data.erp.id", "abc"), 2),
             TestConversation.from(UUID.randomUUID().toString(), defaultChannel, 5)
     );
 
@@ -138,6 +138,12 @@ void canFilterByDisplayName() throws Exception {
         checkConversationsFound(payload, 1);
     }
 
+    @Test
+    void canFilterByDisplayNameIgnoringCasing() throws Exception {
+        String payload = "{\"filters\": \"display_name:" + firstNameToFind.toLowerCase() + "\"}";
+        checkConversationsFound(payload, 1);
+    }
+
     @Test
     void canFilterByTagIds() throws Exception {
         checkConversationsFound("{\"filters\": \"tag_ids:(" + tagId + ")\"}", 2);
@@ -171,6 +177,11 @@ void canFilterForUnknownNames() throws Exception {
         checkConversationsFound("{\"filters\": \"display_name:Ada\"}", 0);
     }
 
+    @Test
+    void canFilterByMetadata() throws Exception {
+        checkConversationsFound("{\"filters\": \"metadata.user_data.erp.id:abc\"}", 1);
+    }
+
     private void checkConversationsFound(String payload, int count) throws InterruptedException {
         retryOnException(
                 () -> webTestHelper.post("/conversations.list", payload, userId)

backend/api/communication/src/test/java/co/airy/core/api/communication/MetadataControllerTest.java

@@ -44,11 +44,19 @@ public class MetadataControllerTest {
     @Autowired
     private WebTestHelper webTestHelper;
 
+    private static final Channel channel = Channel.newBuilder()
+            .setConnectionState(ChannelConnectionState.CONNECTED)
+            .setId(UUID.randomUUID().toString())
+            .setSource("facebook")
+            .setSourceChannelId("ps-id")
+            .build();
+
     @BeforeAll
     static void beforeAll() throws Exception {
         kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, getTopics());
 
         kafkaTestHelper.beforeAll();
+        kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channel.getId().toString(), channel));
     }
 
     @AfterAll
@@ -63,14 +71,7 @@ void beforeEach() throws Exception {
 
     @Test
     void canUpsertMetadata() throws Exception {
-        final Channel channel = Channel.newBuilder()
-                .setConnectionState(ChannelConnectionState.CONNECTED)
-                .setId(UUID.randomUUID().toString())
-                .setSource("facebook")
-                .setSourceChannelId("ps-id")
-                .build();
         final String conversationId = UUID.randomUUID().toString();
-
         kafkaTestHelper.produceRecord(new ProducerRecord<>(applicationCommunicationChannels.name(), channel.getId(), channel));
         final List<ProducerRecord<String, SpecificRecordBase>> producerRecords = generateRecords(conversationId, channel, 1);
         kafkaTestHelper.produceRecords(producerRecords);
@@ -93,4 +94,12 @@ void canUpsertMetadata() throws Exception {
                 "Conversations list metadata is not present"
         );
     }
+
+    @Test
+    void failsOnNonStringFieldValues() throws Exception {
+        webTestHelper.post("/metadata.upsert",
+                "{\"subject\": \"channel\", \"id\": \"" + channel.getId() + "\", \"data\": {\"sentFrom\": 123}}",
+                "user-id")
+                .andExpect(status().isBadRequest());
+    }
 }

backend/model/metadata/src/main/java/co/airy/model/metadata/MetadataObjectMapper.java

@@ -2,6 +2,7 @@
 
 import co.airy.avro.communication.Metadata;
 import co.airy.model.metadata.dto.MetadataMap;
+import co.airy.model.metadata.dto.MetadataNode;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.JsonNode;
@@ -65,13 +66,14 @@ private static void applyMetadata(ObjectNode root, String key, String value) {
     }
 
     private static void setValue(ObjectNode node, String key, String value) {
-        if (key.endsWith("count")) {
+        final MetadataNode metadataNode = new MetadataNode(key, value);
+        if (metadataNode.getValueType().equals(MetadataNode.ValueType.NUMBER)) {
             try {
                 node.put(key, Integer.valueOf(value));
                 return;
             } catch (NumberFormatException expected) {
             }
-        } else if (key.endsWith("content")) {
+        } else if (metadataNode.getValueType().equals(MetadataNode.ValueType.OBJECT)) {
             // This condition allows us to store message content in metadata
             try {
                 node.set(key, objectMapper.readTree(value));