diff --git a/backend/media/BUILD b/backend/media/BUILD index 1a40ef2a06..0d16878012 100644 --- a/backend/media/BUILD +++ b/backend/media/BUILD @@ -7,11 +7,9 @@ app_deps = [ "//backend:base_app", "//backend/model/message", "//backend/model/metadata", - "//lib/java/mapping", + "//lib/java/uuid", "//lib/java/url", - "//lib/java/spring/kafka/core:spring-kafka-core", - "//lib/java/spring/kafka/streams:spring-kafka-streams", - "@maven//:javax_xml_bind_jaxb_api", + "//lib/java/spring/web:spring-web", "@maven//:org_springframework_retry_spring_retry", "@maven//:org_aspectj_aspectjweaver", "@maven//:com_amazonaws_aws_java_sdk_core", diff --git a/backend/media/src/main/java/co/airy/core/media/MediaController.java b/backend/media/src/main/java/co/airy/core/media/MediaController.java new file mode 100644 index 0000000000..3a7e5d73b5 --- /dev/null +++ b/backend/media/src/main/java/co/airy/core/media/MediaController.java @@ -0,0 +1,61 @@ +package co.airy.core.media; + +import co.airy.core.media.services.MediaUpload; +import co.airy.log.AiryLoggerFactory; +import co.airy.spring.web.payload.RequestErrorResponsePayload; +import co.airy.uuid.UUIDv5; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.slf4j.Logger; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +@RestController +public class MediaController { + private static final Logger log = AiryLoggerFactory.getLogger(MediaController.class); + private final MediaUpload mediaUpload; + + public MediaController(MediaUpload mediaUpload) { + this.mediaUpload = mediaUpload; + } + + + @PostMapping(value = "/media.uploadFile", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity mediaUpload(@RequestParam("file") MultipartFile multipartFile) { + final String originalFileName = multipartFile.getOriginalFilename(); + + if (originalFileName == null) { + return ResponseEntity.unprocessableEntity().body(new RequestErrorResponsePayload("Request is missing original file name")); + } + + try { + final InputStream is = multipartFile.getInputStream(); + String fileName = UUIDv5.fromFile(is).toString(); + + final String originalFileExtension = originalFileName.contains(".") ? originalFileName.substring(originalFileName.lastIndexOf(".")) : ""; + fileName = fileName.concat(originalFileExtension); + + return ResponseEntity.ok(new MediaUploadResponsePayload(mediaUpload.uploadMedia(is, fileName))); + } catch (Exception e) { + log.error("Media upload failed:", e); + return ResponseEntity.badRequest().body(new RequestErrorResponsePayload(String.format("Media Upload failed with error: %s", e.getMessage()))); + } + } +} + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +class MediaUploadResponsePayload { + private String mediaUrl; +} + diff --git a/backend/media/src/main/java/co/airy/core/media/MessageMediaResolver.java b/backend/media/src/main/java/co/airy/core/media/MessageMediaResolver.java deleted file mode 100644 index 198f7f985e..0000000000 --- a/backend/media/src/main/java/co/airy/core/media/MessageMediaResolver.java +++ /dev/null @@ -1,111 +0,0 @@ -package co.airy.core.media; - -import co.airy.avro.communication.Message; -import co.airy.avro.communication.Metadata; -import co.airy.core.media.dto.MessageMediaRequest; -import co.airy.core.media.services.MediaUpload; -import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; -import co.airy.log.AiryLoggerFactory; -import co.airy.mapping.ContentMapper; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.DataUrl; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.slf4j.Logger; -import org.springframework.stereotype.Component; - -import javax.xml.bind.DatatypeConverter; -import java.net.MalformedURLException; -import java.net.URL; -import java.nio.charset.StandardCharsets; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static co.airy.model.metadata.MetadataRepository.getId; -import static co.airy.model.metadata.MetadataRepository.newMessageMetadata; - -@Component -public class MessageMediaResolver { - private final Logger log = AiryLoggerFactory.getLogger(MessageMediaResolver.class); - private final String applicationCommunicationMetadata = new ApplicationCommunicationMetadata().name(); - private final KafkaProducer producer; - private final MediaUpload mediaUpload; - private final ContentMapper mapper; - private final ExecutorService executor; - - public MessageMediaResolver(KafkaProducer producer, - MediaUpload mediaUpload, - ContentMapper mapper) { - this.producer = producer; - this.mediaUpload = mediaUpload; - this.mapper = mapper; - this.executor = Executors.newSingleThreadExecutor(); - } - - public void onMessageMediaRequest(String messageId, MessageMediaRequest messageMediaRequest) { - executor.submit(() -> processMessageMediaRequests(messageId, messageMediaRequest)); - } - - private void processMessageMediaRequests(String messageId, MessageMediaRequest messageMediaRequest) { - final Message message = messageMediaRequest.getMessage(); - final Map metadataMap = Optional.ofNullable(messageMediaRequest.getMetadata()).orElse(new HashMap<>()); - - final List contentList = mapper.renderWithDefaultAndLog(message, metadataMap); - - for (Content content : contentList) { - if (!(content instanceof DataUrl)) { - continue; - } - - final String sourceUrl = ((DataUrl) content).getUrl(); - - try { - final URL url = new URL(sourceUrl); - - if (!mediaUpload.isUserStorageUrl(url) && !hasPersistentUrl(metadataMap, sourceUrl)) { - final String persistentUrl = mediaUpload.uploadMedia(url.openStream(), getFileName(sourceUrl)); - - final Metadata metadata = newMessageMetadata(messageId, getMessageKey(sourceUrl), persistentUrl); - storeMetadata(metadata); - } - } catch (ExecutionException | InterruptedException exception) { - throw new RuntimeException(exception); - } catch (MalformedURLException exception) { - // If it's not a URL, this is an error on the source side - log.warn("Source data url field is not a URL", exception); - } catch (Exception exception) { - log.error("Fetching message source content failed {}", messageMediaRequest); - } - } - } - - private void storeMetadata(Metadata metadata) throws ExecutionException, InterruptedException { - final String metadataKey = getId(metadata).toString(); - producer.send(new ProducerRecord<>(applicationCommunicationMetadata, metadataKey, metadata)).get(); - } - - private String getFileName(String sourceUrl) { - try { - final MessageDigest digest = MessageDigest.getInstance("SHA-256"); - final String urlHash = DatatypeConverter.printHexBinary(digest.digest(sourceUrl.getBytes(StandardCharsets.UTF_8))); - return String.format("data_%s", urlHash.toLowerCase()); - } catch (NoSuchAlgorithmException e) { - throw new RuntimeException(e); - } - } - - private boolean hasPersistentUrl(Map metadataMap, String sourceUrl) { - return metadataMap.containsKey(getMessageKey(sourceUrl)); - } - - private String getMessageKey(String sourceUrl) { - return String.format("data_%s", sourceUrl); - } -} diff --git a/backend/media/src/main/java/co/airy/core/media/MetadataResolver.java b/backend/media/src/main/java/co/airy/core/media/MetadataResolver.java deleted file mode 100644 index d5c304da22..0000000000 --- a/backend/media/src/main/java/co/airy/core/media/MetadataResolver.java +++ /dev/null @@ -1,96 +0,0 @@ -package co.airy.core.media; - -import co.airy.avro.communication.Metadata; -import co.airy.core.media.services.MediaUpload; -import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; -import co.airy.log.AiryLoggerFactory; -import co.airy.model.metadata.MetadataKeys; -import co.airy.model.metadata.Subject; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.slf4j.Logger; -import org.springframework.stereotype.Component; - -import java.net.URL; -import java.time.Instant; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import static co.airy.model.metadata.MetadataRepository.getId; -import static co.airy.model.metadata.MetadataRepository.getSubject; -import static co.airy.model.metadata.MetadataRepository.isConversationMetadata; - -@Component -public class MetadataResolver { - private final Logger log = AiryLoggerFactory.getLogger(MetadataResolver.class); - private final String applicationCommunicationMetadata = new ApplicationCommunicationMetadata().name(); - private final KafkaProducer producer; - private final MediaUpload mediaUpload; - private final ExecutorService executor; - - public MetadataResolver( - KafkaProducer producer, - MediaUpload mediaUpload) { - this.producer = producer; - this.mediaUpload = mediaUpload; - this.executor = Executors.newSingleThreadExecutor(); - } - - public boolean shouldResolve(Metadata metadata) { - if (metadata == null) { - return false; - } - - URL dataUrl; - - try { - dataUrl = new URL(metadata.getValue()); - } catch (Exception ignored) { - return false; - } - - return isConversationMetadata(metadata) - && metadata.getKey().equals(MetadataKeys.ConversationKeys.Contact.AVATAR_URL) - && !mediaUpload.isUserStorageUrl(dataUrl); - } - - public void onMetadata(Metadata metadata) { - executor.submit(() -> processMetadataMediaRequest(metadata)); - } - - public void processMetadataMediaRequest(Metadata metadata) { - URL dataUrl; - - try { - dataUrl = new URL(metadata.getValue()); - } catch (Exception exception) { - log.error("Metadata value not a valid url despite filtering {}", metadata, exception); - return; - } - - final String resolvedKey = metadata.getKey() + ".resolved"; - final Subject subject = getSubject(metadata); - final String fileName = String.format("%s/%s", subject.getIdentifier(), resolvedKey); - - try { - final String userStorageUrl = mediaUpload.uploadMedia(dataUrl.openStream(), fileName); - - storeMetadata(Metadata.newBuilder() - .setSubject(subject.toString()) - .setKey(resolvedKey) - .setValue(userStorageUrl) - .setTimestamp(Instant.now().toEpochMilli()) - .build()); - } catch (ExecutionException | InterruptedException exception) { - throw new RuntimeException(exception); - } catch (Exception exception) { - log.error("Failed to upload metadata data url {}", metadata, exception); - } - } - - private void storeMetadata(Metadata metadata) throws ExecutionException, InterruptedException { - final String metadataKey = getId(metadata).toString(); - producer.send(new ProducerRecord<>(applicationCommunicationMetadata, metadataKey, metadata)).get(); - } -} diff --git a/backend/media/src/main/java/co/airy/core/media/Stores.java b/backend/media/src/main/java/co/airy/core/media/Stores.java deleted file mode 100644 index f867ab5def..0000000000 --- a/backend/media/src/main/java/co/airy/core/media/Stores.java +++ /dev/null @@ -1,87 +0,0 @@ -package co.airy.core.media; - -import co.airy.avro.communication.Message; -import co.airy.avro.communication.Metadata; -import co.airy.core.media.dto.MessageMediaRequest; -import co.airy.kafka.schema.application.ApplicationCommunicationMessages; -import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; -import co.airy.kafka.streams.KafkaStreamsWrapper; -import co.airy.log.AiryLoggerFactory; -import org.apache.kafka.streams.KafkaStreams; -import org.apache.kafka.streams.KeyValue; -import org.apache.kafka.streams.StreamsBuilder; -import org.apache.kafka.streams.kstream.KStream; -import org.apache.kafka.streams.kstream.KTable; -import org.slf4j.Logger; -import org.springframework.beans.factory.DisposableBean; -import org.springframework.boot.context.event.ApplicationStartedEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.Map; - -import static co.airy.model.message.MessageRepository.isNewMessage; -import static co.airy.model.metadata.MetadataRepository.getSubject; -import static co.airy.model.metadata.MetadataRepository.isMessageMetadata; - -@Component -public class Stores implements ApplicationListener, DisposableBean { - private final Logger log = AiryLoggerFactory.getLogger(Stores.class); - - private static final String appId = "media.Resolver"; - private final KafkaStreamsWrapper streams; - private final MetadataResolver metadataResolver; - private final MessageMediaResolver messageResolver; - - public Stores(KafkaStreamsWrapper streams, - MetadataResolver metadataResolver, - MessageMediaResolver messageResolver) { - this.streams = streams; - this.metadataResolver = metadataResolver; - this.messageResolver = messageResolver; - } - - @Override - public void onApplicationEvent(ApplicationStartedEvent event) { - final StreamsBuilder builder = new StreamsBuilder(); - - final KStream metadataTable = builder.stream(new ApplicationCommunicationMetadata().name()); - - final KTable> messageMetadataTable = metadataTable.toTable() - .filter((metadataId, metadata) -> isMessageMetadata(metadata)) - .groupBy((metadataId, metadata) -> KeyValue.pair(getSubject(metadata).getIdentifier(), metadata)) - .aggregate(HashMap::new, (metadataId, metadata, metadataMap) -> { - metadataMap.put(metadata.getKey(), metadata.getValue()); - return metadataMap; - }, (metadataId, metadata, metadataMap) -> { - metadataMap.remove(metadata.getKey()); - return metadataMap; - }); - - metadataTable - .filter((metadataId, metadata) -> metadataResolver.shouldResolve(metadata)) - .foreach((metadataId, metadata) -> metadataResolver.onMetadata(metadata)); - - builder.stream(new ApplicationCommunicationMessages().name()) - // Since the message content is immutable we only have to fetch - // the media for new messages - .filter((messageId, message) -> isNewMessage(message)) - .leftJoin(messageMetadataTable, MessageMediaRequest::new) - .foreach(messageResolver::onMessageMediaRequest); - - streams.start(builder.build(), appId); - } - - @Override - public void destroy() { - if (streams != null) { - streams.close(); - } - } - - // visible for testing - KafkaStreams.State getStreamState() { - return streams.state(); - } -} diff --git a/backend/media/src/main/java/co/airy/core/media/config/AwsConfig.java b/backend/media/src/main/java/co/airy/core/media/config/AwsConfig.java index 6dc2388204..7d1113fbc9 100644 --- a/backend/media/src/main/java/co/airy/core/media/config/AwsConfig.java +++ b/backend/media/src/main/java/co/airy/core/media/config/AwsConfig.java @@ -13,9 +13,9 @@ public class AwsConfig { @Bean - public AmazonS3 amazonS3Client(@Value("${storage.s3.key}") final String mediaS3Key, - @Value("${storage.s3.secret}") final String mediaS3Secret, - @Value("${storage.s3.region}") final String region) { + public AmazonS3 amazonS3Client(@Value("${s3.key}") final String mediaS3Key, + @Value("${s3.secret}") final String mediaS3Secret, + @Value("${s3.region}") final String region) { AWSCredentials credentials = new BasicAWSCredentials( mediaS3Key, mediaS3Secret diff --git a/backend/media/src/main/java/co/airy/core/media/dto/MessageMediaRequest.java b/backend/media/src/main/java/co/airy/core/media/dto/MessageMediaRequest.java deleted file mode 100644 index 2e3a55b4d8..0000000000 --- a/backend/media/src/main/java/co/airy/core/media/dto/MessageMediaRequest.java +++ /dev/null @@ -1,17 +0,0 @@ -package co.airy.core.media.dto; - -import co.airy.avro.communication.Message; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.io.Serializable; -import java.util.Map; - -@Data -@NoArgsConstructor -@AllArgsConstructor -public class MessageMediaRequest implements Serializable { - private Message message; - private Map metadata; -} diff --git a/backend/media/src/main/java/co/airy/core/media/services/MediaUpload.java b/backend/media/src/main/java/co/airy/core/media/services/MediaUpload.java index 9ab3fa6c27..24ca272099 100644 --- a/backend/media/src/main/java/co/airy/core/media/services/MediaUpload.java +++ b/backend/media/src/main/java/co/airy/core/media/services/MediaUpload.java @@ -20,23 +20,21 @@ public class MediaUpload { private final AmazonS3 amazonS3Client; private final String bucket; + private final String path; private final URL host; public MediaUpload(AmazonS3 amazonS3Client, - @Value("${storage.s3.bucket}") String bucket, - @Value("${storage.s3.path}") String path) throws MalformedURLException { + @Value("${s3.bucket}") String bucket, + @Value("${s3.path}") String path) throws MalformedURLException { this.amazonS3Client = amazonS3Client; this.bucket = bucket; - URL bucketHost = new URL(String.format("https://%s.s3.amazonaws.com/", bucket)); - this.host = new URL(bucketHost, path); - } - - public boolean isUserStorageUrl(URL dataUrl) { - return dataUrl.getHost().equals(host.getHost()); + this.path = appendSlash(path); + this.host = new URL(String.format("https://%s.s3.amazonaws.com/", bucket)); } @Retryable - public String uploadMedia(final InputStream is, final String fileName) throws Exception { + public String uploadMedia(final InputStream is, String fileName) throws Exception { + fileName = path + fileName; final String contentType = resolveContentType(fileName, is); final ObjectMetadata objectMetadata = new ObjectMetadata(); @@ -57,4 +55,8 @@ private String resolveContentType(final String fileName, final InputStream is) t final String contentType = URLConnection.guessContentTypeFromStream(is); return contentType == null ? URLConnection.guessContentTypeFromName(fileName) : contentType; } + + private String appendSlash(String path) { + return !path.endsWith("/") ? path + "/" : path; + } } diff --git a/backend/media/src/main/resources/application.properties b/backend/media/src/main/resources/application.properties index d7e3c82c0f..81e8559b1c 100644 --- a/backend/media/src/main/resources/application.properties +++ b/backend/media/src/main/resources/application.properties @@ -1,10 +1,13 @@ +spring.servlet.multipart.max-file-size=512MB +spring.servlet.multipart.max-request-size=512MB + kafka.brokers=${KAFKA_BROKERS} kafka.schema-registry-url=${KAFKA_SCHEMA_REGISTRY_URL} kafka.suppress-interval-ms=${KAFKA_SUPPRESS_INTERVAL_MS:3000} -storage.s3.key=${STORAGE_S3_KEY} -storage.s3.secret=${STORAGE_S3_SECRET} -storage.s3.bucket=${STORAGE_S3_BUCKET} -storage.s3.region=${STORAGE_S3_REGION} -storage.s3.path=${STORAGE_S3_PATH:/} +s3.key=${s3Key} +s3.secret=${s3Secret} +s3.bucket=${s3Bucket} +s3.region=${s3Region} +s3.path=${s3Path:/} diff --git a/backend/media/src/test/java/co/airy/core/media/MediaControllerTest.java b/backend/media/src/test/java/co/airy/core/media/MediaControllerTest.java new file mode 100644 index 0000000000..99146d24f0 --- /dev/null +++ b/backend/media/src/test/java/co/airy/core/media/MediaControllerTest.java @@ -0,0 +1,97 @@ +package co.airy.core.media; + +import co.airy.core.media.services.MediaUpload; +import co.airy.kafka.test.KafkaTestHelper; +import co.airy.kafka.test.junit.SharedKafkaTestResource; +import co.airy.spring.core.AirySpringBootApplication; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.PutObjectRequest; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMultipartHttpServletRequestBuilder; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; + +import java.io.InputStream; + +import static co.airy.test.Timing.retryOnException; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest(classes = AirySpringBootApplication.class) +@TestPropertySource(value = "classpath:test.properties") +@AutoConfigureMockMvc +@ExtendWith(SpringExtension.class) +public class MediaControllerTest { + @RegisterExtension + public static final SharedKafkaTestResource sharedKafkaTestResource = new SharedKafkaTestResource(); + private static KafkaTestHelper kafkaTestHelper; + + @BeforeAll + static void beforeAll() throws Exception { + kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource); + kafkaTestHelper.beforeAll(); + } + + @AfterAll + static void afterAll() throws Exception { + kafkaTestHelper.afterAll(); + } + + + MediaUpload mediaUpload; + + @MockBean + private AmazonS3 amazonS3; + + @Autowired + private MockMvc mvc; + + @Value("${s3.bucket}") + private String bucket; + + @Value("${s3.path}") + private String path; + + @BeforeEach + void beforeEach() throws Exception { + MockitoAnnotations.openMocks(this); + mediaUpload = new MediaUpload(amazonS3, bucket, path); + } + + + @Test + void shouldCallS3ToUploadImageDirect() throws Exception { + final InputStream is = getClass().getResourceAsStream("giphy.gif"); + + MockMultipartFile mockMultipartFile = new MockMultipartFile("file", "giphy.gif", + MediaType.MULTIPART_FORM_DATA_VALUE, is); + + MockMultipartHttpServletRequestBuilder builder = + (MockMultipartHttpServletRequestBuilder) MockMvcRequestBuilders.multipart("/media.uploadFile") + .file(mockMultipartFile) + .contentType(MediaType.MULTIPART_FORM_DATA_VALUE); + + retryOnException( + () -> { + mvc.perform(builder).andExpect(status().isOk()); + Mockito.verify(amazonS3, Mockito.times(1)).putObject(Mockito.any(PutObjectRequest.class)); + }, + "Something went wrong while uploading an image" + ); + } +} diff --git a/backend/media/src/test/java/co/airy/core/media/MessagesTest.java b/backend/media/src/test/java/co/airy/core/media/MessagesTest.java deleted file mode 100644 index c6a9c00534..0000000000 --- a/backend/media/src/test/java/co/airy/core/media/MessagesTest.java +++ /dev/null @@ -1,148 +0,0 @@ -package co.airy.core.media; - -import co.airy.avro.communication.DeliveryState; -import co.airy.avro.communication.Message; -import co.airy.avro.communication.Metadata; -import co.airy.core.media.services.MediaUpload; -import co.airy.kafka.schema.application.ApplicationCommunicationMessages; -import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; -import co.airy.kafka.test.KafkaTestHelper; -import co.airy.kafka.test.junit.SharedKafkaTestResource; -import co.airy.mapping.ContentMapper; -import co.airy.mapping.model.Audio; -import co.airy.spring.core.AirySpringBootApplication; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.hamcrest.core.StringEndsWith; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.time.Instant; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - -import static co.airy.test.Timing.retryOnException; -import static org.apache.kafka.streams.KafkaStreams.State.RUNNING; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@SpringBootTest(classes = AirySpringBootApplication.class) -@TestPropertySource(value = "classpath:test.properties") -@ExtendWith(SpringExtension.class) -public class MessagesTest { - @RegisterExtension - public static final SharedKafkaTestResource sharedKafkaTestResource = new SharedKafkaTestResource(); - private static KafkaTestHelper kafkaTestHelper; - private static final ApplicationCommunicationMessages applicationCommunicationMessages = new ApplicationCommunicationMessages(); - private static final ApplicationCommunicationMetadata applicationCommunicationMetadata = new ApplicationCommunicationMetadata(); - - @BeforeAll - static void beforeAll() throws Exception { - kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, - applicationCommunicationMetadata, - applicationCommunicationMessages - ); - - kafkaTestHelper.beforeAll(); - } - - @AfterAll - static void afterAll() throws Exception { - kafkaTestHelper.afterAll(); - } - - @Autowired - Stores stores; - - MediaUpload mediaUpload; - - @MockBean - private AmazonS3 amazonS3; - - @MockBean - private ContentMapper mapper; - - @Value("${storage.s3.bucket}") - private String bucket; - - @Value("${storage.s3.path}") - private String path; - - @BeforeEach - void beforeEach() throws Exception { - MockitoAnnotations.openMocks(this); - mediaUpload = new MediaUpload(amazonS3, bucket, path); - retryOnException(() -> assertEquals(stores.getStreamState(), RUNNING), "Failed to reach RUNNING state."); - } - - @Test - void storesMessageUrlsWithRetries() throws Exception { - final String originalUrl = "https://picsum.photos/1/1"; - final String urlHash = "64ff04d5ca0ad5951e64e3669c3dbd9159675e177a2ba237bf334495f4778da5"; - final String messageId = UUID.randomUUID().toString(); - - final String expectedUrl = String.format("https://%s.s3.amazonaws.com%sdata_%s", - bucket, path, urlHash); - - final ArgumentCaptor s3PutCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); - when(mapper.renderWithDefaultAndLog(Mockito.any(), Mockito.any())).thenReturn(List.of(new Audio(originalUrl))); - - // Simulate a failure to trigger one retry - when(amazonS3.putObject(s3PutCaptor.capture())) - .thenThrow(AmazonServiceException.class) - .thenReturn(new PutObjectResult()); - - kafkaTestHelper.produceRecord( - new ProducerRecord<>(applicationCommunicationMessages.name(), messageId, - Message.newBuilder() - .setId(messageId) - .setSource("fakesource") - .setSentAt(Instant.now().toEpochMilli()) - .setUpdatedAt(null) - .setSenderId("sourceConversationId") - .setIsFromContact(true) - .setDeliveryState(DeliveryState.DELIVERED) - .setConversationId("conversationId") - .setChannelId("channelId") - .setContent("mocked") - .build() - )); - - TimeUnit.SECONDS.sleep(10); - - List> metadataRecords = kafkaTestHelper.consumeRecords(2, applicationCommunicationMetadata.name()); - Metadata metadata = metadataRecords.stream() - .filter((record) -> record.value().getKey().equals(String.format("data_%s", originalUrl))) - .findFirst().get().value(); - - assertThat(metadata.getValue(), equalTo(expectedUrl)); - - verify(amazonS3, times(2)).putObject(Mockito.any(PutObjectRequest.class)); - final PutObjectRequest putObjectRequest = s3PutCaptor.getValue(); - assertThat(putObjectRequest.getBucketName(), equalTo(bucket)); - // The filename we wrote to the metadata has to match the file key we write to S3 - assertThat(metadata.getValue(), StringEndsWith.endsWith(putObjectRequest.getKey())); - } -} diff --git a/backend/media/src/test/java/co/airy/core/media/MetadataTest.java b/backend/media/src/test/java/co/airy/core/media/MetadataTest.java deleted file mode 100644 index 60bc60a316..0000000000 --- a/backend/media/src/test/java/co/airy/core/media/MetadataTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package co.airy.core.media; - -import co.airy.avro.communication.Metadata; -import co.airy.core.media.services.MediaUpload; -import co.airy.kafka.schema.application.ApplicationCommunicationMessages; -import co.airy.kafka.schema.application.ApplicationCommunicationMetadata; -import co.airy.kafka.test.KafkaTestHelper; -import co.airy.kafka.test.junit.SharedKafkaTestResource; -import co.airy.model.metadata.MetadataKeys; -import co.airy.spring.core.AirySpringBootApplication; -import com.amazonaws.AmazonServiceException; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.PutObjectResult; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.extension.RegisterExtension; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import org.mockito.MockitoAnnotations; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.MockBean; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.List; -import java.util.UUID; - -import static co.airy.model.metadata.MetadataRepository.newConversationMetadata; -import static co.airy.test.Timing.retryOnException; -import static org.apache.kafka.streams.KafkaStreams.State.RUNNING; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@SpringBootTest(classes = AirySpringBootApplication.class) -@TestPropertySource(value = "classpath:test.properties") -@ExtendWith(SpringExtension.class) -public class MetadataTest { - @RegisterExtension - public static final SharedKafkaTestResource sharedKafkaTestResource = new SharedKafkaTestResource(); - private static KafkaTestHelper kafkaTestHelper; - private static final ApplicationCommunicationMessages applicationCommunicationMessages = new ApplicationCommunicationMessages(); - private static final ApplicationCommunicationMetadata applicationCommunicationMetadata = new ApplicationCommunicationMetadata(); - - @BeforeAll - static void beforeAll() throws Exception { - kafkaTestHelper = new KafkaTestHelper(sharedKafkaTestResource, - applicationCommunicationMetadata, - applicationCommunicationMessages - ); - - kafkaTestHelper.beforeAll(); - } - - @AfterAll - static void afterAll() throws Exception { - kafkaTestHelper.afterAll(); - } - - @Autowired - Stores stores; - - @Autowired - - MediaUpload mediaUpload; - - @MockBean - private AmazonS3 amazonS3; - - @Value("${storage.s3.bucket}") - private String bucket; - - @Value("${storage.s3.path}") - private String path; - - @BeforeEach - void beforeEach() throws Exception { - MockitoAnnotations.openMocks(this); - mediaUpload = new MediaUpload(amazonS3, bucket, path); - retryOnException(() -> assertEquals(stores.getStreamState(), RUNNING), "Failed to reach RUNNING state."); - } - - @Test - void storesMetadataUrlsWithRetries() throws Exception { - final String conversationId = UUID.randomUUID().toString(); - final String originalUrl = "https://picsum.photos/1/1"; - final String metadataId = UUID.randomUUID().toString(); - final String expectedUrl = String.format("https://%s.s3.amazonaws.com%s%s/%s.resolved", - bucket, - path, - conversationId, - MetadataKeys.ConversationKeys.Contact.AVATAR_URL); - - final ArgumentCaptor s3PutCaptor = ArgumentCaptor.forClass(PutObjectRequest.class); - - // Simulate a failure to trigger one retry - when(amazonS3.putObject(s3PutCaptor.capture())) - .thenThrow(AmazonServiceException.class) - .thenReturn(new PutObjectResult()); - - kafkaTestHelper.produceRecord( - new ProducerRecord<>(applicationCommunicationMetadata.name(), metadataId, - newConversationMetadata(conversationId, - MetadataKeys.ConversationKeys.Contact.AVATAR_URL, - originalUrl) - )); - - List> metadataRecords = kafkaTestHelper.consumeRecords(2, applicationCommunicationMetadata.name()); - Metadata metadata = metadataRecords.stream() - .filter((record) -> !record.key().equals(metadataId)) - .findFirst().get().value(); - - assertThat(metadata.getValue(), equalTo(expectedUrl)); - - verify(amazonS3, times(2)).putObject(Mockito.any(PutObjectRequest.class)); - final PutObjectRequest putObjectRequest = s3PutCaptor.getValue(); - assertThat(putObjectRequest.getBucketName(), equalTo(bucket)); - assertThat(putObjectRequest.getKey(), - equalTo(String.format("%s/%s", conversationId, MetadataKeys.ConversationKeys.Contact.AVATAR_URL + ".resolved"))); - } -} diff --git a/backend/media/src/test/resources/giphy.gif b/backend/media/src/test/resources/giphy.gif new file mode 100644 index 0000000000..aac3cffd2b Binary files /dev/null and b/backend/media/src/test/resources/giphy.gif differ diff --git a/backend/media/src/test/resources/test.properties b/backend/media/src/test/resources/test.properties index 86dde47a14..1a1f793e18 100644 --- a/backend/media/src/test/resources/test.properties +++ b/backend/media/src/test/resources/test.properties @@ -1,10 +1,8 @@ kafka.cleanup=true -kafka.cache.max.bytes=0 kafka.commit-interval-ms=100 -kafka.suppress-interval-ms=0 -storage.s3.key=no -storage.s3.secret=no -storage.s3.bucket=mybucket -storage.s3.region=us-east-1 -storage.s3.path=/core-media/ +s3.key=no +s3.secret=no +s3.bucket=mybucket +s3.region=us-east-1 +s3.path=/core-media/ diff --git a/docs/docs/api/endpoints/attachments.md b/docs/docs/api/endpoints/attachments.md new file mode 100644 index 0000000000..9198f4f8f1 --- /dev/null +++ b/docs/docs/api/endpoints/attachments.md @@ -0,0 +1,30 @@ +--- +title: Attachments +sidebar_label: Attachments +--- + +Many sources allow for sending and receiving attachments such as files, videos, images or audio recordings. In order +to persist media you need to enable the media service by providing a storage option in your [airy.yaml config](getting-started/installation/configuration.md). + +## Upload a file + +`POST /media.uploadFile` + +Expects a multi-part form upload including the original filename + +**Sample curl** + +```shell script +curl http://airy.core/media.uploadFile \ +-X POST \ +-H "Content-Type: multipart/form-data" \ +--form file=@test_image.jpg +``` + +**Sample response** + +```json5 +{ + "media_url": "http://your-storage-provider.com/path/uuid.jpg" +} +``` diff --git a/docs/docs/getting-started/installation/configuration.md b/docs/docs/getting-started/installation/configuration.md index f4943edc1c..91fee27b6c 100644 --- a/docs/docs/getting-started/installation/configuration.md +++ b/docs/docs/getting-started/installation/configuration.md @@ -81,7 +81,7 @@ cluster and Redis. - `maxBackoff` set this to the maximum number of seconds the webhook should wait between retries with exponential backoff - `media` - - `resolver` + - `storage` - `s3Key` set this to your AWS S3 access key id - `s3Secret` set this to your AWS S3 secret access key - `s3Bucket` set this to your AWS S3 bucket diff --git a/infrastructure/helm-chart/charts/apps/charts/media/charts/resolver/templates/deployment.yaml b/infrastructure/helm-chart/charts/apps/charts/media/charts/resolver/templates/deployment.yaml index c12e2d10bc..aeb9eb3f4e 100644 --- a/infrastructure/helm-chart/charts/apps/charts/media/charts/resolver/templates/deployment.yaml +++ b/infrastructure/helm-chart/charts/apps/charts/media/charts/resolver/templates/deployment.yaml @@ -28,6 +28,11 @@ spec: - name: app image: "{{ .Values.global.kubernetes.containerRegistry}}/{{ .Values.image }}:{{ .Values.global.kubernetes.appImageTag }}" imagePullPolicy: Always + envFrom: + - configMapRef: + name: security + - configMapRef: + name: "media-storage" env: - name: KAFKA_BROKERS valueFrom: @@ -44,31 +49,6 @@ spec: configMapKeyRef: name: kafka-config key: KAFKA_COMMIT_INTERVAL_MS - - name: STORAGE_S3_KEY - valueFrom: - configMapKeyRef: - name: "{{ .Values.component }}" - key: s3Key - - name: STORAGE_S3_SECRET - valueFrom: - configMapKeyRef: - name: "{{ .Values.component }}" - key: s3Secret - - name: STORAGE_S3_BUCKET - valueFrom: - configMapKeyRef: - name: "{{ .Values.component }}" - key: s3Bucket - - name: STORAGE_S3_REGION - valueFrom: - configMapKeyRef: - name: "{{ .Values.component }}" - key: s3Region - - name: STORAGE_S3_PATH - valueFrom: - configMapKeyRef: - name: "{{ .Values.component }}" - key: s3Path livenessProbe: httpGet: path: /actuator/health diff --git a/infrastructure/helm-chart/templates/ingress.yaml b/infrastructure/helm-chart/templates/ingress.yaml index 62fa979768..823343367c 100644 --- a/infrastructure/helm-chart/templates/ingress.yaml +++ b/infrastructure/helm-chart/templates/ingress.yaml @@ -99,6 +99,13 @@ spec: name: api-communication port: number: 80 + - path: /media.uploadFile + pathType: Prefix + backend: + service: + name: media-resolver + port: + number: 80 - path: /login pathType: Prefix backend: diff --git a/lib/java/mapping/BUILD b/lib/java/mapping/BUILD deleted file mode 100644 index e83e7d9243..0000000000 --- a/lib/java/mapping/BUILD +++ /dev/null @@ -1,34 +0,0 @@ -load("@com_github_airyhq_bazel_tools//lint:buildifier.bzl", "check_pkg") -load("//tools/build:java_library.bzl", "custom_java_library") -load("//tools/build:junit5.bzl", "junit5") - -lib_deps = [ - "//:lombok", - "//:spring", - "//:jackson", - "//lib/java/log", - "//backend/model/message:message", - "@maven//:javax_validation_validation_api", -] - -custom_java_library( - name = "mapping", - srcs = glob(["src/main/java/co/airy/mapping/**/*.java"]), - visibility = ["//visibility:public"], - deps = lib_deps, -) - -[ - junit5( - file = file, - resources = glob(["src/test/resources/**/*"]), - deps = lib_deps + [ - ":mapping", - "//backend:base_test", - "//lib/java/spring/core:spring-core", - ], - ) - for file in glob(["src/test/java/**/*Test.java"]) -] - -check_pkg(name = "buildifier") diff --git a/lib/java/mapping/README.md b/lib/java/mapping/README.md deleted file mode 100644 index 11e6913c0a..0000000000 --- a/lib/java/mapping/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# Mapping library - -This library is responsible for mapping source ingestion message content -strings to a payload structure that can be sent via network APIs. - -## Motivation - -We learned that the safest way of handling ingestion message data from -`n` sources with up to `m` content schemata each is to keep the data "as is" -and map the content to a usable schema dynamically. - -This gives us the ability to address mapping bugs by fixing code (cheap) -rather than fixing streaming data (expensive). diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/ContentMapper.java b/lib/java/mapping/src/main/java/co/airy/mapping/ContentMapper.java deleted file mode 100644 index db4d9af853..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/ContentMapper.java +++ /dev/null @@ -1,76 +0,0 @@ -package co.airy.mapping; - -import co.airy.avro.communication.Message; -import co.airy.log.AiryLoggerFactory; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.DataUrl; -import co.airy.mapping.model.Text; -import org.slf4j.Logger; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Component -public class ContentMapper { - private final Logger log = AiryLoggerFactory.getLogger(ContentMapper.class); - private final Map mappers = new HashMap<>(); - private final OutboundMapper outboundMapper; - - public ContentMapper(List sourceMappers, - OutboundMapper outboundMapper) { - for (SourceMapper mapper : sourceMappers) { - mapper.getIdentifiers().forEach((identifier) -> mappers.put(identifier, mapper)); - } - this.outboundMapper = outboundMapper; - } - - public List render(Message message) throws Exception { - return render(message, Map.of()); - } - - public List render(Message message, Map metadata) throws Exception { - if (!message.getIsFromContact()) { - return outboundMapper.render(message.getContent()); - } - - final SourceMapper sourceMapper = mappers.get(message.getSource()); - if (sourceMapper == null) { - throw new Exception("can not map message source " + message.getSource()); - } - - final List contentList = sourceMapper.render(message.getContent()); - - /* - * Source content can contain data urls that expire. Therefore we upload this data - * to a user provided storage and dynamically replace the urls here in the mapper - */ - for (Content content : contentList) { - // Replace the url if the metadata contains a key "data_{source url}" - if (content instanceof DataUrl) { - final String dataKey = String.format("data_%s", ((DataUrl) content).getUrl()); - final String persistentUrl = metadata.get(dataKey); - - if (persistentUrl != null) { - ((DataUrl) content).setUrl(persistentUrl); - } - } - } - - return contentList; - } - - public List renderWithDefaultAndLog(Message message) { - return renderWithDefaultAndLog(message, Map.of()); - } - - public List renderWithDefaultAndLog(Message message, Map metadata) { - try { - return this.render(message, metadata); - } catch (Exception e) { - log.error("Failed to render message {}", message, e); - return List.of(new Text("This content cannot be displayed")); - } - } -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/OutboundMapper.java b/lib/java/mapping/src/main/java/co/airy/mapping/OutboundMapper.java deleted file mode 100644 index e96f2b708e..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/OutboundMapper.java +++ /dev/null @@ -1,22 +0,0 @@ -package co.airy.mapping; - -import co.airy.mapping.model.Content; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.stereotype.Component; - -import java.util.List; - -@Component -public class OutboundMapper { - - private final ObjectMapper objectMapper; - - public OutboundMapper() { - this.objectMapper = new ObjectMapper(); - } - - public List render(String payload) throws Exception { - final Content content = objectMapper.readValue(payload, Content.class); - return List.of(content); - } -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/SourceMapper.java b/lib/java/mapping/src/main/java/co/airy/mapping/SourceMapper.java deleted file mode 100644 index 8484c3d5e0..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/SourceMapper.java +++ /dev/null @@ -1,11 +0,0 @@ -package co.airy.mapping; - -import co.airy.mapping.model.Content; - -import java.util.List; - -public interface SourceMapper { - List getIdentifiers(); - - List render(String payload) throws Exception; -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/Audio.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/Audio.java deleted file mode 100644 index dbc8f89f2a..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/Audio.java +++ /dev/null @@ -1,21 +0,0 @@ -package co.airy.mapping.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class Audio extends Content implements DataUrl, Serializable { - @NotNull - private String url; -} - diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/Content.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/Content.java deleted file mode 100644 index bdcd8534f2..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/Content.java +++ /dev/null @@ -1,16 +0,0 @@ -package co.airy.mapping.model; - -import com.fasterxml.jackson.annotation.JsonSubTypes; -import com.fasterxml.jackson.annotation.JsonTypeInfo; - -@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, include = JsonTypeInfo.As.PROPERTY, property = "type") -@JsonSubTypes({ - @JsonSubTypes.Type(value = Text.class, name = "text"), - @JsonSubTypes.Type(value = Audio.class, name = "audio"), - @JsonSubTypes.Type(value = File.class, name = "file"), - @JsonSubTypes.Type(value = Image.class, name = "image"), - @JsonSubTypes.Type(value = Video.class, name = "video"), - @JsonSubTypes.Type(value = SourceTemplate.class, name = "source.template") -}) -public abstract class Content { -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/DataUrl.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/DataUrl.java deleted file mode 100644 index f846ae8fef..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/DataUrl.java +++ /dev/null @@ -1,6 +0,0 @@ -package co.airy.mapping.model; - -public interface DataUrl { - void setUrl(String url); - String getUrl(); -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/File.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/File.java deleted file mode 100644 index 85281ca518..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/File.java +++ /dev/null @@ -1,21 +0,0 @@ -package co.airy.mapping.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class File extends Content implements DataUrl, Serializable { - @NotNull - private String url; -} - diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/Image.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/Image.java deleted file mode 100644 index 622c0ea4df..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/Image.java +++ /dev/null @@ -1,20 +0,0 @@ -package co.airy.mapping.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class Image extends Content implements DataUrl, Serializable { - @NotNull - private String url; -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/SourceTemplate.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/SourceTemplate.java deleted file mode 100644 index 84456cdc02..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/SourceTemplate.java +++ /dev/null @@ -1,21 +0,0 @@ -package co.airy.mapping.model; - -import com.fasterxml.jackson.databind.JsonNode; -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class SourceTemplate extends Content implements Serializable { - @NotNull - private JsonNode payload; -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/Text.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/Text.java deleted file mode 100644 index ec56b357b5..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/Text.java +++ /dev/null @@ -1,20 +0,0 @@ -package co.airy.mapping.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class Text extends Content implements Serializable { - @NotNull - private String text; -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/model/Video.java b/lib/java/mapping/src/main/java/co/airy/mapping/model/Video.java deleted file mode 100644 index e4db5ced10..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/model/Video.java +++ /dev/null @@ -1,21 +0,0 @@ -package co.airy.mapping.model; - -import lombok.AllArgsConstructor; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; -import lombok.NoArgsConstructor; - -import javax.validation.constraints.NotNull; -import java.io.Serializable; - -@Data -@Builder -@NoArgsConstructor -@AllArgsConstructor -@EqualsAndHashCode(callSuper = false) -public class Video extends Content implements DataUrl, Serializable { - @NotNull - private String url; -} - diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/sources/facebook/FacebookMapper.java b/lib/java/mapping/src/main/java/co/airy/mapping/sources/facebook/FacebookMapper.java deleted file mode 100644 index 85705ae58b..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/sources/facebook/FacebookMapper.java +++ /dev/null @@ -1,72 +0,0 @@ -package co.airy.mapping.sources.facebook; - -import co.airy.mapping.SourceMapper; -import co.airy.mapping.model.Audio; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.File; -import co.airy.mapping.model.Image; -import co.airy.mapping.model.SourceTemplate; -import co.airy.mapping.model.Text; -import co.airy.mapping.model.Video; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.function.Function; - -@Component -public class FacebookMapper implements SourceMapper { - - private final ObjectMapper objectMapper; - - public FacebookMapper() { - this.objectMapper = new ObjectMapper(); - } - - private final Map> mediaContentFactory = Map.of( - "image", Image::new, - "video", Video::new, - "audio", Audio::new, - "file", File::new - ); - - @Override - public List getIdentifiers() { - return List.of("facebook"); - } - - @Override - public List render(String payload) throws Exception { - final JsonNode jsonNode = objectMapper.readTree(payload); - final JsonNode messageNode = jsonNode.get("message"); - - List contents = new ArrayList<>(); - - if (messageNode.get("text") != null) { - contents.add(new Text(messageNode.get("text").textValue())); - } - - if (messageNode.get("attachments") != null) { - messageNode.get("attachments") - .elements() - .forEachRemaining(attachmentNode -> { - final String attachmentType = attachmentNode.get("type").textValue(); - - if (attachmentType.equals("template")) { - contents.add(new SourceTemplate(attachmentNode.get("payload"))); - return; - } - - final String url = attachmentNode.get("payload").get("url").textValue(); - - final Content mediaContent = mediaContentFactory.get(attachmentType).apply(url); - contents.add(mediaContent); - }); - } - - return contents; - } -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/sources/google/GoogleMapper.java b/lib/java/mapping/src/main/java/co/airy/mapping/sources/google/GoogleMapper.java deleted file mode 100644 index 6ce335b784..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/sources/google/GoogleMapper.java +++ /dev/null @@ -1,90 +0,0 @@ -package co.airy.mapping.sources.google; - -import co.airy.mapping.SourceMapper; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.Image; -import co.airy.mapping.model.Text; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.stereotype.Component; - -import java.net.URI; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Map; - -import static java.util.stream.Collectors.toMap; - -@Component -public class GoogleMapper implements SourceMapper { - private final ObjectMapper objectMapper; - - public GoogleMapper() { - this.objectMapper = new ObjectMapper(); - } - - @Override - public List getIdentifiers() { - return List.of("google"); - } - - @Override - public List render(String payload) throws Exception { - final JsonNode jsonNode = objectMapper.readTree(payload); - final JsonNode messageNode = jsonNode.get("message"); - if (messageNode != null) { - return renderMessage(messageNode); - } - final JsonNode suggestionResponseNode = jsonNode.get("suggestionResponse"); - if (suggestionResponseNode != null) { - return renderSuggestionResponse(suggestionResponseNode); - } - - throw new Exception("google mapper only supports `message` and `suggestionResponse`"); - } - - private List renderMessage(JsonNode messageNode) { - final String messageNodeValue = messageNode.get("text").textValue(); - if (isGoogleStorageUrl(messageNodeValue)) { - return List.of(new Image(messageNodeValue)); - } else { - return List.of(new Text(messageNodeValue)); - } - } - - private List renderSuggestionResponse(JsonNode suggestionResponseNode) { - final String textContent = suggestionResponseNode.get("text").textValue(); - return List.of(new Text(textContent)); - } - - private boolean isGoogleStorageUrl(final String url) { - URI uri; - try { - uri = new URI(url); - } catch (URISyntaxException e) { - return false; - } - - if (!uri.getHost().startsWith("storage.googleapis.com")) { - return false; - } - - final Map params = List.of(uri.getQuery().split("&")) - .stream() - .map(param -> param.split("=")) - .map(l -> Map.of(l[0], l[1])) - .flatMap(map -> map.entrySet().stream()) - .collect(toMap( - m -> m.getKey().toLowerCase(), - Map.Entry::getValue, - (s1, s2) -> s2)); - - return !params.get("x-goog-algorithm").isEmpty() - && !params.get("x-goog-credential").isEmpty() - && !params.get("x-goog-date").isEmpty() - && !params.get("x-goog-expires").isEmpty() - && !params.get("x-goog-signedheaders").isEmpty() - && !params.get("x-goog-signature").isEmpty(); - } - -} diff --git a/lib/java/mapping/src/main/java/co/airy/mapping/sources/twilio/TwilioMapper.java b/lib/java/mapping/src/main/java/co/airy/mapping/sources/twilio/TwilioMapper.java deleted file mode 100644 index e971a426c0..0000000000 --- a/lib/java/mapping/src/main/java/co/airy/mapping/sources/twilio/TwilioMapper.java +++ /dev/null @@ -1,84 +0,0 @@ -package co.airy.mapping.sources.twilio; - -import co.airy.mapping.SourceMapper; -import co.airy.mapping.model.Audio; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.File; -import co.airy.mapping.model.Image; -import co.airy.mapping.model.Text; -import co.airy.mapping.model.Video; -import org.springframework.stereotype.Component; - -import java.net.URLDecoder; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.function.BiFunction; - -import static java.util.stream.Collectors.toMap; - -@Component -public class TwilioMapper implements SourceMapper { - - private final Map>> mediaProcessFunctions = Map.of( - "jpg", processImageFun, - "jpeg", processImageFun, - "png", processImageFun, - "mp4", processVideoFun, - "mp3", processAudioFun, - "ogg", processAudioFun, - "pdf", processFileFun - ); - - private static final BiFunction> processImageFun = (mediaUrl, body) -> List.of(new Text(body), new Image(mediaUrl)); - private static final BiFunction> processVideoFun = (mediaUrl, body) -> List.of(new Video(mediaUrl)); - private static final BiFunction> processAudioFun = (mediaUrl, body) -> List.of(new Audio(mediaUrl)); - private static final BiFunction> processFileFun = (mediaUrl, body) -> List.of(new File(mediaUrl)); - - @Override - public List getIdentifiers() { - return List.of("twilio.sms", "twilio.whatsapp"); - } - - @Override - public List render(String payload) { - Map decodedPayload = parseUrlEncoded(payload); - List contents = new ArrayList<>(); - - final String mediaUrl = decodedPayload.get("MediaUrl"); - - if (mediaUrl != null && !mediaUrl.isBlank()) { - final String[] mediaUrlParts = mediaUrl.split("\\."); - final String mediaExtension = mediaUrlParts[mediaUrlParts.length - 1].toLowerCase(); - final BiFunction> processMediaFunction = mediaProcessFunctions.get(mediaExtension); - contents = processMediaFunction.apply(mediaUrl, decodedPayload.get("Body")); - } else { - contents.add(new Text(decodedPayload.get("Body"))); - } - - return contents; - } - - private static Map parseUrlEncoded(String payload) { - List kvPairs = Arrays.asList(payload.split("&")); - - return kvPairs.stream() - .map((kvPair) -> { - String[] fields = kvPair.split("="); - - if (fields.length != 2) { - return null; - } - - String name = URLDecoder.decode(fields[0], StandardCharsets.UTF_8); - String value = URLDecoder.decode(fields[1], StandardCharsets.UTF_8); - - return List.of(name, value); - }) - .filter(Objects::nonNull) - .collect(toMap((tuple) -> tuple.get(0), (tuple) -> tuple.get(1))); - } -} diff --git a/lib/java/mapping/src/test/java/co/airy/mapping/ContentMapperTest.java b/lib/java/mapping/src/test/java/co/airy/mapping/ContentMapperTest.java deleted file mode 100644 index 833c2ad098..0000000000 --- a/lib/java/mapping/src/test/java/co/airy/mapping/ContentMapperTest.java +++ /dev/null @@ -1,108 +0,0 @@ -package co.airy.mapping; - -import co.airy.avro.communication.DeliveryState; -import co.airy.avro.communication.Message; -import co.airy.mapping.model.Audio; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.Text; -import co.airy.spring.core.AirySpringBootApplication; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.boot.test.mock.mockito.SpyBean; - -import java.time.Instant; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -@SpringBootTest(classes = AirySpringBootApplication.class) -public class ContentMapperTest { - - @SpyBean - private OutboundMapper outboundMapper; - - @Autowired - private ContentMapper mapper; - - @Test - void rendersOutbound() throws Exception { - final String textContent = "Hello World"; - final Text text = new Text(textContent); - - final Message message = Message.newBuilder() - .setId("other-message-id") - .setSource("facebook") - .setSentAt(Instant.now().toEpochMilli()) - .setSenderId("sourceConversationId") - .setIsFromContact(false) - .setDeliveryState(DeliveryState.DELIVERED) - .setConversationId("conversationId") - .setChannelId("channelId") - .setContent((new ObjectMapper()).writeValueAsString(text)) - .build(); - - final Text textMessage = (Text) mapper.render(message).get(0); - - assertThat(textMessage.getText(), equalTo(textContent)); - Mockito.verify(outboundMapper).render(Mockito.anyString()); - } - - @Test - void includesTypeInformation() throws Exception { - final String testText = "Hello world"; - final Text textContent = new Text(testText); - final String value = (new ObjectMapper()).writeValueAsString(textContent); - - final JsonNode jsonNode = (new ObjectMapper()).readTree(value); - - assertThat(jsonNode.get("type").textValue(), equalTo("text")); - assertThat(jsonNode.get("text").textValue(), equalTo(testText)); - } - - - @Test - void replacesDataUrls() throws Exception { - final String originalUrl = "http://example.org/path/sound.wav"; - - final Message message = Message.newBuilder() - .setId("messageId") - .setSource("fakesource") - .setSentAt(Instant.now().toEpochMilli()) - .setSenderId("sourceConversationId") - .setIsFromContact(true) - .setDeliveryState(DeliveryState.DELIVERED) - .setConversationId("conversationId") - .setChannelId("channelId") - .setContent("{\"audio\":\"" + originalUrl + "\"}") - .build(); - - final ContentMapper mapper = new ContentMapper(List.of(new SourceMapper() { - @Override - public List getIdentifiers() { - return List.of("fakesource"); - } - - @Override - public List render(String payload) { - return List.of(new Audio(originalUrl)); - } - }), outboundMapper); - - // No replacement without metadata - Audio audioMessage = (Audio) mapper.render(message).get(0); - assertThat(audioMessage.getUrl(), equalTo(originalUrl)); - - final String persistentUrl = "http://storage.org/path/data"; - final Map messageMetadata = Map.of("data_" + originalUrl, persistentUrl); - - audioMessage = (Audio) mapper.render(message, messageMetadata).get(0); - assertThat(audioMessage.getUrl(), equalTo(persistentUrl)); - } - -} diff --git a/lib/java/mapping/src/test/java/co/airy/mapping/FacebookTest.java b/lib/java/mapping/src/test/java/co/airy/mapping/FacebookTest.java deleted file mode 100644 index b35490fca5..0000000000 --- a/lib/java/mapping/src/test/java/co/airy/mapping/FacebookTest.java +++ /dev/null @@ -1,90 +0,0 @@ -package co.airy.mapping; - -import co.airy.mapping.model.Audio; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.File; -import co.airy.mapping.model.SourceTemplate; -import co.airy.mapping.model.Video; -import co.airy.mapping.sources.facebook.FacebookMapper; -import org.junit.jupiter.api.Test; -import org.springframework.util.StreamUtils; - -import java.nio.charset.StandardCharsets; -import java.util.List; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.everyItem; -import static org.hamcrest.Matchers.hasProperty; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.Is.isA; - -public class FacebookTest { - - private final FacebookMapper mapper = new FacebookMapper(); - - @Test - void textMessage() throws Exception { - final String text = "Hello world"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream("facebook/text.json"), StandardCharsets.UTF_8), text); - - final List contents = mapper.render(sourceContent); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(hasProperty("text", equalTo(text)))); - } - - @Test - void canRenderImages() throws Exception { - final String imageUrl = "https://url-from-facebook.com/123-id"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream("facebook/image.json"), StandardCharsets.UTF_8), imageUrl); - - final List contents = mapper.render(sourceContent); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(hasProperty("url", equalTo(imageUrl)))); - } - - @Test - void canRenderAudio() throws Exception { - final String audioUrl = "https://url-from-facebook-cdn.com/123-id.mp4"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream("facebook/audio.json"), StandardCharsets.UTF_8), audioUrl); - - final List contents = mapper.render(sourceContent); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(isA(Audio.class))); - assertThat(contents, everyItem(hasProperty("url", equalTo(audioUrl)))); - } - - @Test - void canRenderVideo() throws Exception { - final String videoUrl = "https://video.xx.fbcdn.net/v/t42.3356-2/10000000_3670351823059679_8252233555063215828_n.mp4/video-1608240605.mp4?_nc_cat=110&ccb=2&_nc_sid=060d78&_nc_ohc=o7KPshTGwGwAX9xsWIT&vabr=1741655&_nc_ht=video.xx&oh=76581365554981fa6e343c3b2fa65aef&oe=5FDD193B"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream("facebook/video.json"), StandardCharsets.UTF_8), videoUrl); - - final List contents = mapper.render(sourceContent); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(isA(Video.class))); - assertThat(contents, everyItem(hasProperty("url", equalTo(videoUrl)))); - } - - @Test - void canRenderFile() throws Exception { - final String fileUrl = "https://cdn.fbsbx.com/v/t59.2708-21/file_identifier.pdf/file_name.pdf"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream("facebook/file.json"), StandardCharsets.UTF_8), fileUrl); - - final List contents = mapper.render(sourceContent); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(isA(File.class))); - assertThat(contents, everyItem(hasProperty("url", equalTo(fileUrl)))); - } - - @Test - void canRenderTemplates() throws Exception { - final List templateTypes = List.of("generic"); - - for (String templateType : templateTypes) { - final String content = StreamUtils.copyToString(getClass().getClassLoader().getResourceAsStream(String.format("facebook/template_%s.json", templateType)), StandardCharsets.UTF_8); - final List contents = mapper.render(content); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(isA(SourceTemplate.class))); - } - } -} diff --git a/lib/java/mapping/src/test/java/co/airy/mapping/GoogleTest.java b/lib/java/mapping/src/test/java/co/airy/mapping/GoogleTest.java deleted file mode 100644 index f33159bbaa..0000000000 --- a/lib/java/mapping/src/test/java/co/airy/mapping/GoogleTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package co.airy.mapping; - -import co.airy.mapping.model.Image; -import co.airy.mapping.model.Text; -import co.airy.mapping.sources.google.GoogleMapper; -import org.junit.jupiter.api.Test; -import org.springframework.util.StreamUtils; - -import java.nio.charset.StandardCharsets; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; - -public class GoogleTest { - private final GoogleMapper mapper = new GoogleMapper(); - - @Test - void canRenderText() throws Exception { - final String textContent = "Hello World"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader() - .getResourceAsStream("google/text.json"), StandardCharsets.UTF_8), textContent); - - final Text message = (Text) mapper.render(sourceContent).get(0); - assertThat(message.getText(), equalTo(textContent)); - } - - @Test - void canRenderImage() throws Exception { - final String signedImageUrl = "https://storage.googleapis.com/business-messages-us/936640919331/jzsu6cdguNGsBhmGJGuLs1DS?x-goog-algorithm\u003dGOOG4-RSA-SHA256\u0026x-goog-credential\u003duranium%40rcs-uranium.iam.gserviceaccount.com%2F20190826%2Fauto%2Fstorage%2Fgoog4_request\u0026x-goog-date\u003d20190826T201038Z\u0026x-goog-expires\u003d604800\u0026x-goog-signedheaders\u003dhost\u0026x-goog-signature\u003d89dbf7a74d21ab42ad25be071b37840a544a43d68e67270382054e1442d375b0b53d15496dbba12896b9d88a6501cac03b5cfca45d789da3e0cae75b050a89d8f54c1ffb27e467bd6ba1d146b7d42e30504c295c5c372a46e44728f554ba74b7b99bd9c6d3ed45f18588ed1b04522af1a47330cff73a711a6a8c65bb15e3289f480486f6695127e1014727cac949e284a7f74afd8220840159c589d48dddef1cc97b248dfc34802570448242eac4d7190b1b10a008404a330b4ff6f9656fa84e87f9a18ab59dc9b91e54ad11ffdc0ad1dc9d1ccc7855c0d263d93fce6f999971ec79879f922b582cf3bb196a1fedc3eefa226bb412e49af7dfd91cc072608e98"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader() - .getResourceAsStream("google/text.json"), StandardCharsets.UTF_8), signedImageUrl); - - final Image message = (Image) mapper.render(sourceContent).get(0); - assertThat(message.getUrl(), equalTo(signedImageUrl)); - } - - @Test - void canRenderSuggestionResponses() throws Exception { - final String textContent = "Hello World"; - final String sourceContent = String.format(StreamUtils.copyToString(getClass().getClassLoader() - .getResourceAsStream("google/suggestionResponse.json"), StandardCharsets.UTF_8), textContent); - - final Text message = (Text) mapper.render(sourceContent).get(0); - assertThat(message.getText(), equalTo(textContent)); - } -} diff --git a/lib/java/mapping/src/test/java/co/airy/mapping/TwilioTest.java b/lib/java/mapping/src/test/java/co/airy/mapping/TwilioTest.java deleted file mode 100644 index 70109cc582..0000000000 --- a/lib/java/mapping/src/test/java/co/airy/mapping/TwilioTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package co.airy.mapping; - -import co.airy.mapping.model.Audio; -import co.airy.mapping.model.Content; -import co.airy.mapping.model.File; -import co.airy.mapping.model.Image; -import co.airy.mapping.model.Text; -import co.airy.mapping.model.Video; -import co.airy.mapping.sources.twilio.TwilioMapper; -import org.junit.jupiter.api.Test; - -import java.util.List; - -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasProperty; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.Every.everyItem; -import static org.hamcrest.core.Is.isA; - -public class TwilioTest { - private final TwilioMapper mapper = new TwilioMapper(); - - @Test - void canRenderText() { - final String body = "Hello World"; - - String event = "ApiVersion=2010-04-01&SmsSid=SMbc31b6419de618d65076200c54676476&SmsStatus=received" + - "&SmsMessageSid=SMbc31b6419de618d65076200c54676476&NumSegments=1&To=whatsapp%3A%2B" + - "&From=whatsapp%3A%2B&MessageSid=SMbc31b6419de618d65076200c54676476" + - "&Body=" + body + "&AccountSid=AC64c9ab479b849275b7b50bd19540c602&NumMedia=0"; - - final List contents = mapper.render(event); - assertThat(contents, hasSize(1)); - assertThat(contents, everyItem(hasProperty("text", equalTo(body)))); - } - - @Test - void canRenderImage() { - final String body = "Heres a picture of an owl!"; - final String imageUrl = "https://demo.twilio.com/owl.png"; - - String event = "ApiVersion=2010-04-01&SmsSid=SMbc31b6419de618d65076200c54676476&SmsStatus=received" + - "&SmsMessageSid=SMbc31b6419de618d65076200c54676476&NumSegments=1&To=whatsapp%3A%2B" + - "&From=whatsapp%3A%2B&MessageSid=SMbc31b6419de618d65076200c54676476" + - "&Body=" + body + "&AccountSid=AC64c9ab479b849275b7b50bd19540c602&NumMedia=0" + - "&MediaUrl=" + imageUrl; - - final List message = mapper.render(event); - assertThat(message, hasItem(isA(Text.class))); - assertThat(message, hasItem(isA(Image.class))); - assertThat(message, hasItem(hasProperty("url", equalTo(imageUrl)))); - } - - @Test - void canRenderAudio() { - final String audioUrl = "https://demo.twilio.com/owl.mp3"; - - String event = "ApiVersion=2010-04-01&SmsSid=SMbc31b6419de618d65076200c54676476&SmsStatus=received" + - "&SmsMessageSid=SMbc31b6419de618d65076200c54676476&NumSegments=1&To=whatsapp%3A%2B" + - "&From=whatsapp%3A%2B&MessageSid=SMbc31b6419de618d65076200c54676476" + - "&MediaUrl=" + audioUrl; - final List message = mapper.render(event); - - assertThat(message, everyItem(isA(Audio.class))); - assertThat(message, everyItem(hasProperty("url", equalTo(audioUrl)))); - } - - @Test - void canRenderVideo() { - final String videoUrl = "https://demo.twilio.com/owl.mp4"; - - String event = "ApiVersion=2010-04-01&SmsSid=SMbc31b6419de618d65076200c54676476&SmsStatus=received" + - "&SmsMessageSid=SMbc31b6419de618d65076200c54676476&NumSegments=1&To=whatsapp%3A%2B" + - "&From=whatsapp%3A%2B&MessageSid=SMbc31b6419de618d65076200c54676476" + - "&MediaUrl=" + videoUrl; - final List message = mapper.render(event); - - assertThat(message, everyItem(isA(Video.class))); - assertThat(message, everyItem(hasProperty("url", equalTo(videoUrl)))); - } - - @Test - void canRenderFile() { - final String fileUrl = "https://demo.twilio.com/file.pdf"; - - String event = "ApiVersion=2010-04-01&SmsSid=SMbc31b6419de618d65076200c54676476&SmsStatus=received" + - "&SmsMessageSid=SMbc31b6419de618d65076200c54676476&NumSegments=1&To=whatsapp%3A%2B" + - "&From=whatsapp%3A%2B&MessageSid=SMbc31b6419de618d65076200c54676476" + - "&MediaUrl=" + fileUrl; - final List message = mapper.render(event); - - assertThat(message, everyItem(isA(File.class))); - assertThat(message, everyItem(hasProperty("url", equalTo(fileUrl)))); - } -} diff --git a/lib/java/mapping/src/test/resources/facebook/audio.json b/lib/java/mapping/src/test/resources/facebook/audio.json deleted file mode 100644 index 53f27884b0..0000000000 --- a/lib/java/mapping/src/test/resources/facebook/audio.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "sender": { - "id": "4616529495039079" - }, - "recipient": { - "id": "778234505682382" - }, - "timestamp": 1609577719481, - "message": { - "mid": "m_pKvXdyjEV4mYkSF0F6otXRwIEODhgYRbCR-HCFhf7WzGQ4iAVNlyESVoVKZS-BHiErXDbdEvqFiDedW4h6twNQ", - "attachments": [ - { - "type": "audio", - "payload": { - "url": "%s" - } - } - ] - } -} \ No newline at end of file diff --git a/lib/java/mapping/src/test/resources/facebook/file.json b/lib/java/mapping/src/test/resources/facebook/file.json deleted file mode 100644 index 32781c5e84..0000000000 --- a/lib/java/mapping/src/test/resources/facebook/file.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "sender": { - "id": "205272702823080" - }, - "recipient": { - "id": "1988408424614138" - }, - "timestamp": 1608188342869, - "message": { - "mid": "m_nb9pmQ65hY39kEdLqIDYPA4G-2dMMotKb_j8BxVy_IrITEweFGxuCKq6AQZDpU0-L3nDIVgepkGoqcR4D8HGxg", - "is_echo": true, - "app_id": 113868326204563, - "attachments": [ - { - "type": "file", - "payload": { - "url": "%s" - } - } - ] - } -} \ No newline at end of file diff --git a/lib/java/mapping/src/test/resources/facebook/image.json b/lib/java/mapping/src/test/resources/facebook/image.json deleted file mode 100644 index c4297708bd..0000000000 --- a/lib/java/mapping/src/test/resources/facebook/image.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "sender": { - "id": "1555785507858425" - }, - "recipient": { - "id": "107331939315579" - }, - "timestamp": 1550140045218, - "message": { - "mid": "MRipA_57_ahMME9DHzJLh2Do-QOtGOTT4nLEl1p6TH66Vx7UoXlfbHzJNixl6sdfHBjqfM0OAABIN-olixxHVg", - "seq": 51802, - "attachments": [ - { - "type": "image", - "payload": { - "url": "%s" - } - } - ] - } -} \ No newline at end of file diff --git a/lib/java/mapping/src/test/resources/facebook/template_generic.json b/lib/java/mapping/src/test/resources/facebook/template_generic.json deleted file mode 100644 index 9012127a66..0000000000 --- a/lib/java/mapping/src/test/resources/facebook/template_generic.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "sender": { - "id": "4616529495039079" - }, - "recipient": { - "id": "778234505682382" - }, - "timestamp": 1550050473934, - "message": { - "is_echo": true, - "app_id": 123, - "mid": "l9sIeXHGFbkAL5m62DbqF2nKw8PPGcoZ0ruggAoYBrnyu8w-rcnEazyvqHqp3VeTu8k3NK-N1fCnAdzcs9kcUw", - "seq": 85083, - "attachments": [ - { - "title": "test", - "url": null, - "type": "template", - "payload": { - "template_type": "generic", - "sharable": true, - "elements": [ - { - "title": "awdadw", - "image_url": "https://airy-layer-production.s3.amazonaws.com/templates/8787c530-2f72-11e9-867e-f7de52fd949f.jpeg", - "subtitle": "adww" - } - ] - } - } - ] - } -} diff --git a/lib/java/mapping/src/test/resources/facebook/text.json b/lib/java/mapping/src/test/resources/facebook/text.json deleted file mode 100644 index 768da6d04b..0000000000 --- a/lib/java/mapping/src/test/resources/facebook/text.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "sender": { - "id": "12345" - }, - "recipient": { - "id": "12345" - }, - "timestamp": 1550824236438, - "message": { - "text": "%s" - } -} diff --git a/lib/java/mapping/src/test/resources/facebook/video.json b/lib/java/mapping/src/test/resources/facebook/video.json deleted file mode 100644 index f9a28a64d1..0000000000 --- a/lib/java/mapping/src/test/resources/facebook/video.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "sender": { - "id": "3448289328620117" - }, - "recipient": { - "id": "569078233236402" - }, - "timestamp": 1608240605982, - "message": { - "mid": "m_b42WdoNzwv4d_UL-vuRmVwoQrkTsoIshZoEKAiAARYac6QzyDxKM3qnHXt5JifI5x86p7_Q2SPQM-8bBRFLbdA", - "attachments": [ - { - "type": "video", - "payload": { - "url": "%s" - } - } - ] - } -} \ No newline at end of file diff --git a/lib/java/mapping/src/test/resources/google/suggestionResponse.json b/lib/java/mapping/src/test/resources/google/suggestionResponse.json deleted file mode 100644 index 5b6f1110a1..0000000000 --- a/lib/java/mapping/src/test/resources/google/suggestionResponse.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "suggestionResponse": { - "message": "conversations/11ab7bf3-0410-46a6-bcce-1877ca6957b0/messages/11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "postbackData": "postback-data", - "createTime": "2020-07-13T12:54:50.479632Z", - "text": "%s", - "type": "REPLY" - }, - "context": {}, - "sendTime": "2020-05-14T12:45:55.302Z", - "conversationId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "customAgentId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "requestId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "agent": "brands/11ab7bf3-0410-46a6-bcce-1877ca6957b0/agents/11ab7bf3-0410-46a6-bcce-1877ca6957b0" -} diff --git a/lib/java/mapping/src/test/resources/google/text.json b/lib/java/mapping/src/test/resources/google/text.json deleted file mode 100644 index fa2156c57f..0000000000 --- a/lib/java/mapping/src/test/resources/google/text.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "message": { - "name": "conversations/11ab7bf3-0410-46a6-bcce-1877ca6957b0/messages/11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "text": "%s", - "createTime": "2020-05-14T12:45:54.531828Z", - "messageId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0" - }, - "context": {}, - "sendTime": "2020-05-14T12:45:55.302Z", - "conversationId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "customAgentId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "requestId": "11ab7bf3-0410-46a6-bcce-1877ca6957b0", - "agent": "brands/11ab7bf3-0410-46a6-bcce-1877ca6957b0/agents/11ab7bf3-0410-46a6-bcce-1877ca6957b0" -} diff --git a/lib/java/uuid/BUILD b/lib/java/uuid/BUILD index fd9a73cd53..abc2aa7241 100644 --- a/lib/java/uuid/BUILD +++ b/lib/java/uuid/BUILD @@ -1,5 +1,6 @@ load("@com_github_airyhq_bazel_tools//lint:buildifier.bzl", "check_pkg") load("@rules_java//java:defs.bzl", "java_library") +load("//tools/build:junit5.bzl", "junit5") # Due to a bug in the abbreviation checkstyle rule # we need to skip style checking on this lib @@ -9,4 +10,15 @@ java_library( visibility = ["//visibility:public"], ) +[ + junit5( + file = file, + deps = [ + ":uuid", + "//:junit", + ], + ) + for file in glob(["src/test/java/**/*Test.java"]) +] + check_pkg(name = "buildifier") diff --git a/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5.java b/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5.java index 4286744057..39988a9844 100644 --- a/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5.java +++ b/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5.java @@ -1,48 +1,57 @@ package co.airy.uuid; +import java.io.IOException; +import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.Arrays; import java.util.UUID; +import static co.airy.uuid.UUIDv5Builder.fromBytes; + /** * Unfortunately the Java standard library does not provide a UUID v5 implementation * This implementation is taken out from https://github.com/eugenp/tutorials/tree/master/core-java-modules/core-java/ * Reference: https://www.baeldung.com/java-uuid */ public class UUIDv5 { + public static UUID fromNamespaceAndName(String namespace, String name) { - String source = namespace + name; - byte[] bytes = source.getBytes(StandardCharsets.UTF_8); + return fromName(namespace + name); + } + + public static UUID fromName(String name) { + byte[] data = name.getBytes(StandardCharsets.UTF_8); + + final MessageDigest md = getDigest(); + byte[] bytes = Arrays.copyOfRange(md.digest(data), 0, 16); return fromBytes(bytes); } - private static UUID fromBytes(byte[] name) { - MessageDigest md; - try { - md = MessageDigest.getInstance("SHA-1"); - } catch (NoSuchAlgorithmException nsae) { - throw new InternalError("SHA-1 not supported", nsae); + public static UUID fromFile(InputStream inputStream) throws IOException { + if (inputStream.markSupported()) { + inputStream.mark(Integer.MAX_VALUE); } - byte[] bytes = Arrays.copyOfRange(md.digest(name), 0, 16); - bytes[6] &= 0x0f; /* clear version */ - bytes[6] |= 0x50; /* set to version 5 */ - bytes[8] &= 0x3f; /* clear variant */ - bytes[8] |= 0x80; /* set to IETF variant */ - return construct(bytes); - } - private static UUID construct(byte[] data) { - long msb = 0; - long lsb = 0; - assert data.length == 16 : "data must be 16 bytes in length"; + MessageDigest md = getDigest(); + try (DigestInputStream dis = new DigestInputStream(inputStream, md)) { + while (dis.read() != -1) ; //empty loop to clear the data + md = dis.getMessageDigest(); + } - for (int i = 0; i < 8; i++) - msb = (msb << 8) | (data[i] & 0xff); + if (inputStream.markSupported()) { + inputStream.reset(); + } + return fromBytes(md.digest()); + } - for (int i = 8; i < 16; i++) - lsb = (lsb << 8) | (data[i] & 0xff); - return new UUID(msb, lsb); + private static MessageDigest getDigest() { + try { + return MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException nsae) { + throw new InternalError("SHA-1 not supported", nsae); + } } } diff --git a/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5Builder.java b/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5Builder.java new file mode 100644 index 0000000000..f828679a34 --- /dev/null +++ b/lib/java/uuid/src/main/java/co/airy/uuid/UUIDv5Builder.java @@ -0,0 +1,32 @@ +package co.airy.uuid; + +import java.util.UUID; + +class UUIDv5Builder { + + /** + * @param data byte input of length 16 + * @return UUID version 5 + */ + public static UUID fromBytes(byte[] data) { + assert data.length == 16 : "data must be 16 bytes in length"; + data[6] &= 0x0f; /* clear version */ + data[6] |= 0x50; /* set to version 5 */ + data[8] &= 0x3f; /* clear variant */ + data[8] |= 0x80; /* set to IETF variant */ + return construct(data); + } + + private static UUID construct(byte[] data) { + long msb = 0; + long lsb = 0; + assert data.length == 16 : "data must be 16 bytes in length"; + + for (int i = 0; i < 8; i++) + msb = (msb << 8) | (data[i] & 0xff); + + for (int i = 8; i < 16; i++) + lsb = (lsb << 8) | (data[i] & 0xff); + return new UUID(msb, lsb); + } +} diff --git a/lib/java/uuid/src/test/java/co/airy/uuid/UUIDv5Test.java b/lib/java/uuid/src/test/java/co/airy/uuid/UUIDv5Test.java new file mode 100644 index 0000000000..eb9f03844d --- /dev/null +++ b/lib/java/uuid/src/test/java/co/airy/uuid/UUIDv5Test.java @@ -0,0 +1,36 @@ +package co.airy.uuid; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.core.IsEqual.equalTo; + +public class UUIDv5Test { + + @Test + void uuidFromString() { + assertThat(UUIDv5.fromNamespaceAndName("test", "test").toString(),equalTo("51abb963-6078-5efb-b888-d8457a7c76f8")); + assertThat(UUIDv5.fromNamespaceAndName("source", "98ce13ef-5469-4db3-af62-579be61cfaa4").toString(),equalTo("e5fc97f1-2f2d-53c5-85c7-97a83b17e9ff")); + assertThat(UUIDv5.fromNamespaceAndName("source", "98ce13ef-5469-4db3-af62-579be61cfaa4".repeat(1_000_000)).toString(),equalTo("2c2ab940-6393-5349-bd87-bb236a541827")); + } + + @Test + void uuidFromFile() throws Exception { + assertThat(UUIDv5.fromFile(new ByteArrayInputStream(("testtest").getBytes())).toString(), equalTo("51abb963-6078-5efb-b888-d8457a7c76f8")); + assertThat(UUIDv5.fromFile(new ByteArrayInputStream(("source98ce13ef-5469-4db3-af62-579be61cfaa4").getBytes())).toString(), equalTo("e5fc97f1-2f2d-53c5-85c7-97a83b17e9ff")); + } + + + @Test + void inputStreamIsReset() throws Exception { + final String testString = "source" + "98ce13ef-5469-4db3-af62-579be61cfaa4".repeat(1_000_000); + final ByteArrayInputStream inputStream = new ByteArrayInputStream(testString.getBytes()); + assertThat(UUIDv5.fromFile(inputStream).toString(), equalTo("2c2ab940-6393-5349-bd87-bb236a541827")); + + final byte[] bytes = inputStream.readAllBytes(); + assertThat(new String(bytes, StandardCharsets.UTF_8), equalTo(testString)); + } +}