From b003bf0b94c6a0ac9ccbe1ec664b33f40a8f78c2 Mon Sep 17 00:00:00 2001 From: WinterYukky <49480575+WinterYukky@users.noreply.github.com> Date: Tue, 31 Oct 2023 03:17:42 +0000 Subject: [PATCH] Revert "Feature/revert template converter (#107)" This reverts commit 52e4d9d7a41a6518f2a4036007a0104ceba13479. --- .../aws-kendra-datasource.json | 23 ++++- aws-kendra-datasource/pom.xml | 5 ++ .../amazon/kendra/datasource/Translator.java | 6 +- .../datasource/convert/TemplateConverter.java | 40 +++++++++ .../datasource/utils/DocumentTypeAdapter.java | 33 +++++++ .../kendra/datasource/TranslatorTest.java | 43 ++++++++- .../convert/TemplateConverterTest.java | 88 +++++++++++++++++++ .../utils/DocumentTypeAdapterTest.java | 35 ++++++++ 8 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/convert/TemplateConverter.java create mode 100644 aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapter.java create mode 100644 aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/convert/TemplateConverterTest.java create mode 100644 aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapterTest.java diff --git a/aws-kendra-datasource/aws-kendra-datasource.json b/aws-kendra-datasource/aws-kendra-datasource.json index ec3adb9..75fbe9b 100644 --- a/aws-kendra-datasource/aws-kendra-datasource.json +++ b/aws-kendra-datasource/aws-kendra-datasource.json @@ -1384,6 +1384,18 @@ "OrganizationId" ] }, + "TemplateConfiguration": { + "type": "object", + "properties": { + "Template": { + "type": "string" + } + }, + "additionalProperties": false, + "required": [ + "Template" + ] + }, "DataSourceConfiguration": { "type": "object", "properties": { @@ -1416,6 +1428,9 @@ }, "WorkDocsConfiguration": { "$ref": "#/definitions/WorkDocsConfiguration" + }, + "TemplateConfiguration": { + "$ref": "#/definitions/TemplateConfiguration" } }, "additionalProperties": false, @@ -1469,6 +1484,11 @@ "required": [ "WorkDocsConfiguration" ] + }, + { + "required": [ + "TemplateConfiguration" + ] } ] }, @@ -1492,7 +1512,8 @@ "CONFLUENCE", "GOOGLEDRIVE", "WEBCRAWLER", - "WORKDOCS" + "WORKDOCS", + "TEMPLATE" ] }, "Description": { diff --git a/aws-kendra-datasource/pom.xml b/aws-kendra-datasource/pom.xml index df49131..6d698a2 100644 --- a/aws-kendra-datasource/pom.xml +++ b/aws-kendra-datasource/pom.xml @@ -66,6 +66,11 @@ 2.26.0 test + + com.google.code.gson + gson + 2.8.6 + diff --git a/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/Translator.java b/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/Translator.java index da4d970..5424ff9 100644 --- a/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/Translator.java +++ b/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/Translator.java @@ -21,6 +21,7 @@ import software.amazon.kendra.datasource.convert.SalesforceConverter; import software.amazon.kendra.datasource.convert.ServiceNowConverter; import software.amazon.kendra.datasource.convert.SharePointConverter; +import software.amazon.kendra.datasource.convert.TemplateConverter; import software.amazon.kendra.datasource.convert.WebCrawlerConverter; import software.amazon.kendra.datasource.convert.WorkDocsConverter; import software.amazon.kendra.datasource.convert.cde.CustomDocumentEnrichmentConfigurationConverter; @@ -233,6 +234,7 @@ static software.amazon.awssdk.services.kendra.model.DataSourceConfiguration toSd modelDataSourceConfiguration.googleDriveConfiguration(GoogleDriveConverter.toSdkDataSourceConfiguration(dataSourceConfiguration.getGoogleDriveConfiguration())); modelDataSourceConfiguration.webCrawlerConfiguration(WebCrawlerConverter.toSdkDataSourceConfiguration(dataSourceConfiguration.getWebCrawlerConfiguration())); modelDataSourceConfiguration.workDocsConfiguration(WorkDocsConverter.toSdkDataSourceConfiguration(dataSourceConfiguration.getWorkDocsConfiguration())); + modelDataSourceConfiguration.templateConfiguration(TemplateConverter.toSdkDataSourceConfiguration(dataSourceConfiguration.getTemplateConfiguration())); return modelDataSourceConfiguration.build(); } @@ -258,7 +260,9 @@ static DataSourceConfiguration toModelDataSourceConfiguration( return WebCrawlerConverter.toModelDataSourceConfiguration(dataSourceConfiguration.webCrawlerConfiguration()); } else if (DataSourceType.WORKDOCS.toString().equals(dataSourceType)) { return WorkDocsConverter.toModelDataSourceConfiguration(dataSourceConfiguration.workDocsConfiguration()); - } else { + } else if(DataSourceType.TEMPLATE.toString().equals(dataSourceType)){ + return TemplateConverter.toModelDataSourceConfiguration(dataSourceConfiguration.templateConfiguration()); + }else { return null; } } diff --git a/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/convert/TemplateConverter.java b/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/convert/TemplateConverter.java new file mode 100644 index 0000000..d6baabc --- /dev/null +++ b/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/convert/TemplateConverter.java @@ -0,0 +1,40 @@ +package software.amazon.kendra.datasource.convert; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import software.amazon.awssdk.core.document.Document; +import software.amazon.kendra.datasource.DataSourceConfiguration; +import software.amazon.kendra.datasource.TemplateConfiguration; +import software.amazon.kendra.datasource.utils.DocumentTypeAdapter; + +public class TemplateConverter { + private static final Gson builder = new GsonBuilder() + .registerTypeAdapter(Document.class, new DocumentTypeAdapter()) + .setPrettyPrinting() + .create(); + + public static software.amazon.awssdk.services.kendra.model.TemplateConfiguration toSdkDataSourceConfiguration(TemplateConfiguration model) { + if (model == null) { + return null; + } + return software.amazon.awssdk.services.kendra.model.TemplateConfiguration.builder() + .template(builder.fromJson(model.getTemplate(), Document.class)) + .build(); + } + + public static DataSourceConfiguration toModelDataSourceConfiguration( + software.amazon.awssdk.services.kendra.model.TemplateConfiguration templateConfiguration) { + return DataSourceConfiguration.builder() + .templateConfiguration(toModel(templateConfiguration)) + .build(); + } + + private static TemplateConfiguration toModel(software.amazon.awssdk.services.kendra.model.TemplateConfiguration sdk) { + if (sdk == null) { + return null; + } + return TemplateConfiguration.builder() + .template(builder.toJson(sdk.template(), Document.class)) + .build(); + } +} diff --git a/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapter.java b/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapter.java new file mode 100644 index 0000000..c6222bd --- /dev/null +++ b/aws-kendra-datasource/src/main/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapter.java @@ -0,0 +1,33 @@ +package software.amazon.kendra.datasource.utils; + +import com.google.gson.JsonDeserializationContext; +import com.google.gson.JsonDeserializer; +import com.google.gson.JsonElement; +import com.google.gson.JsonParseException; +import com.google.gson.JsonSerializationContext; +import com.google.gson.JsonSerializer; +import software.amazon.awssdk.core.document.Document; +import software.amazon.awssdk.protocols.json.internal.unmarshall.document.DocumentUnmarshaller; +import software.amazon.awssdk.protocols.jsoncore.JsonNode; +import software.amazon.awssdk.protocols.jsoncore.JsonNodeParser; + +import java.lang.reflect.Type; + +public class DocumentTypeAdapter implements JsonDeserializer, JsonSerializer { + private final DocumentUnmarshaller documentUnmarshaller = new DocumentUnmarshaller(); + + @Override + public Document deserialize( + final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) + throws JsonParseException { + final JsonNodeParser jsonNodeParser = JsonNodeParser.create(); + final JsonNode jsonNode = jsonNodeParser.parse(jsonElement.toString()); + return jsonNode.visit(documentUnmarshaller); + } + + @Override + public JsonElement serialize( + final Document document, final Type type, final JsonSerializationContext context) { + return context.serialize(document.unwrap()); + } +} diff --git a/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/TranslatorTest.java b/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/TranslatorTest.java index 77d15eb..62eba7e 100644 --- a/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/TranslatorTest.java +++ b/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/TranslatorTest.java @@ -1,6 +1,7 @@ package software.amazon.kendra.datasource; - +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; import org.apache.commons.io.FileUtils; import org.junit.jupiter.api.Test; import software.amazon.awssdk.core.document.Document; @@ -20,6 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import software.amazon.awssdk.services.kendra.model.DataSourceType; +import software.amazon.kendra.datasource.utils.DocumentTypeAdapter; public class TranslatorTest { private static final String DATASOURCE_CONFIGURATION = @@ -546,5 +548,44 @@ void testTranslateS3_WHEN_Configuration_NULL() { assertThat(Translator.translateToCreateRequest(resourceModel)).isEqualTo(expected); } + @Test + void testTranslateToSdkTemplate() throws IOException { + DataSourceConfiguration dataSourceConfiguration = DataSourceConfiguration + .builder() + .templateConfiguration(TemplateConfiguration.builder().template(readFileFromLocal()).build()) + .build(); + assertThat(Translator.toSdkDataSourceConfiguration(dataSourceConfiguration)) + .isEqualTo(software.amazon.awssdk.services.kendra.model.DataSourceConfiguration.builder() + .templateConfiguration(software.amazon.awssdk.services.kendra.model.TemplateConfiguration.builder(). + template(getTemplate()).build()) + .build()); + } + + @Test + void testTranslateToCreateRequest_WithTemplateConfig() throws IOException { + String indexId = "indexId"; + ResourceModel resourceModel = ResourceModel + .builder() + .indexId(indexId) + .type("TEMPLATE") + .dataSourceConfiguration(DataSourceConfiguration.builder() + .templateConfiguration(TemplateConfiguration.builder().template(readFileFromLocal()).build()) + .build()) + .build(); + CreateDataSourceRequest createDataSourceRequest = Translator.translateToCreateRequest(resourceModel); + assertThat(createDataSourceRequest.indexId()).isEqualTo(indexId); + assertThat(createDataSourceRequest.configuration().templateConfiguration()).isNotNull(); + } + + private Document getTemplate() throws IOException { + Gson builder = new GsonBuilder() + .registerTypeAdapter(Document.class, new DocumentTypeAdapter()) + .create(); + return builder.fromJson(this.readFileFromLocal(), Document.class); + } + + private String readFileFromLocal() throws IOException { + return FileUtils.readFileToString(new File(TranslatorTest.DATASOURCE_CONFIGURATION), Charset.defaultCharset()); + } } diff --git a/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/convert/TemplateConverterTest.java b/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/convert/TemplateConverterTest.java new file mode 100644 index 0000000..af859aa --- /dev/null +++ b/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/convert/TemplateConverterTest.java @@ -0,0 +1,88 @@ +package software.amazon.kendra.datasource.convert; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.JsonObject; + +import org.apache.commons.io.FileUtils; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.document.Document; +import software.amazon.kendra.datasource.DataSourceConfiguration; +import software.amazon.kendra.datasource.TemplateConfiguration; +import software.amazon.kendra.datasource.utils.DocumentTypeAdapter; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; + +import static org.assertj.core.api.Assertions.assertThat; + + +public class TemplateConverterTest { + private static final String DATASOURCE_CONFIGURATION = + "testdata/datasourceconfiguration/valid_datasource_configuration.json"; + + @Test + public void testSdkDataSourceConfiguration_Null() { + assertThat(TemplateConverter.toSdkDataSourceConfiguration(null)) + .isEqualTo(null); + } + + @Test + public void testModelDataSourceConfiguration_Null() { + assertThat(TemplateConverter.toModelDataSourceConfiguration(null) + .getTemplateConfiguration()).isEqualTo(null); + } + + @Test + public void testSDKDataSourceConfigurationWithValidConfig() throws IOException { + DataSourceConfiguration dataSourceConfiguration = DataSourceConfiguration.builder() + .templateConfiguration(TemplateConfiguration.builder() + .template(readFileFromLocal(DATASOURCE_CONFIGURATION)) + .build()) + .build(); + + software.amazon.awssdk.services.kendra.model.TemplateConfiguration expectedDataSourceConfiguration = + software.amazon.awssdk.services.kendra.model.TemplateConfiguration.builder() + .template(getTemplate(DATASOURCE_CONFIGURATION)).build(); + + assertThat(TemplateConverter.toSdkDataSourceConfiguration(dataSourceConfiguration.getTemplateConfiguration())) + .isEqualTo(expectedDataSourceConfiguration); + + } + + @Test + public void testModelDataSourceConfigurationWithValidConfig() throws IOException { + software.amazon.awssdk.services.kendra.model.TemplateConfiguration sdkDataSourceConfiguration = + software.amazon.awssdk.services.kendra.model.TemplateConfiguration.builder() + .template(getTemplate(DATASOURCE_CONFIGURATION)).build(); + + DataSourceConfiguration expectedDataSourceConfiguration = DataSourceConfiguration.builder() + .templateConfiguration(TemplateConfiguration.builder() + .template(readFileFromLocal(DATASOURCE_CONFIGURATION)) + .build()) + .build(); + + Gson gson = new Gson(); + DataSourceConfiguration result = TemplateConverter.toModelDataSourceConfiguration(sdkDataSourceConfiguration); + JsonObject resultTemplateConfig = gson.fromJson(result.getTemplateConfiguration().getTemplate(), JsonObject.class); + + JsonObject expectedDataSourceConfigurationTemplateConfig = gson.fromJson( + expectedDataSourceConfiguration.getTemplateConfiguration().getTemplate(), JsonObject.class + ); + + assertThat(expectedDataSourceConfigurationTemplateConfig) + .isEqualTo(resultTemplateConfig); + } + + private Document getTemplate(String filePath) throws IOException { + Gson builder = new GsonBuilder() + .registerTypeAdapter(Document.class, new DocumentTypeAdapter()) + .create(); + return builder.fromJson(this.readFileFromLocal(filePath), Document.class); + } + + private String readFileFromLocal(String filePath) throws IOException { + return FileUtils.readFileToString(new File(filePath), Charset.defaultCharset()); + } +} diff --git a/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapterTest.java b/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapterTest.java new file mode 100644 index 0000000..030bafd --- /dev/null +++ b/aws-kendra-datasource/src/test/java/software/amazon/kendra/datasource/utils/DocumentTypeAdapterTest.java @@ -0,0 +1,35 @@ +package software.amazon.kendra.datasource.utils; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.core.document.Document; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DocumentTypeAdapterTest { + + private static final String TYPE = "type"; + + private static final String SERIALIZED_DOCUMENT = "{\"type\":\"test\"}"; + private static final Document DOCUMENT = Document.mapBuilder().putString(TYPE, "test").build(); + private Gson gson; + + @BeforeEach + void setup() { + this.gson = new GsonBuilder() + .registerTypeAdapter(Document.class, new DocumentTypeAdapter()) + .create(); + } + + @Test + public void documentSerialization_valid_argument_success() { + assertEquals(SERIALIZED_DOCUMENT, gson.toJson(DOCUMENT, Document.class)); + } + + @Test + public void documentDeserialization_valid_argument_success() { + assertEquals(DOCUMENT, gson.fromJson(SERIALIZED_DOCUMENT, Document.class)); + } +}