From 3fee56d0d0a07e63b9ad60a78d723dc3f0de08d5 Mon Sep 17 00:00:00 2001
From: Anan Zhuang <ananzh@amazon.com>
Date: Thu, 21 Nov 2024 21:19:28 +0000
Subject: [PATCH] enable s3

Signed-off-by: Anan Zhuang <ananzh@amazon.com>
---
 .../public/datasets/s3_type.test.ts           | 58 +++++++++----------
 .../public/datasets/s3_type.ts                | 28 +++++----
 2 files changed, 45 insertions(+), 41 deletions(-)

diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts
index 1a6066f72f14..6a2d5cc6182c 100644
--- a/src/plugins/query_enhancements/public/datasets/s3_type.test.ts
+++ b/src/plugins/query_enhancements/public/datasets/s3_type.test.ts
@@ -141,7 +141,9 @@ describe('s3TypeConfig', () => {
 
     it('should fetch data sources for unknown type', async () => {
       mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-        savedObjects: [{ id: 'ds1', attributes: { title: 'DataSource 1' } }],
+        savedObjects: [
+          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '3.0' } },
+        ],
       });
 
       const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
@@ -152,37 +154,33 @@ describe('s3TypeConfig', () => {
       expect(result.children?.[0].title).toBe('DataSource 1');
       expect(result.hasNext).toBe(true);
     });
-  });
 
-  it('should NOT filter out data sources regardless of version', async () => {
-    mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
-      savedObjects: [
-        { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
-        {
-          id: 'ds2',
-          attributes: { title: 'DataSource 2', dataSourceVersion: '' }, // empty version
-        },
-        { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
-        {
-          id: 'ds4',
-          attributes: { title: 'DataSource 4', dataSourceVersion: '.0' }, // invalid version
-        },
-      ],
-    });
+    it('should filter out data sources with versions lower than 1.0.0', async () => {
+      mockSavedObjectsClient.find = jest.fn().mockResolvedValue({
+        savedObjects: [
+          { id: 'ds1', attributes: { title: 'DataSource 1', dataSourceVersion: '1.0' } },
+          {
+            id: 'ds2',
+            attributes: { title: 'DataSource 2', dataSourceVersion: '' },
+          },
+          { id: 'ds3', attributes: { title: 'DataSource 3', dataSourceVersion: '2.17.0' } },
+          {
+            id: 'ds4',
+            attributes: { title: 'DataSource 4', dataSourceVersion: '.0' },
+          },
+        ],
+      });
+
+      const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
+        { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
+      ]);
 
-    const result = await s3TypeConfig.fetch(mockServices as IDataPluginServices, [
-      { id: 'unknown', title: 'Unknown', type: 'UNKNOWN' },
-    ]);
-
-    // Verify all data sources are included
-    expect(result.children).toHaveLength(4);
-    expect(result.children?.map((child) => child.title)).toEqual([
-      'DataSource 1',
-      'DataSource 2',
-      'DataSource 3',
-      'DataSource 4',
-    ]);
-    expect(result.hasNext).toBe(true);
+      expect(result.children).toHaveLength(2);
+      expect(result.children?.[0].title).toBe('DataSource 1');
+      expect(result.children?.[1].title).toBe('DataSource 3');
+      expect(result.children?.some((child) => child.title === 'DataSource 2')).toBe(false);
+      expect(result.hasNext).toBe(true);
+    });
   });
 
   test('fetchFields returns table fields', async () => {
diff --git a/src/plugins/query_enhancements/public/datasets/s3_type.ts b/src/plugins/query_enhancements/public/datasets/s3_type.ts
index 4e8c41959f2d..c13b5e898670 100644
--- a/src/plugins/query_enhancements/public/datasets/s3_type.ts
+++ b/src/plugins/query_enhancements/public/datasets/s3_type.ts
@@ -6,6 +6,7 @@
 import { i18n } from '@osd/i18n';
 import { trimEnd } from 'lodash';
 import { HttpSetup, SavedObjectsClientContract } from 'opensearch-dashboards/public';
+import semver from 'semver';
 import {
   DATA_STRUCTURE_META_TYPES,
   DEFAULT_DATA,
@@ -197,17 +198,22 @@ const fetchDataSources = async (client: SavedObjectsClientContract): Promise<Dat
     type: 'data-source',
     perPage: 10000,
   });
-  const dataSources: DataStructure[] = resp.savedObjects.map((savedObject) => ({
-    id: savedObject.id,
-    title: savedObject.attributes.title,
-    type: 'DATA_SOURCE',
-    meta: {
-      query: {
-        id: savedObject.id,
-      },
-      type: DATA_STRUCTURE_META_TYPES.CUSTOM,
-    } as DataStructureCustomMeta,
-  }));
+  const dataSources: DataStructure[] = resp.savedObjects
+    .filter((savedObject) => {
+      const coercedVersion = semver.coerce(savedObject.attributes.dataSourceVersion);
+      return coercedVersion ? semver.satisfies(coercedVersion, '>=1.0.0') : false;
+    })
+    .map((savedObject) => ({
+      id: savedObject.id,
+      title: savedObject.attributes.title,
+      type: 'DATA_SOURCE',
+      meta: {
+        query: {
+          id: savedObject.id,
+        },
+        type: DATA_STRUCTURE_META_TYPES.CUSTOM,
+      } as DataStructureCustomMeta,
+    }));
   return dataSources;
 };