Add s3 storage artifact route and ui integration of it

backend/package.json

@@ -50,6 +50,7 @@
     "http-errors": "^1.8.0",
     "js-yaml": "^4.0.0",
     "lodash": "^4.17.21",
+    "minio": "^7.1.3",
     "pino": "^8.11.0",
     "prom-client": "^14.0.1",
     "ts-node": "^10.9.1"

backend/src/routes/api/storage/index.ts

@@ -0,0 +1,63 @@
+import { FastifyInstance, FastifyReply } from 'fastify';
+import { createMinioClient, getObjectStream } from './storageUtils';
+import { getDashboardConfig } from '../../../utils/resourceUtils';
+import { getDirectCallOptions } from '../../../utils/directCallUtils';
+import { getAccessToken } from '../../../utils/directCallUtils';
+import { OauthFastifyRequest } from '../../../types';
+
+export default async (fastify: FastifyInstance): Promise<void> => {
+  const dashConfig = getDashboardConfig();
+  if (dashConfig?.spec.dashboardConfig.disableS3Endpoint === false) {
+    fastify.get(
+      '/:namespace/:bucket',
+      async (
+        request: OauthFastifyRequest<{
+          Querystring: Record<string, string>;
+          Params: { '*': string; [key: string]: string };
+          Body: { [key: string]: unknown };
+        }>,
+        reply: FastifyReply,
+      ) => {
+        try {
+          const { namespace, bucket } = request.params;
+          const query = request.query;
+          const key = query.key;
+
+          const requestOptions = await getDirectCallOptions(fastify, request, request.url);
+          const token = getAccessToken(requestOptions);
+
+          const stream = await getObjectStream({
+            bucket,
+            client: await createMinioClient(fastify, token, namespace),
+            key,
+          });
+
+          reply.type('text/plain');
+
+          await new Promise<void>((resolve, reject) => {
+            stream.on('data', (chunk) => {
+              reply.raw.write(chunk);
+            });
+
+            stream.on('end', () => {
+              reply.raw.end();
+              resolve();
+            });
+
+            stream.on('error', (err) => {
+              fastify.log.error('Stream error:', err);
+              reply.raw.statusCode = 500;
+              reply.raw.end(err.message);
+              reject(err);
+            });
+          });
+
+          return;
+        } catch (err) {
+          reply.code(500).send(err.message);
+          return reply;
+        }
+      },
+    );
+  }
+};

backend/src/routes/api/storage/storageUtils.ts

@@ -0,0 +1,118 @@
+import { Client as MinioClient } from 'minio';
+import { DSPipelineKind, KubeFastifyInstance } from '../../../types';
+import { Transform, PassThrough } from 'stream';
+
+/**
+ * Create minio client with aws instance profile credentials if needed.
+ * @param config minio client options where `accessKey` and `secretKey` are optional.
+ */
+export async function createMinioClient(
+  fastify: KubeFastifyInstance,
+  token: string,
+  namespace: string,
+): Promise<MinioClient> {
+  try {
+    const dspaResponse = await fastify.kube.customObjectsApi
+      .listNamespacedCustomObject(
+        'datasciencepipelinesapplications.opendatahub.io',
+        'v1alpha1',
+        namespace,
+        'datasciencepipelinesapplications',
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        {
+          headers: {
+            authorization: `Bearer ${token}`,
+          },
+        },
+      )
+      .catch((e) => {
+        throw `A ${
+          e.statusCode
+        } error occurred when trying to fetch dspa aws storage credentials: ${
+          e.response?.body?.message || e?.response?.statusMessage
+        }`;
+      });
+
+    const dspas = (
+      dspaResponse?.body as {
+        items: DSPipelineKind[];
+      }
+    )?.items;
+
+    if (!dspas || !dspas.length) {
+      throw 'No Data Science Pipeline Application found';
+    }
+
+    // check if data connection is available
+    if (
+      dspas[0].status.conditions.find((condition) => condition.type === 'ObjectStoreAvailable')
+        .status !== 'True'
+    ) {
+      throw 'Object store is not available';
+    }
+
+    // always get the first one
+    const externalStorage = dspas[0].spec.objectStorage.externalStorage;
+
+    if (externalStorage) {
+      const { region, host: endPoint, s3CredentialsSecret } = externalStorage;
+
+      // get secret
+      const secret = await fastify.kube.coreV1Api.readNamespacedSecret(
+        s3CredentialsSecret.secretName,
+        namespace,
+        undefined,
+        undefined,
+        undefined,
+        {
+          headers: {
+            authorization: `Bearer ${token}`,
+          },
+        },
+      );
+      const accessKey = atob(secret.body.data[s3CredentialsSecret.accessKey]);
+      const secretKey = atob(secret.body.data[s3CredentialsSecret.secretKey]);
+
+      if (!accessKey || !secretKey) {
+        throw 'Access key or secret key is empty';
+      }
+
+      // sessionToken
+      return new MinioClient({ accessKey, secretKey, endPoint, region });
+    }
+  } catch (err) {
+    console.error('Unable to create minio client: ', err);
+    throw new Error('Unable to create minio client: ' + err);
+  }
+}
+
+/** MinioRequestConfig describes the info required to retrieve an artifact. */
+export interface MinioRequestConfig {
+  bucket: string;
+  key: string;
+  client: MinioClient;
+}
+
+/**
+ * Returns a stream from an object in a s3 compatible object store (e.g. minio).
+ *
+ * @param param.bucket Bucket name to retrieve the object from.
+ * @param param.key Key of the object to retrieve.
+ * @param param.client Minio client.
+ *
+ */
+export async function getObjectStream({
+  bucket,
+  key,
+  client,
+}: MinioRequestConfig): Promise<Transform> {
+  const stream = await client.getObject(bucket, key);
+  return stream.pipe(new PassThrough());
+}

backend/src/types.ts

@@ -37,6 +37,7 @@ export type DashboardConfig = K8sResourceCommon & {
       disableModelMesh: boolean;
       disableAcceleratorProfiles: boolean;
       disablePipelineExperiments: boolean;
+      disableS3Endpoint: boolean;
       disableDistributedWorkloads: boolean;
       disableModelRegistry: boolean;
     };
@@ -1014,9 +1015,83 @@ export type K8sCondition = {
   lastHeartbeatTime?: string;
 };
 
+export type DSPipelineExternalStorageKind = {
+  bucket: string;
+  host: string;
+  port?: '';
+  scheme: string;
+  region: string;
+  s3CredentialsSecret: {
+    accessKey: string;
+    secretKey: string;
+    secretName: string;
+  };
+};
+
 export type DSPipelineKind = K8sResourceCommon & {
+  metadata: {
+    name: string;
+    namespace: string;
+  };
   spec: {
     dspVersion: string;
+    apiServer?: Partial<{
+      apiServerImage: string;
+      artifactImage: string;
+      artifactScriptConfigMap: Partial<{
+        key: string;
+        name: string;
+      }>;
+      enableSamplePipeline: boolean;
+    }>;
+    database?: Partial<{
+      externalDB: Partial<{
+        host: string;
+        passwordSecret: Partial<{
+          key: string;
+          name: string;
+        }>;
+        pipelineDBName: string;
+        port: string;
+        username: string;
+      }>;
+      image: string;
+      mariaDB: Partial<{
+        image: string;
+        passwordSecret: Partial<{
+          key: string;
+          name: string;
+        }>;
+        pipelineDBName: string;
+        username: string;
+      }>;
+    }>;
+    mlpipelineUI?: {
+      configMap?: string;
+      image: string;
+    };
+    persistentAgent?: Partial<{
+      image: string;
+      pipelineAPIServerName: string;
+    }>;
+    scheduledWorkflow?: Partial<{
+      image: string;
+    }>;
+    objectStorage: Partial<{
+      externalStorage: DSPipelineExternalStorageKind;
+      minio: Partial<{
+        bucket: string;
+        image: string;
+        s3CredentialsSecret: Partial<{
+          accessKey: string;
+          secretKey: string;
+          secretName: string;
+        }>;
+      }>;
+    }>;
+    viewerCRD?: Partial<{
+      image: string;
+    }>;
   };
   status?: {
     conditions?: K8sCondition[];

backend/src/utils/constants.ts

@@ -62,6 +62,7 @@ export const blankDashboardCR: DashboardConfig = {
       disableModelMesh: false,
       disableAcceleratorProfiles: false,
       disablePipelineExperiments: true,
+      disableS3Endpoint: true,
       disableDistributedWorkloads: false,
       disableModelRegistry: true,
     },

docs/dashboard-config.md

@@ -61,7 +61,8 @@ spec:
     disableKServeMetrics: true
     disableBiasMetrics: false
     disablePerformanceMetrics: false
-    disablePipelineExperiments: false
+    disablePipelineExperiments: true
+    disableS3Endpoint: true
     disableDistributedWorkloads: false
 ```
 
@@ -157,6 +158,7 @@ spec:
     disableBiasMetrics: false
     disablePerformanceMetrics: false
     disablePipelineExperiments: true
+    disableS3Endpoint: true
   notebookController:
     enabled: true
     gpuSetting: autodetect

frontend/src/__mocks__/mockDashboardConfig.ts

@@ -22,6 +22,7 @@ type MockDashboardConfigType = {
   disablePerformanceMetrics?: boolean;
   disableBiasMetrics?: boolean;
   disablePipelineExperiments?: boolean;
+  disableS3Endpoint?: boolean;
   disableDistributedWorkloads?: boolean;
   disableModelRegistry?: boolean;
   disableNotebookController?: boolean;
@@ -49,6 +50,7 @@ export const mockDashboardConfig = ({
   disablePerformanceMetrics = false,
   disableBiasMetrics = false,
   disablePipelineExperiments = true,
+  disableS3Endpoint = true,
   disableDistributedWorkloads = false,
   disableModelRegistry = true,
   disableNotebookController = false,
@@ -87,6 +89,7 @@ export const mockDashboardConfig = ({
       disableModelMesh,
       disableAcceleratorProfiles,
       disablePipelineExperiments,
+      disableS3Endpoint,
       disableDistributedWorkloads,
       disableModelRegistry,
     },

frontend/src/concepts/areas/const.ts

@@ -73,6 +73,10 @@ export const SupportedAreasStateMap: SupportedAreasState = {
     featureFlags: ['disablePipelineExperiments'],
     reliantAreas: [SupportedArea.DS_PIPELINES],
   },
+  [SupportedArea.S3_ENDPOINT]: {
+    featureFlags: ['disableS3Endpoint'],
+    reliantAreas: [SupportedArea.DS_PIPELINES],
+  },
   [SupportedArea.DISTRIBUTED_WORKLOADS]: {
     featureFlags: ['disableDistributedWorkloads'],
     requiredComponents: [StackComponent.KUEUE],

frontend/src/concepts/areas/types.ts

@@ -24,12 +24,16 @@ export enum SupportedArea {
   HOME = 'home',
 
   /* Standalone areas */
-  DS_PIPELINES = 'ds-pipelines',
   // TODO: Jupyter Tile Support? (outside of feature flags today)
   WORKBENCHES = 'workbenches',
   // TODO: Support Applications/Tile area
   // TODO: Support resources area
 
+  /* Pipelines areas */
+  DS_PIPELINES = 'ds-pipelines',
+  PIPELINE_EXPERIMENTS = 'pipeline-experiments',
+  S3_ENDPOINT = 's3-endpoint',
+
   /* Admin areas */
   BYON = 'bring-your-own-notebook',
   CLUSTER_SETTINGS = 'cluster-settings',
@@ -50,7 +54,6 @@ export enum SupportedArea {
   BIAS_METRICS = 'bias-metrics',
   PERFORMANCE_METRICS = 'performance-metrics',
   TRUSTY_AI = 'trusty-ai',
-  PIPELINE_EXPERIMENTS = 'pipeline-experiments',
 
   /* Distributed Workloads areas */
   DISTRIBUTED_WORKLOADS = 'distributed-workloads',