Add s3 storage artifact route and ui integration of it

backend/package.json

@@ -50,6 +50,7 @@
     "http-errors": "^1.8.0",
     "js-yaml": "^4.0.0",
     "lodash": "^4.17.21",
+    "minio": "^7.1.3",
     "pino": "^8.11.0",
     "prom-client": "^14.0.1",
     "ts-node": "^10.9.1"

backend/src/routes/api/storage/index.ts

@@ -0,0 +1,50 @@
+import { FastifyInstance } from 'fastify';
+import { createMinioClient, getObjectStream } from './storageUtils';
+import { getDashboardConfig } from '../../../utils/resourceUtils';
+
+export default async (fastify: FastifyInstance): Promise<void> => {
+  const dashConfig = getDashboardConfig();
+  if (dashConfig?.spec?.dashboardConfig && !dashConfig.spec.dashboardConfig.disableS3Endpoint) {
+    fastify.get('/:namespace/:bucket', async (request, reply) => {
+      try {
+        const { namespace, bucket } = request.params as {
+          namespace: string;
+          bucket: string;
+        };
+        const query = request.query as { [key: string]: string };
+        const key = query.key;
+
+        const stream = await getObjectStream({
+          bucket,
+          client: await createMinioClient(fastify, namespace),
+          key,
+        });
+
+        reply.type('text/plain');
+
+        await new Promise<void>((resolve, reject) => {
+          stream.on('data', (chunk) => {
+            reply.raw.write(chunk);
+          });
+
+          stream.on('end', () => {
+            reply.raw.end();
+            resolve();
+          });
+
+          stream.on('error', (err) => {
+            fastify.log.error('Stream error:', err);
+            reply.raw.statusCode = 500;
+            reply.raw.end(err.message);
+            reject(err);
+          });
+        });
+
+        return;
+      } catch (err) {
+        reply.code(500).send(err.message);
+        return reply;
+      }
+    });
+  }
+};

backend/src/routes/api/storage/storageUtils.ts

@@ -0,0 +1,96 @@
+import { Client as MinioClient } from 'minio';
+import { DSPipelineKind, KubeFastifyInstance } from '../../../types';
+import { Transform, PassThrough } from 'stream';
+
+/**
+ * Create minio client with aws instance profile credentials if needed.
+ * @param config minio client options where `accessKey` and `secretKey` are optional.
+ */
+export async function createMinioClient(
+  fastify: KubeFastifyInstance,
+  namespace: string,
+): Promise<MinioClient> {
+  try {
+    const dspaResponse = await fastify.kube.customObjectsApi
+      .listNamespacedCustomObject(
+        'datasciencepipelinesapplications.opendatahub.io',
+        'v1alpha1',
+        namespace,
+        'datasciencepipelinesapplications',
+      )
+      .catch((e) => {
+        throw `A ${
+          e.statusCode
+        } error occurred when trying to fetch dspa aws storage credentials: ${
+          e.response?.body?.message || e?.response?.statusMessage
+        }`;
+      });
+
+    const dspas = (
+      dspaResponse?.body as {
+        items: DSPipelineKind[];
+      }
+    )?.items;
+
+    if (!dspas || !dspas.length) {
+      throw 'No Data Science Pipeline Application found';
+    }
+
+    // check if data connection is available
+    if (
+      dspas[0].status.conditions.find((condition) => condition.type === 'ObjectStoreAvailable')
+        .status !== 'True'
+    ) {
+      throw 'Object store is not available';
+    }
+
+    // always get the first one
+    const externalStorage = dspas[0].spec.objectStorage.externalStorage;
+
+    if (externalStorage) {
+      const { region, host: endPoint, s3CredentialsSecret } = externalStorage;
+
+      // get secret
+      const secret = await fastify.kube.coreV1Api.readNamespacedSecret(
+        s3CredentialsSecret.secretName,
+        namespace,
+      );
+      const accessKey = atob(secret.body.data[s3CredentialsSecret.accessKey]);
+      const secretKey = atob(secret.body.data[s3CredentialsSecret.secretKey]);
+
+      if (!accessKey || !secretKey) {
+        throw 'Access key or secret key is empty';
+      }
+
+      // sessionToken
+      return new MinioClient({ accessKey, secretKey, endPoint, region });
+    }
+  } catch (err) {
+    console.error('Unable to create minio client: ', err);
+    throw new Error('Unable to create minio client: ' + err);
+  }
+}
+
+/** MinioRequestConfig describes the info required to retrieve an artifact. */
+export interface MinioRequestConfig {
+  bucket: string;
+  key: string;
+  client: MinioClient;
+}
+
+/**
+ * Returns a stream from an object in a s3 compatible object store (e.g. minio).
+ *
+ * @param param.bucket Bucket name to retrieve the object from.
+ * @param param.key Key of the object to retrieve.
+ * @param param.client Minio client.
+ *
+ */
+export async function getObjectStream({
+  bucket,
+  key,
+  client,
+}: MinioRequestConfig): Promise<Transform> {
+  const stream = await client.getObject(bucket, key);
+  return stream.pipe(new PassThrough());
+}

backend/src/types.ts

@@ -37,6 +37,7 @@ export type DashboardConfig = K8sResourceCommon & {
       disableModelMesh: boolean;
       disableAcceleratorProfiles: boolean;
       disablePipelineExperiments: boolean;
+      disableS3Endpoint: boolean;
       disableDistributedWorkloads: boolean;
       disableModelRegistry: boolean;
     };
@@ -1014,9 +1015,83 @@ export type K8sCondition = {
   lastHeartbeatTime?: string;
 };
 
+export type DSPipelineExternalStorageKind = {
+  bucket: string;
+  host: string;
+  port?: '';
+  scheme: string;
+  region: string;
+  s3CredentialsSecret: {
+    accessKey: string;
+    secretKey: string;
+    secretName: string;
+  };
+};
+
 export type DSPipelineKind = K8sResourceCommon & {
+  metadata: {
+    name: string;
+    namespace: string;
+  };
   spec: {
     dspVersion: string;
+    apiServer?: Partial<{
+      apiServerImage: string;
+      artifactImage: string;
+      artifactScriptConfigMap: Partial<{
+        key: string;
+        name: string;
+      }>;
+      enableSamplePipeline: boolean;
+    }>;
+    database?: Partial<{
+      externalDB: Partial<{
+        host: string;
+        passwordSecret: Partial<{
+          key: string;
+          name: string;
+        }>;
+        pipelineDBName: string;
+        port: string;
+        username: string;
+      }>;
+      image: string;
+      mariaDB: Partial<{
+        image: string;
+        passwordSecret: Partial<{
+          key: string;
+          name: string;
+        }>;
+        pipelineDBName: string;
+        username: string;
+      }>;
+    }>;
+    mlpipelineUI?: {
+      configMap?: string;
+      image: string;
+    };
+    persistentAgent?: Partial<{
+      image: string;
+      pipelineAPIServerName: string;
+    }>;
+    scheduledWorkflow?: Partial<{
+      image: string;
+    }>;
+    objectStorage: Partial<{
+      externalStorage: DSPipelineExternalStorageKind;
+      minio: Partial<{
+        bucket: string;
+        image: string;
+        s3CredentialsSecret: Partial<{
+          accessKey: string;
+          secretKey: string;
+          secretName: string;
+        }>;
+      }>;
+    }>;
+    viewerCRD?: Partial<{
+      image: string;
+    }>;
   };
   status?: {
     conditions?: K8sCondition[];

backend/src/utils/constants.ts

@@ -62,6 +62,7 @@ export const blankDashboardCR: DashboardConfig = {
       disableModelMesh: false,
       disableAcceleratorProfiles: false,
       disablePipelineExperiments: true,
+      disableS3Endpoint: true,
       disableDistributedWorkloads: false,
       disableModelRegistry: true,
     },

docs/dashboard-config.md

@@ -61,7 +61,8 @@ spec:
     disableKServeMetrics: true
     disableBiasMetrics: false
     disablePerformanceMetrics: false
-    disablePipelineExperiments: false
+    disablePipelineExperiments: true
+    disableS3Endpoint: true
     disableDistributedWorkloads: false
 ```
 
@@ -157,6 +158,7 @@ spec:
     disableBiasMetrics: false
     disablePerformanceMetrics: false
     disablePipelineExperiments: true
+    disableS3Endpoint: true
   notebookController:
     enabled: true
     gpuSetting: autodetect

frontend/src/__mocks__/mockDashboardConfig.ts

@@ -22,6 +22,7 @@ type MockDashboardConfigType = {
   disablePerformanceMetrics?: boolean;
   disableBiasMetrics?: boolean;
   disablePipelineExperiments?: boolean;
+  disableS3Endpoint?: boolean;
   disableDistributedWorkloads?: boolean;
   disableModelRegistry?: boolean;
   disableNotebookController?: boolean;
@@ -49,6 +50,7 @@ export const mockDashboardConfig = ({
   disablePerformanceMetrics = false,
   disableBiasMetrics = false,
   disablePipelineExperiments = true,
+  disableS3Endpoint = true,
   disableDistributedWorkloads = false,
   disableModelRegistry = true,
   disableNotebookController = false,
@@ -87,6 +89,7 @@ export const mockDashboardConfig = ({
       disableModelMesh,
       disableAcceleratorProfiles,
       disablePipelineExperiments,
+      disableS3Endpoint,
       disableDistributedWorkloads,
       disableModelRegistry,
     },

frontend/src/concepts/areas/const.ts

@@ -73,6 +73,10 @@ export const SupportedAreasStateMap: SupportedAreasState = {
     featureFlags: ['disablePipelineExperiments'],
     reliantAreas: [SupportedArea.DS_PIPELINES],
   },
+  [SupportedArea.S3_ENDPOINT]: {
+    featureFlags: ['disableS3Endpoint'],
+    reliantAreas: [SupportedArea.DS_PIPELINES],
+  },
   [SupportedArea.DISTRIBUTED_WORKLOADS]: {
     featureFlags: ['disableDistributedWorkloads'],
     requiredComponents: [StackComponent.KUEUE],

frontend/src/concepts/areas/types.ts

@@ -24,12 +24,16 @@ export enum SupportedArea {
   HOME = 'home',
 
   /* Standalone areas */
-  DS_PIPELINES = 'ds-pipelines',
   // TODO: Jupyter Tile Support? (outside of feature flags today)
   WORKBENCHES = 'workbenches',
   // TODO: Support Applications/Tile area
   // TODO: Support resources area
 
+  /* Pipelines areas */
+  DS_PIPELINES = 'ds-pipelines',
+  PIPELINE_EXPERIMENTS = 'pipeline-experiments',
+  S3_ENDPOINT = 's3-endpoint',
+
   /* Admin areas */
   BYON = 'bring-your-own-notebook',
   CLUSTER_SETTINGS = 'cluster-settings',
@@ -50,7 +54,6 @@ export enum SupportedArea {
   BIAS_METRICS = 'bias-metrics',
   PERFORMANCE_METRICS = 'performance-metrics',
   TRUSTY_AI = 'trusty-ai',
-  PIPELINE_EXPERIMENTS = 'pipeline-experiments',
 
   /* Distributed Workloads areas */
   DISTRIBUTED_WORKLOADS = 'distributed-workloads',

frontend/src/concepts/pipelines/content/artifacts/ArtifactUriLink.tsx

@@ -0,0 +1,47 @@
+import React from 'react';
+import { Link } from 'react-router-dom';
+import { usePipelinesAPI } from '~/concepts/pipelines/context';
+import { useIsAreaAvailable, SupportedArea } from '~/concepts/areas';
+import { extractS3UriComponents } from './utils';
+
+interface ArtifactUriLinkProps {
+  uri?: string;
+}
+
+export const ArtifactUriLink: React.FC<ArtifactUriLinkProps> = ({ uri }) => {
+  const { namespace } = usePipelinesAPI();
+  const isS3EndpointAvailable = useIsAreaAvailable(SupportedArea.S3_ENDPOINT).status;
+
+  const url = React.useMemo(() => {
+    if (!uri || !isS3EndpointAvailable) {
+      return;
+    }
+
+    // Check if the uri starts with http or https return it as is
+    if (uri.startsWith('http://') || uri.startsWith('https://')) {
+      return uri;
+    }
+
+    // Otherwise check if the uri is s3
+    // If it is not s3, return undefined as we only support fetching from s3 buckets
+    const uriComponents = extractS3UriComponents(uri);
+    if (!uriComponents) {
+      return;
+    }
+
+    const { bucket, path } = uriComponents;
+
+    // /api/storage/${namespace}/${bucket}?key=${path}
+    return `/api/storage/${namespace}/${bucket}?key=${encodeURIComponent(path)}`;
+  }, [isS3EndpointAvailable, namespace, uri]);
+
+  if (!url) {
+    return uri;
+  }
+
+  return (
+    <Link target="_blank" rel="noopener noreferrer" to={url}>
+      {uri}
+    </Link>
+  );
+};

frontend/src/concepts/pipelines/content/artifacts/utils.ts

@@ -0,0 +1,13 @@
+export function extractS3UriComponents(uri: string): { bucket: string; path: string } | undefined {
+  const s3Prefix = 's3://';
+  if (!uri.startsWith(s3Prefix)) {
+    return;
+  }
+
+  const s3UrlWithoutPrefix = uri.slice(s3Prefix.length);
+  const firstSlashIndex = s3UrlWithoutPrefix.indexOf('/');
+  const bucket = s3UrlWithoutPrefix.substring(0, firstSlashIndex);
+  const path = s3UrlWithoutPrefix.substring(firstSlashIndex + 1);
+
+  return { bucket, path };
+}