Add Azure ingestion flow to web console (apache#9437)

* add support for azure * change bucket to container * add azure to input menu * remove static-azure
himanshug · Mar 3, 2020 · 99095c4 · 99095c4
1 parent 1fd865b
commit 99095c4
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 1 deletion.
diff --git a/web-console/assets/azure.png b/web-console/assets/azure.png
diff --git a/web-console/src/utils/ingestion-spec.tsx b/web-console/src/utils/ingestion-spec.tsx
@@ -61,6 +61,7 @@ export type IngestionComboType =
   | 'index_parallel:druid'
   | 'index_parallel:inline'
   | 'index_parallel:s3'
+  | 'index_parallel:azure'
   | 'index_parallel:google'
   | 'index_parallel:hdfs';
 
@@ -102,6 +103,7 @@ export function getIngestionComboType(spec: IngestionSpec): IngestionComboType |
         case 'druid':
         case 'inline':
         case 's3':
+        case 'azure':
         case 'google':
         case 'hdfs':
           return `${ioConfig.type}:${inputSource.type}` as IngestionComboType;
@@ -128,6 +130,9 @@ export function getIngestionTitle(ingestionType: IngestionComboTypeWithExtra): s
     case 'index_parallel:s3':
       return 'Amazon S3';
 
+    case 'index_parallel:azure':
+      return 'Azure Data Lake';
+
     case 'index_parallel:google':
       return 'Google Cloud Storage';
 
@@ -180,6 +185,9 @@ export function getRequiredModule(ingestionType: IngestionComboTypeWithExtra): s
     case 'index_parallel:s3':
       return 'druid-s3-extensions';
 
+    case 'index_parallel:azure':
+      return 'druid-azure-extensions';
+
     case 'index_parallel:google':
       return 'druid-google-extensions';
 
@@ -983,7 +991,7 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
     name: 'inputSource.type',
     label: 'Source type',
     type: 'string',
-    suggestions: ['local', 'http', 'inline', 's3', 'google', 'hdfs'],
+    suggestions: ['local', 'http', 'inline', 's3', 'azure', 'google', 'hdfs'],
     info: (
       <p>
         Druid connects to raw data through{' '}
@@ -1208,6 +1216,67 @@ export function getIoConfigFormFields(ingestionComboType: IngestionComboType): F
         },
       ];
 
+    case 'index_parallel:azure':
+      return [
+        inputSourceType,
+        {
+          name: 'inputSource.uris',
+          label: 'Azure URIs',
+          type: 'string-array',
+          placeholder:
+            'azure://your-container/some-file1.ext, azure://your-container/some-file2.ext',
+          required: true,
+          defined: ioConfig =>
+            !deepGet(ioConfig, 'inputSource.prefixes') && !deepGet(ioConfig, 'inputSource.objects'),
+          info: (
+            <>
+              <p>
+                The full Azure URI of your file. To ingest from multiple URIs, use commas to
+                separate each individual URI.
+              </p>
+              <p>Either Azure URIs or prefixes or objects must be set.</p>
+            </>
+          ),
+        },
+        {
+          name: 'inputSource.prefixes',
+          label: 'Azure prefixes',
+          type: 'string-array',
+          placeholder: 'azure://your-container/some-path1, azure://your-container/some-path2',
+          required: true,
+          defined: ioConfig =>
+            !deepGet(ioConfig, 'inputSource.uris') && !deepGet(ioConfig, 'inputSource.objects'),
+          info: (
+            <>
+              <p>A list of paths (with bucket) where your files are stored.</p>
+              <p>Either Azure URIs or prefixes or objects must be set.</p>
+            </>
+          ),
+        },
+        {
+          name: 'inputSource.objects',
+          label: 'Azure objects',
+          type: 'json',
+          placeholder: '{"bucket":"your-container", "path":"some-file.ext"}',
+          required: true,
+          defined: ioConfig => deepGet(ioConfig, 'inputSource.objects'),
+          info: (
+            <>
+              <p>
+                JSON array of{' '}
+                <ExternalLink
+                  href={`https://druid.apache.org/docs/${DRUID_DOCS_VERSION}/development/extensions-core/azure.html`}
+                >
+                  S3 Objects
+                </ExternalLink>
+                .
+              </p>
+              <p>Either Azure URIs or prefixes or objects must be set.</p>
+            </>
+          ),
+        },
+      ];
+
     case 'index_parallel:google':
       return [
         inputSourceType,
@@ -1423,6 +1492,7 @@ function issueWithInputSource(inputSource: InputSource | undefined): string | un
       break;
 
     case 's3':
+    case 'azure':
     case 'google':
       if (
         !nonEmptyArray(inputSource.uris) &&
@@ -1478,6 +1548,7 @@ export function getIoConfigTuningFormFields(
   switch (ingestionComboType) {
     case 'index_parallel:http':
     case 'index_parallel:s3':
+    case 'index_parallel:azure':
     case 'index_parallel:google':
     case 'index_parallel:hdfs':
       return [
@@ -1815,6 +1886,7 @@ export function guessDataSourceName(spec: IngestionSpec): string | undefined {
           }
 
         case 's3':
+        case 'azure':
         case 'google':
           const actualPath = (inputSource.objects || EMPTY_ARRAY)[0];
           const uriPath =

diff --git a/web-console/src/views/load-data-view/load-data-view.tsx b/web-console/src/views/load-data-view/load-data-view.tsx
@@ -704,6 +704,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
           {this.renderIngestionCard('kafka')}
           {this.renderIngestionCard('kinesis')}
           {this.renderIngestionCard('index_parallel:s3')}
+          {this.renderIngestionCard('index_parallel:azure')}
           {this.renderIngestionCard('index_parallel:google')}
           {this.renderIngestionCard('index_parallel:hdfs')}
           {this.renderIngestionCard('index_parallel:druid')}
@@ -781,6 +782,9 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       case 'index_parallel:s3':
         return <p>Load text based, orc, or parquet data from Amazon S3.</p>;
 
+      case 'index_parallel:azure':
+        return <p>Load text based, orc, or parquet data from Azure.</p>;
+
       case 'index_parallel:google':
         return <p>Load text based, orc, or parquet data from the Google Blobstore.</p>;
 
@@ -831,6 +835,7 @@ export class LoadDataView extends React.PureComponent<LoadDataViewProps, LoadDat
       case 'index_parallel:druid':
       case 'index_parallel:inline':
       case 'index_parallel:s3':
+      case 'index_parallel:azure':
       case 'index_parallel:google':
       case 'index_parallel:hdfs':
       case 'kafka':