feat: Multi bucket support (#1742)

* feat: init multi-bucket support

* refactor: BackendOutput type

* update package-lock

* temp

* feat: update schema and client_config

* feat: increment schema and client_config versions to 1.1

* rename to buckets and remove friendlyName

* chore: add changeset

* chore: update API.md

* fix: remove duplicated code

* fix: construct test

* feat: validate isDefault

* chore: remove as any

* test: fix client_config_contributor

* chore: update package-lock

* fix: getOutput

* refactor: backend_output_client

* refactor: DefaultBackendOutputClient

* revert changes on addBackendOutputEntry, use appendToBackendOutputList

* update JSdocs for construct

* make storageOutput more specific

* add factoryCounter and hasDefault

* use factoryCounter as postfix

* update changeset

* fix: buckets output schema

* feat: add name to buckets

* feat: throw no isdefault error before deploy

* chore: remove post-deploy error

* refactor: use factoryCounter for policy

* refactor: pass buckets to appendToBackendOutputList

* remove StorageOutputPayloadToStore

* fix construct test

* revert BackendOutput type

* chore: update changeset

* fix: client_config_contributer_v1 and test

* fix: pin api-extractor to 7.40.0

* chore: update API.md

* refactor: use global factoryCounter for policy

* refactor: use addBackendOutputEntry for default

* test: fix storage construct test

* this works

* move DeepPartial

* remove StorageBucketsPayload

* refactor: use Aspects to validate storage

* fix: add buckets to metadata

* fix: factory unit test

* fix: metadata_output_storage_strategy unit test

* fix: not export AmplifyStorage

* refactor: addOrUpdateMetadata

* update package-lock

* update package-lock

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Amplifiyer <[email protected]>

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Amplifiyer <[email protected]>

* refactor: remove redundant code in StorageClientConfigContributor

* test: add unit tests for construct and factory

* fix: one bucket no default case

* fix: unit test

* test log currentCodebaseOutputs and npmOutputs

* fix: convert bucketName, storageRegion into bucket_name, aws_region

* fix: client_config_contributor_v1.test

* refactor: use Aspect to replact factory static

* fix: buckets keys

* fix: unit test

* fix MultipleDefaultBucketError and add more unit tests

* remove policyCount static

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Amplifiyer <[email protected]>

* refactor StorageValidator

* Refactor: storeOutput to factory

* remove comment

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Kamil Sobol <[email protected]>

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Kamil Sobol <[email protected]>

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Kamil Sobol <[email protected]>

* Update packages/backend-storage/src/construct.ts

Co-authored-by: Kamil Sobol <[email protected]>

* Revert "test log currentCodebaseOutputs and npmOutputs"

This reverts commit 4ce8db5.

* rename defaultStorageFound

* remove firstStorage from getInstance

* rename buckets to storage, and fix unit tests

* use isStorageProcessed

* fix construct test

* fix StackMetadataBackendOutputStorageStrategy unit test

* fix a typo to make lint happy

* Update packages/backend-storage/src/factory.ts

Co-authored-by: Amplifiyer <[email protected]>

* Update packages/client-config/src/client-config-contributor/client_config_contributor_v1.ts

Co-authored-by: Edward Foyle <[email protected]>

* fix type error in client_config_contributer_v1

* refactor: use early return

* test: fix test name

* refactor: move Aspects to another file

* refactor: remove this.node

* Update packages/backend-storage/src/storage_outputs_aspect.ts

Co-authored-by: Amplifiyer <[email protected]>

* add isMatch to match outputs

* add unit test for aspects

---------

Co-authored-by: Kamil Sobol <[email protected]>
Co-authored-by: Amplifiyer <[email protected]>
Co-authored-by: Kamil Sobol <[email protected]>
Co-authored-by: Edward Foyle <[email protected]>

.changeset/sharp-penguins-impress.md

@@ -0,0 +1,11 @@
+---
+'@aws-amplify/deployed-backend-client': minor
+'@aws-amplify/backend-output-schemas': minor
+'@aws-amplify/backend-output-storage': minor
+'@aws-amplify/backend-storage': minor
+'@aws-amplify/client-config': minor
+'@aws-amplify/plugin-types': minor
+'@aws-amplify/backend': minor
+---
+
+support adding more than one bucket

packages/backend-output-schemas/API.md

@@ -275,24 +275,29 @@ export const unifiedBackendOutputSchema: z.ZodObject<{
         payload: z.ZodObject<{
             bucketName: z.ZodString;
             storageRegion: z.ZodString;
+            buckets: z.ZodOptional<z.ZodString>;
         }, "strip", z.ZodTypeAny, {
             bucketName: string;
             storageRegion: string;
+            buckets?: string | undefined;
         }, {
             bucketName: string;
             storageRegion: string;
+            buckets?: string | undefined;
         }>;
     }, "strip", z.ZodTypeAny, {
         version: "1";
         payload: {
             bucketName: string;
             storageRegion: string;
+            buckets?: string | undefined;
         };
     }, {
         version: "1";
         payload: {
             bucketName: string;
             storageRegion: string;
+            buckets?: string | undefined;
         };
     }>]>>;
     "AWS::Amplify::Custom": z.ZodOptional<z.ZodDiscriminatedUnion<"version", [z.ZodObject<{
@@ -391,6 +396,7 @@ export const unifiedBackendOutputSchema: z.ZodObject<{
         payload: {
             bucketName: string;
             storageRegion: string;
+            buckets?: string | undefined;
         };
     } | undefined;
     "AWS::Amplify::Function"?: {
@@ -455,6 +461,7 @@ export const unifiedBackendOutputSchema: z.ZodObject<{
         payload: {
             bucketName: string;
             storageRegion: string;
+            buckets?: string | undefined;
         };
     } | undefined;
     "AWS::Amplify::Function"?: {
@@ -684,24 +691,29 @@ export const versionedStorageOutputSchema: z.ZodDiscriminatedUnion<"version", [z
     payload: z.ZodObject<{
         bucketName: z.ZodString;
         storageRegion: z.ZodString;
+        buckets: z.ZodOptional<z.ZodString>;
     }, "strip", z.ZodTypeAny, {
         bucketName: string;
         storageRegion: string;
+        buckets?: string | undefined;
     }, {
         bucketName: string;
         storageRegion: string;
+        buckets?: string | undefined;
     }>;
 }, "strip", z.ZodTypeAny, {
     version: "1";
     payload: {
         bucketName: string;
         storageRegion: string;
+        buckets?: string | undefined;
     };
 }, {
     version: "1";
     payload: {
         bucketName: string;
         storageRegion: string;
+        buckets?: string | undefined;
     };
 }>]>;
 

packages/backend-output-schemas/src/storage/v1.ts

@@ -1,9 +1,16 @@
 import { z } from 'zod';
 
+const bucketSchema = z.object({
+  name: z.string(),
+  bucketName: z.string(),
+  storageRegion: z.string(),
+});
+
 export const storageOutputSchema = z.object({
   version: z.literal('1'),
   payload: z.object({
     bucketName: z.string(),
     storageRegion: z.string(),
+    buckets: z.string(z.array(bucketSchema)).optional(), // JSON serialized array of bucketSchema
   }),
 });

packages/backend-output-storage/API.md

@@ -8,6 +8,7 @@
 
 import { BackendOutputEntry } from '@aws-amplify/plugin-types';
 import { BackendOutputStorageStrategy } from '@aws-amplify/plugin-types';
+import { DeepPartial } from '@aws-amplify/plugin-types';
 import * as _os from 'os';
 import { PackageJsonReader } from '@aws-amplify/platform-core';
 import { Stack } from 'aws-cdk-lib';
@@ -37,7 +38,7 @@ export type Platform = 'Mac' | 'Windows' | 'Linux' | 'Other';
 export class StackMetadataBackendOutputStorageStrategy implements BackendOutputStorageStrategy<BackendOutputEntry> {
     constructor(stack: Stack);
     addBackendOutputEntry: (keyName: string, backendOutputEntry: BackendOutputEntry) => void;
-    appendToBackendOutputList: (keyName: string, backendOutputEntry: BackendOutputEntry) => void;
+    appendToBackendOutputList: (keyName: string, backendOutputEntry: DeepPartial<BackendOutputEntry>) => void;
 }
 
 // (No @packageDocumentation comment for this package)

packages/backend-output-storage/src/stack_metadata_output_storage_strategy.test.ts

@@ -16,17 +16,18 @@ void describe('StackMetadataBackendOutputStorageStrategy', () => {
       outputStorage.addBackendOutputEntry('TestStorageOutput', {
         version: '1',
         payload: {
-          something: 'special',
+          bucketName: 'test-bucket',
+          storageRegion: 'us-west-2',
         },
       });
 
       const template = Template.fromStack(stack);
-      template.hasOutput('something', { Value: 'special' });
+      template.hasOutput('bucketName', { Value: 'test-bucket' });
       template.templateMatches({
         Metadata: {
           TestStorageOutput: {
             version: '1',
-            stackOutputs: ['something'],
+            stackOutputs: ['bucketName', 'storageRegion'],
           },
         },
       });
@@ -41,7 +42,8 @@ void describe('StackMetadataBackendOutputStorageStrategy', () => {
       outputStorage.addBackendOutputEntry('TestStorageOutput', {
         version: '44',
         payload: {
-          something: 'special',
+          bucketName: 'test-bucket',
+          storageRegion: 'us-west-2',
         },
       });
 
@@ -61,17 +63,19 @@ void describe('StackMetadataBackendOutputStorageStrategy', () => {
       outputStorage.appendToBackendOutputList('TestStorageOutput', {
         version: '1',
         payload: {
-          something: 'special',
+          buckets: 'test-bucket',
         },
       });
 
       const template = Template.fromStack(stack);
-      template.hasOutput('something', { Value: JSON.stringify(['special']) });
+      template.hasOutput('buckets', {
+        Value: JSON.stringify(['test-bucket']),
+      });
       template.templateMatches({
         Metadata: {
           TestStorageOutput: {
             version: '1',
-            stackOutputs: ['something'],
+            stackOutputs: ['buckets'],
           },
         },
       });
@@ -86,57 +90,93 @@ void describe('StackMetadataBackendOutputStorageStrategy', () => {
       outputStorage.appendToBackendOutputList('TestStorageOutput', {
         version: '1',
         payload: {
-          something: 'special',
+          buckets: JSON.stringify({
+            name: 'test-bucket',
+            bucketName: 'test-bucket',
+            storageRegion: 'us-west-2',
+          }),
+        },
+      });
+      outputStorage.addBackendOutputEntry('TestStorageOutput', {
+        version: '1',
+        payload: {
+          bucketName: 'test-bucket-two',
+          storageRegion: 'us-west-2',
         },
       });
       outputStorage.appendToBackendOutputList('TestStorageOutput', {
         version: '1',
         payload: {
-          something: 'otherSpecial',
+          buckets: JSON.stringify({
+            name: 'test-bucket-two',
+            bucketName: 'test-bucket-two',
+            storageRegion: 'us-west-2',
+          }),
         },
       });
       const template = Template.fromStack(stack);
-      template.hasOutput('something', {
-        Value: JSON.stringify(['special', 'otherSpecial']),
+      template.hasOutput('buckets', {
+        Value: JSON.stringify([
+          '{"name":"test-bucket","bucketName":"test-bucket","storageRegion":"us-west-2"}',
+          '{"name":"test-bucket-two","bucketName":"test-bucket-two","storageRegion":"us-west-2"}',
+        ]),
       });
       template.templateMatches({
         Metadata: {
           TestStorageOutput: {
             version: '1',
-            stackOutputs: ['something'],
+            stackOutputs: ['buckets', 'bucketName', 'storageRegion'],
           },
         },
       });
     });
 
-    void it('appends a cdk token to an existing list in stack output', () => {
+    void it('appends a cdk token to an existing list in stack output with two buckets', () => {
       const testToken = Token.asString('testToken');
       const app = new App();
       const stack = new Stack(app);
       const outputStorage = new StackMetadataBackendOutputStorageStrategy(
         stack
       );
+      outputStorage.addBackendOutputEntry('TestStorageOutput', {
+        version: '1',
+        payload: {
+          bucketName: testToken,
+          storageRegion: 'us-west-2',
+        },
+      });
       outputStorage.appendToBackendOutputList('TestStorageOutput', {
         version: '1',
         payload: {
-          something: 'special',
+          buckets: JSON.stringify({
+            name: testToken,
+            bucketName: testToken,
+            storageRegion: 'us-west-2',
+          }),
         },
       });
       outputStorage.appendToBackendOutputList('TestStorageOutput', {
         version: '1',
         payload: {
-          something: testToken,
+          buckets: JSON.stringify({
+            name: 'test-bucket-two',
+            bucketName: 'test-bucket-two',
+            storageRegion: 'us-west-2',
+          }),
         },
       });
       const template = Template.fromStack(stack);
-      template.hasOutput('something', {
-        Value: JSON.stringify(['special', 'testToken']),
+      template.hasOutput('buckets', {
+        Value: JSON.stringify([
+          '{"name":"testToken","bucketName":"testToken","storageRegion":"us-west-2"}',
+          '{"name":"test-bucket-two","bucketName":"test-bucket-two","storageRegion":"us-west-2"}',
+        ]),
       });
       template.templateMatches({
         Metadata: {
           TestStorageOutput: {
             version: '1',
-            stackOutputs: ['something'],
+            stackOutputs: ['buckets', 'bucketName', 'storageRegion'],
           },
         },
       });
@@ -151,7 +191,7 @@ void describe('StackMetadataBackendOutputStorageStrategy', () => {
       outputStorage.appendToBackendOutputList('TestStorageOutput', {
         version: '1',
         payload: {
-          something: 'special',
+          buckets: 'test-bucket',
         },
       });
 
@@ -160,7 +200,8 @@ void describe('StackMetadataBackendOutputStorageStrategy', () => {
           outputStorage.appendToBackendOutputList('TestStorageOutput', {
             version: '2',
             payload: {
-              something: 'otherSpecial',
+              bucketName: 'test-bucket',
+              storageRegion: 'us-west-2',
             },
           }),
         {