Skip to content

Commit

Permalink
WIP: S3 improvements (#16167)
Browse files Browse the repository at this point in the history
  • Loading branch information
@cirospaciari
cirospaciari authored Jan 5, 2025
1 parent 8a469cc commit 034f776
Show file tree
Hide file tree
Showing 21 changed files with 2,974 additions and 2,673 deletions.
6 changes: 3 additions & 3 deletions src/bun.js/api/server.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ const linux = std.os.linux;
const Async = bun.Async;
const httplog = Output.scoped(.Server, false);
const ctxLog = Output.scoped(.RequestContext, false);
const AWS = @import("../../s3.zig").AWSCredentials;
const S3 = bun.S3;
const BlobFileContentResult = struct {
data: [:0]const u8,

Expand Down Expand Up @@ -3182,7 +3182,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
this.endWithoutBody(this.shouldCloseConnection());
this.deref();
}
pub fn onS3SizeResolved(result: AWS.S3StatResult, this: *RequestContext) void {
pub fn onS3SizeResolved(result: S3.S3StatResult, this: *RequestContext) void {
defer {
this.deref();
}
Expand Down Expand Up @@ -3254,7 +3254,7 @@ fn NewRequestContext(comptime ssl_enabled: bool, comptime debug_mode: bool, comp
const path = blob.store.?.data.s3.path();
const env = globalThis.bunVM().transpiler.env;

credentials.s3Stat(path, @ptrCast(&onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
S3.stat(credentials, path, @ptrCast(&onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);

return;
}
Expand Down
1 change: 1 addition & 0 deletions src/bun.js/bindings/JSS3Bucket.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ JSC_DECLARE_HOST_FUNCTION(functionS3Bucket_size);

static const HashTableValue JSS3BucketPrototypeTableValues[] = {
{ "unlink"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_unlink, 0 } },
{ "delete"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_unlink, 0 } },
{ "write"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_write, 1 } },
{ "presign"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_presign, 1 } },
{ "exists"_s, static_cast<unsigned>(PropertyAttribute::Function | PropertyAttribute::ReadOnly), NoIntrinsic, { HashTableValue::NativeFunctionType, functionS3Bucket_exists, 1 } },
Expand Down
6 changes: 3 additions & 3 deletions src/bun.js/event_loop.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ const ReadFileTask = WebCore.Blob.ReadFile.ReadFileTask;
const WriteFileTask = WebCore.Blob.WriteFile.WriteFileTask;
const napi_async_work = JSC.napi.napi_async_work;
const FetchTasklet = Fetch.FetchTasklet;
const AWS = @import("../s3.zig").AWSCredentials;
const S3HttpSimpleTask = AWS.S3HttpSimpleTask;
const S3HttpDownloadStreamingTask = AWS.S3HttpDownloadStreamingTask;
const S3 = bun.S3;
const S3HttpSimpleTask = S3.S3HttpSimpleTask;
const S3HttpDownloadStreamingTask = S3.S3HttpDownloadStreamingTask;

const JSValue = JSC.JSValue;
const js = JSC.C;
Expand Down
24 changes: 12 additions & 12 deletions src/bun.js/webcore/S3Bucket.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@ const PathOrBlob = JSC.Node.PathOrBlob;
const ZigString = JSC.ZigString;
const Method = bun.http.Method;
const S3File = @import("./S3File.zig");
const AWSCredentials = bun.AWSCredentials;
const S3Credentials = bun.S3.S3Credentials;

const S3BucketOptions = struct {
credentials: *AWSCredentials,
options: bun.S3.MultiPartUpload.MultiPartUploadOptions = .{},
credentials: *S3Credentials,
options: bun.S3.MultiPartUploadOptions = .{},
acl: ?bun.S3.ACL = null,
pub usingnamespace bun.New(@This());

Expand All @@ -20,7 +20,7 @@ const S3BucketOptions = struct {
}
};

pub fn writeFormatCredentials(credentials: *AWSCredentials, options: bun.S3.MultiPartUpload.MultiPartUploadOptions, acl: ?bun.S3.ACL, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
pub fn writeFormatCredentials(credentials: *S3Credentials, options: bun.S3.MultiPartUploadOptions, acl: ?bun.S3.ACL, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
try writer.writeAll("\n");

{
Expand All @@ -37,7 +37,7 @@ pub fn writeFormatCredentials(credentials: *AWSCredentials, options: bun.S3.Mult
formatter.printComma(Writer, writer, enable_ansi_colors) catch bun.outOfMemory();
try writer.writeAll("\n");

const region = if (credentials.region.len > 0) credentials.region else AWSCredentials.guessRegion(credentials.endpoint);
const region = if (credentials.region.len > 0) credentials.region else S3Credentials.guessRegion(credentials.endpoint);
try formatter.writeIndent(Writer, writer);
try writer.writeAll(comptime bun.Output.prettyFmt("<r>region<d>:<r> \"", enable_ansi_colors));
try writer.print(comptime bun.Output.prettyFmt("<r><b>{s}<r>\"", enable_ansi_colors), .{region});
Expand Down Expand Up @@ -133,7 +133,7 @@ pub fn call(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe: *
};
errdefer path.deinit();
const options = args.nextEat();
var blob = Blob.new(try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl));
var blob = Blob.new(try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl));
blob.allocator = bun.default_allocator;
return blob.toJS(globalThis);
}
Expand All @@ -151,7 +151,7 @@ pub fn presign(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe
errdefer path.deinit();

const options = args.nextEat();
var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
defer blob.detach();
return S3File.getPresignUrlFrom(&blob, globalThis, options);
}
Expand All @@ -168,7 +168,7 @@ pub fn exists(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe:
};
errdefer path.deinit();
const options = args.nextEat();
var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
defer blob.detach();
return S3File.S3BlobStatTask.exists(globalThis, &blob);
}
Expand All @@ -185,7 +185,7 @@ pub fn size(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe: *
};
errdefer path.deinit();
const options = args.nextEat();
var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
defer blob.detach();
return S3File.S3BlobStatTask.size(globalThis, &blob);
}
Expand All @@ -203,7 +203,7 @@ pub fn write(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe:
};

const options = args.nextEat();
var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
defer blob.detach();
var blob_internal: PathOrBlob = .{ .blob = blob };
return Blob.writeFileInternal(globalThis, &blob_internal, data, .{
Expand All @@ -221,7 +221,7 @@ pub fn unlink(ptr: *S3BucketOptions, globalThis: *JSC.JSGlobalObject, callframe:
};
errdefer path.deinit();
const options = args.nextEat();
var blob = try S3File.constructS3FileWithAWSCredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
var blob = try S3File.constructS3FileWithS3CredentialsAndOptions(globalThis, path, options, ptr.credentials, ptr.options, ptr.acl);
defer blob.detach();
return blob.store.?.data.s3.unlink(blob.store.?, globalThis, options);
}
Expand All @@ -235,7 +235,7 @@ pub fn construct(globalThis: *JSC.JSGlobalObject, callframe: *JSC.CallFrame) cal
if (options.isEmptyOrUndefinedOrNull() or !options.isObject()) {
globalThis.throwInvalidArguments("Expected S3 options to be passed", .{}) catch return null;
}
var aws_options = AWSCredentials.getCredentialsWithOptions(globalThis.bunVM().transpiler.env.getAWSCredentials(), .{}, options, null, globalThis) catch return null;
var aws_options = S3Credentials.getCredentialsWithOptions(globalThis.bunVM().transpiler.env.getS3Credentials(), .{}, options, null, globalThis) catch return null;
defer aws_options.deinit();
return S3BucketOptions.new(.{
.credentials = aws_options.credentials.dupe(),
Expand Down
33 changes: 16 additions & 17 deletions src/bun.js/webcore/S3File.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ const Method = bun.http.Method;
const strings = bun.strings;
const Output = bun.Output;
const S3Bucket = @import("./S3Bucket.zig");
const S3 = bun.S3;

pub fn writeFormat(s3: *Blob.S3Store, comptime Formatter: type, formatter: *Formatter, writer: anytype, comptime enable_ansi_colors: bool) !void {
try writer.writeAll(comptime Output.prettyFmt("<r>S3Ref<r>", enable_ansi_colors));
Expand Down Expand Up @@ -214,19 +215,19 @@ fn constructS3FileInternalStore(
options: ?JSC.JSValue,
) bun.JSError!Blob {
// get credentials from env
const existing_credentials = globalObject.bunVM().transpiler.env.getAWSCredentials();
return constructS3FileWithAWSCredentials(globalObject, path, options, existing_credentials);
const existing_credentials = globalObject.bunVM().transpiler.env.getS3Credentials();
return constructS3FileWithS3Credentials(globalObject, path, options, existing_credentials);
}
/// if the credentials have changed, we need to clone it, if not we can just ref/deref it
pub fn constructS3FileWithAWSCredentialsAndOptions(
pub fn constructS3FileWithS3CredentialsAndOptions(
globalObject: *JSC.JSGlobalObject,
path: JSC.Node.PathLike,
options: ?JSC.JSValue,
default_credentials: *AWS,
default_options: bun.S3.MultiPartUpload.MultiPartUploadOptions,
default_credentials: *S3.S3Credentials,
default_options: bun.S3.MultiPartUploadOptions,
default_acl: ?bun.S3.ACL,
) bun.JSError!Blob {
var aws_options = try AWS.getCredentialsWithOptions(default_credentials.*, default_options, options, default_acl, globalObject);
var aws_options = try S3.S3Credentials.getCredentialsWithOptions(default_credentials.*, default_options, options, default_acl, globalObject);
defer aws_options.deinit();

const store = brk: {
Expand Down Expand Up @@ -268,13 +269,13 @@ pub fn constructS3FileWithAWSCredentialsAndOptions(
return blob;
}

pub fn constructS3FileWithAWSCredentials(
pub fn constructS3FileWithS3Credentials(
globalObject: *JSC.JSGlobalObject,
path: JSC.Node.PathLike,
options: ?JSC.JSValue,
existing_credentials: AWS,
existing_credentials: S3.S3Credentials,
) bun.JSError!Blob {
var aws_options = try AWS.getCredentialsWithOptions(existing_credentials, .{}, options, null, globalObject);
var aws_options = try S3.S3Credentials.getCredentialsWithOptions(existing_credentials, .{}, options, null, globalObject);
defer aws_options.deinit();
const store = Blob.Store.initS3(path, null, aws_options.credentials, bun.default_allocator) catch bun.outOfMemory();
errdefer store.deinit();
Expand Down Expand Up @@ -318,14 +319,12 @@ fn constructS3FileInternal(
return ptr;
}

const AWS = bun.S3.AWSCredentials;

pub const S3BlobStatTask = struct {
promise: JSC.JSPromise.Strong,
store: *Blob.Store,
usingnamespace bun.New(S3BlobStatTask);

pub fn onS3ExistsResolved(result: AWS.S3StatResult, this: *S3BlobStatTask) void {
pub fn onS3ExistsResolved(result: S3.S3StatResult, this: *S3BlobStatTask) void {
defer this.deinit();
const globalThis = this.promise.globalObject().?;
switch (result) {
Expand All @@ -346,7 +345,7 @@ pub const S3BlobStatTask = struct {
}
}

pub fn onS3SizeResolved(result: AWS.S3StatResult, this: *S3BlobStatTask) void {
pub fn onS3SizeResolved(result: S3.S3StatResult, this: *S3BlobStatTask) void {
defer this.deinit();
const globalThis = this.promise.globalObject().?;

Expand All @@ -371,7 +370,7 @@ pub const S3BlobStatTask = struct {
const path = blob.store.?.data.s3.path();
const env = globalThis.bunVM().transpiler.env;

credentials.s3Stat(path, @ptrCast(&S3BlobStatTask.onS3ExistsResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3ExistsResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
return promise;
}

Expand All @@ -386,7 +385,7 @@ pub const S3BlobStatTask = struct {
const path = blob.store.?.data.s3.path();
const env = globalThis.bunVM().transpiler.env;

credentials.s3Stat(path, @ptrCast(&S3BlobStatTask.onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
S3.stat(credentials, path, @ptrCast(&S3BlobStatTask.onS3SizeResolved), this, if (env.getHttpProxy(true, null)) |proxy| proxy.href else null);
return promise;
}

Expand All @@ -405,7 +404,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *JSC.JSGlobalObject, extra_opt
var method: bun.http.Method = .GET;
var expires: usize = 86400; // 1 day default

var credentialsWithOptions: AWS.AWSCredentialsWithOptions = .{
var credentialsWithOptions: S3.S3CredentialsWithOptions = .{
.credentials = this.store.?.data.s3.getCredentials().*,
};
defer {
Expand Down Expand Up @@ -434,7 +433,7 @@ pub fn getPresignUrlFrom(this: *Blob, globalThis: *JSC.JSGlobalObject, extra_opt
.method = method,
.acl = credentialsWithOptions.acl,
}, .{ .expires = expires }) catch |sign_err| {
return AWS.throwSignError(sign_err, globalThis);
return S3.throwSignError(sign_err, globalThis);
};
defer result.deinit();
var str = bun.String.fromUTF8(result.url);
Expand Down
Loading

0 comments on commit 034f776

Please sign in to comment.