Skip to content

Commit

Permalink
3rd party S3 supports
Browse files Browse the repository at this point in the history
  • Loading branch information
Xin Zheng committed Nov 1, 2024
1 parent 01fd914 commit 41b0bca
Show file tree
Hide file tree
Showing 5 changed files with 129 additions and 37 deletions.
16 changes: 11 additions & 5 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,10 +223,15 @@ The following configuration values are available:
| rejectStorageWriteOnLowMemory | redis | false | When set to _true_, PUT requests with the x-importance-level header can be rejected when memory gets low |
| freeMemoryCheckIntervalMs | redis | 60000 | The interval in milliseconds to calculate the actual memory usage |
| redisReadyCheckIntervalMs | redis | -1 | The interval in milliseconds to calculate the "ready state" of redis. When value < 1, no "ready state" will be calculated |
| awsS3Region | aws-s3 | | The region of S3 server |
| awsS3BucketName | aws-s3 | | The S3 bucket name |
| awsS3AccessKeyId | aws-s3 | | The AWS access key Id |
| awsS3SecretAccessKey | aws-s3 | | The AWS secret access key |
| awsS3Region | s3 | | The region of AWS S3 server, with local service such localstack, also need set a valid region |
| s3BucketName | s3 | | The S3 bucket name |
| s3AccessKeyId | s3 | | The s3 access key Id |
| s3SecretAccessKey | s3 | | The s3 secret access key |
| localS3 | s3 | | is 3rd party S3-like services |
| s3Endpoint | s3 | | 3rd party S3-like services endpoint |
| s3Port | s3 | | 3rd party S3-like services port |
| createBucketIfNotExist | s3 | | create bucket if bucket not exist, related permission required |


### Configuration util

Expand Down Expand Up @@ -260,8 +265,9 @@ The data is stored hierarchically on the file system. This is the default storag

### S3 storage
The data is stored in a S3 instance.

#### AWS S3
Before use, need sign up in AWS service at https://docs.aws.amazon.com/SetUp/latest/UserGuide/setup-AWSsignup.html
you also need create bucket from S3 web console first

### Redis Storage
The data is stored in a redis database.
Expand Down
7 changes: 5 additions & 2 deletions src/main/java/org/swisspush/reststorage/RestStorageMod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,11 @@ private Future<Storage> createStorage(ModuleConfiguration moduleConfiguration) {
break;
case s3:
promise.complete(new S3FileSystemStorage(vertx, exceptionFactory, moduleConfiguration.getRoot(),
moduleConfiguration.getAwsS3Region(), moduleConfiguration.getAwsS3BucketName(),
moduleConfiguration.getAwsS3AccessKeyId(), moduleConfiguration.getAwsS3SecretAccessKey()));
moduleConfiguration.getAwsS3Region(), moduleConfiguration.getS3BucketName(),
moduleConfiguration.getS3AccessKeyId(), moduleConfiguration.getS3SecretAccessKey(),
moduleConfiguration.getS3UseTlsConnection(), moduleConfiguration.isLocalS3(),
moduleConfiguration.getS3Endpoint(), moduleConfiguration.getS3Port(),
moduleConfiguration.getCreateBucketIfNotExist()));
break;
case redis:
createRedisStorage(vertx, moduleConfiguration).onComplete(event -> {
Expand Down
7 changes: 2 additions & 5 deletions src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,11 @@ private void listDirBlocking(Path path, int offset, int count, Promise<Collectio
log.trace("Processing entry '{}'", entryName);
// Create resource representing currently processed directory entry.
final Resource resource;
if (Files.isDirectory(entry)) {
if (entryName.endsWith(S3_PATH_SEPARATOR)) {
resource = new CollectionResource();
entryName = entryName.replace(S3_PATH_SEPARATOR, "");
} else if (Files.isRegularFile(entry)) {
resource = new DocumentResource();
} else {
resource = new Resource();
resource.exists = false;
resource = new DocumentResource();
}
resource.name = entryName;
collection.items.add(resource);
Expand Down
56 changes: 46 additions & 10 deletions src/main/java/org/swisspush/reststorage/s3/S3FileSystemStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.vertx.core.Handler;
import io.vertx.core.Vertx;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.swisspush.reststorage.CollectionResource;
Expand All @@ -21,6 +22,7 @@
import java.net.URI;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystemAlreadyExistsException;
import java.nio.file.FileSystemException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
Expand All @@ -31,6 +33,7 @@
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

Expand Down Expand Up @@ -62,23 +65,56 @@ private static S3FileSystem getFileSystem(URI uri) {
}

public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFactory, String rootPath,
String awsS3Region, String awsS3BucketName, String awsS3AccessKeyId, String awsS3SecretAccessKey) {
String awsS3Region, String s3BucketName, String s3AccessKeyId, String s3SecretAccessKey,
boolean useTlsConnection, boolean isLocalS3, String s3Endpoint, int s3Port, boolean createBucketIfNotExist) {
this.vertx = vertx;
this.exceptionFactory = exceptionFactory;
Objects.requireNonNull(s3BucketName, "BucketName must not be null");
Objects.requireNonNull(awsS3Region, "Region must not be null");
Objects.requireNonNull(awsS3BucketName, "BucketName must not be null");
Objects.requireNonNull(awsS3AccessKeyId, "AccessKeyId must not be null");
Objects.requireNonNull(awsS3SecretAccessKey, "SecretAccessKey must not be null");
System.setProperty("aws.region", awsS3Region);

if (!s3BucketName.startsWith("s3:") && !s3BucketName.startsWith("s3x:")) {
if (isLocalS3) {
// AWS SDK required those properties to be set
System.setProperty("aws.accessKeyId", "local");
System.setProperty("aws.secretAccessKey", "local");
// for nom-AWS
// s3x://[key:secret@]endpoint[:port]/bucket
String credentials = "";
if (StringUtils.isNotEmpty(s3AccessKeyId) && StringUtils.isNotEmpty(s3SecretAccessKey)) {
credentials = s3AccessKeyId + ":" + s3SecretAccessKey + "@";
}
String port = "";
if (s3Port > 0) {
port = ":" + s3Port;
}
s3BucketName = "s3x://" + credentials + s3Endpoint + port + "/" + s3BucketName;
if (!useTlsConnection) {
System.setProperty("s3.spi.endpoint-protocol", "http");
}
} else {
// for AWS S3
Objects.requireNonNull(s3AccessKeyId, "AccessKeyId must not be null");
Objects.requireNonNull(s3SecretAccessKey, "SecretAccessKey must not be null");
System.setProperty("aws.accessKeyId", s3AccessKeyId);
System.setProperty("aws.secretAccessKey", s3SecretAccessKey);
s3BucketName = "s3://" + s3BucketName;
}
}

System.setProperty("aws.region",awsS3Region);
System.setProperty("aws.accessKeyId", awsS3AccessKeyId);
System.setProperty("aws.secretAccessKey", awsS3SecretAccessKey);
var uri = URI.create(s3BucketName);

if (!awsS3BucketName.startsWith("s3:") && !awsS3BucketName.startsWith("s3x:")) {
awsS3BucketName = "s3://" + awsS3BucketName;
if (createBucketIfNotExist) {
try (var fs = FileSystems.newFileSystem(uri,
Map.of("locationConstraint", awsS3Region))) {
System.out.println(fs.toString());
} catch (FileSystemAlreadyExistsException e) {
log.info("Bucket already exists: ", e);
} catch (IOException e) {
log.error("Failed to create bucket", e);
}
}

var uri = URI.create(awsS3BucketName);
fileSystem = getFileSystem(uri);
root = fileSystem.getPath(rootPath);

Expand Down
80 changes: 65 additions & 15 deletions src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,15 @@ public enum StorageType {
private int maxRedisWaitingHandlers = 2048;
private int maxStorageExpandSubresources = 1000;

private String awsS3BucketName = null;
private String s3BucketName = null;
private String awsS3Region = null;
private String awsS3AccessKeyId = null;
private String awsS3SecretAccessKey = null;
private String s3AccessKeyId = null;
private String s3SecretAccessKey = null;
private boolean s3UseTlsConnection = true;
private boolean createBucketIfNotExist = false;
private boolean localS3 = false;
private String s3Endpoint = null;
private int s3Port = 0;

public ModuleConfiguration root(String root) {
this.root = root;
Expand Down Expand Up @@ -301,18 +306,43 @@ public ModuleConfiguration awsS3Region(String awsS3Region) {
return this;
}

public ModuleConfiguration awsS3BucketName(String awsS3BucketName) {
this.awsS3BucketName = awsS3BucketName;
public ModuleConfiguration s3BucketName(String awsS3BucketName) {
this.s3BucketName = awsS3BucketName;
return this;
}

public ModuleConfiguration awsS3AccessKeyId(String awsS3AccessKeyId) {
this.awsS3AccessKeyId = awsS3AccessKeyId;
public ModuleConfiguration s3AccessKeyId(String awsS3AccessKeyId) {
this.s3AccessKeyId = awsS3AccessKeyId;
return this;
}

public ModuleConfiguration awsS3SecretAccessKey(String awsS3SecretAccessKey) {
this.awsS3SecretAccessKey = awsS3SecretAccessKey;
public ModuleConfiguration s3SecretAccessKey(String awsS3SecretAccessKey) {
this.s3SecretAccessKey = awsS3SecretAccessKey;
return this;
}

public ModuleConfiguration s3UseTlsConnection(boolean s3UseTlsConnection) {
this.s3UseTlsConnection = s3UseTlsConnection;
return this;
}

public ModuleConfiguration s3Endpoint(String s3Endpoint) {
this.s3Endpoint = s3Endpoint;
return this;
}

public ModuleConfiguration s3Port(int s3Port) {
this.s3Port = s3Port;
return this;
}

public ModuleConfiguration createBucketIfNotExist(boolean createBucketIfNotExist) {
this.createBucketIfNotExist = createBucketIfNotExist;
return this;
}

public ModuleConfiguration localS3(boolean localS3) {
this.localS3 = localS3;
return this;
}

Expand Down Expand Up @@ -493,16 +523,36 @@ public String getAwsS3Region() {
return awsS3Region;
}

public String getAwsS3BucketName() {
return awsS3BucketName;
public String getS3BucketName() {
return s3BucketName;
}

public String getS3AccessKeyId() {
return s3AccessKeyId;
}

public String getS3SecretAccessKey() {
return s3SecretAccessKey;
}

public boolean getS3UseTlsConnection() {
return s3UseTlsConnection;
}

public String getS3Endpoint() {
return s3Endpoint;
}

public int getS3Port() {
return s3Port;
}

public String getAwsS3AccessKeyId() {
return awsS3AccessKeyId;
public boolean getCreateBucketIfNotExist() {
return createBucketIfNotExist;
}

public String getAwsS3SecretAccessKey() {
return awsS3SecretAccessKey;
public boolean isLocalS3() {
return localS3;
}

public JsonObject asJsonObject() {
Expand Down

0 comments on commit 41b0bca

Please sign in to comment.