Skip to content

Commit

Permalink
Merge pull request #198 from swisspost/local-s3-supports
Browse files Browse the repository at this point in the history 
Local s3 support added
  • Loading branch information
@dominik-cnx
dominik-cnx authored Nov 12, 2024
2 parents 01fd914 + 74ddd0d commit c320063
Show file tree
Hide file tree
Showing 5 changed files with 133 additions and 39 deletions.
22 changes: 16 additions & 6 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,10 +223,15 @@ The following configuration values are available:
| rejectStorageWriteOnLowMemory | redis | false | When set to _true_, PUT requests with the x-importance-level header can be rejected when memory gets low |
| freeMemoryCheckIntervalMs | redis | 60000 | The interval in milliseconds to calculate the actual memory usage |
| redisReadyCheckIntervalMs | redis | -1 | The interval in milliseconds to calculate the "ready state" of redis. When value < 1, no "ready state" will be calculated |
| awsS3Region | aws-s3 | | The region of S3 server |
| awsS3BucketName | aws-s3 | | The S3 bucket name |
| awsS3AccessKeyId | aws-s3 | | The AWS access key Id |
| awsS3SecretAccessKey | aws-s3 | | The AWS secret access key |
| awsS3Region | s3 | | The region of AWS S3 server, with local service such localstack, also need set a valid region |
| s3BucketName | s3 | | The S3 bucket name |
| s3AccessKeyId | s3 | | The s3 access key Id |
| s3SecretAccessKey | s3 | | The s3 secret access key |
| localS3 | s3 | | Set to true in order to use a local S3 instance instead of AWS |
| localS3Endpoint | s3 | | The endpoint/host to use in case that localS3 is set to true, e.g. 127.0.0.1 (in my case it had to be an IP) |
| localS3Port | s3 | | The port to use in case that localS3 is set to true, e.g. 4566 |
| createBucketIfNotExist | s3 | | create bucket if bucket not exist, related permission required |


### Configuration util

Expand Down Expand Up @@ -260,8 +265,13 @@ The data is stored hierarchically on the file system. This is the default storag

### S3 storage
The data is stored in a S3 instance.
Before use, need sign up in AWS service at https://docs.aws.amazon.com/SetUp/latest/UserGuide/setup-AWSsignup.html
you also need create bucket from S3 web console first

#### AWS S3
See https://aws.amazon.com/s3 it is also possible to use a local instance using https://docs.localstack.cloud/user-guide/aws/s3/


docker run --rm -p 4566:4566 -v ./s3:/var/lib/localstack localstack/localstack:s3-latest


### Redis Storage
The data is stored in a redis database.
Expand Down
7 changes: 5 additions & 2 deletions src/main/java/org/swisspush/reststorage/RestStorageMod.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,11 @@ private Future<Storage> createStorage(ModuleConfiguration moduleConfiguration) {
break;
case s3:
promise.complete(new S3FileSystemStorage(vertx, exceptionFactory, moduleConfiguration.getRoot(),
moduleConfiguration.getAwsS3Region(), moduleConfiguration.getAwsS3BucketName(),
moduleConfiguration.getAwsS3AccessKeyId(), moduleConfiguration.getAwsS3SecretAccessKey()));
moduleConfiguration.getAwsS3Region(), moduleConfiguration.getS3BucketName(),
moduleConfiguration.getS3AccessKeyId(), moduleConfiguration.getS3SecretAccessKey(),
moduleConfiguration.getS3UseTlsConnection(), moduleConfiguration.isLocalS3(),
moduleConfiguration.getLocalS3Endpoint(), moduleConfiguration.getLocalS3Port(),
moduleConfiguration.getCreateBucketIfNotPresentYet()));
break;
case redis:
createRedisStorage(vertx, moduleConfiguration).onComplete(event -> {
Expand Down
7 changes: 2 additions & 5 deletions src/main/java/org/swisspush/reststorage/s3/S3FileSystemDirLister.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,11 @@ private void listDirBlocking(Path path, int offset, int count, Promise<Collectio
log.trace("Processing entry '{}'", entryName);
// Create resource representing currently processed directory entry.
final Resource resource;
if (Files.isDirectory(entry)) {
if (entryName.endsWith(S3_PATH_SEPARATOR)) {
resource = new CollectionResource();
entryName = entryName.replace(S3_PATH_SEPARATOR, "");
} else if (Files.isRegularFile(entry)) {
resource = new DocumentResource();
} else {
resource = new Resource();
resource.exists = false;
resource = new DocumentResource();
}
resource.name = entryName;
collection.items.add(resource);
Expand Down
56 changes: 45 additions & 11 deletions src/main/java/org/swisspush/reststorage/s3/S3FileSystemStorage.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import io.vertx.core.Handler;
import io.vertx.core.Vertx;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.swisspush.reststorage.CollectionResource;
Expand All @@ -21,6 +22,7 @@
import java.net.URI;
import java.nio.file.DirectoryNotEmptyException;
import java.nio.file.DirectoryStream;
import java.nio.file.FileSystemAlreadyExistsException;
import java.nio.file.FileSystemException;
import java.nio.file.FileSystems;
import java.nio.file.FileVisitResult;
Expand All @@ -31,6 +33,7 @@
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

Expand Down Expand Up @@ -62,23 +65,54 @@ private static S3FileSystem getFileSystem(URI uri) {
}

public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFactory, String rootPath,
String awsS3Region, String awsS3BucketName, String awsS3AccessKeyId, String awsS3SecretAccessKey) {
String awsS3Region, String s3BucketName, String s3AccessKeyId, String s3SecretAccessKey,
boolean useTlsConnection, boolean isLocalS3, String localS3Endpoint, int localS3Port, boolean createBucketIfNotPresentYet) {
this.vertx = vertx;
this.exceptionFactory = exceptionFactory;
Objects.requireNonNull(s3BucketName, "BucketName must not be null");
Objects.requireNonNull(awsS3Region, "Region must not be null");
Objects.requireNonNull(awsS3BucketName, "BucketName must not be null");
Objects.requireNonNull(awsS3AccessKeyId, "AccessKeyId must not be null");
Objects.requireNonNull(awsS3SecretAccessKey, "SecretAccessKey must not be null");
System.setProperty("aws.region", awsS3Region);

System.setProperty("aws.region",awsS3Region);
System.setProperty("aws.accessKeyId", awsS3AccessKeyId);
System.setProperty("aws.secretAccessKey", awsS3SecretAccessKey);
if (isLocalS3) {
// local S3, AWS SDK requires these two properties to be set
System.setProperty("aws.accessKeyId", "local");
System.setProperty("aws.secretAccessKey", "local");

if (!awsS3BucketName.startsWith("s3:") && !awsS3BucketName.startsWith("s3x:")) {
awsS3BucketName = "s3://" + awsS3BucketName;
String credentials = "";
if (StringUtils.isNotEmpty(s3AccessKeyId) && StringUtils.isNotEmpty(s3SecretAccessKey)) {
credentials = s3AccessKeyId + ":" + s3SecretAccessKey + "@";
}
String port = "";
if (localS3Port > 0) {
port = ":" + localS3Port;
}
// s3x://[key:secret@]endpoint[:port]/bucket
s3BucketName = "s3x://" + credentials + localS3Endpoint + port + "/" + s3BucketName;
if (!useTlsConnection) {
System.setProperty("s3.spi.endpoint-protocol", "http");
}
} else {
// AWS S3
Objects.requireNonNull(s3AccessKeyId, "AccessKeyId must not be null");
Objects.requireNonNull(s3SecretAccessKey, "SecretAccessKey must not be null");
System.setProperty("aws.accessKeyId", s3AccessKeyId);
System.setProperty("aws.secretAccessKey", s3SecretAccessKey);
s3BucketName = "s3://" + s3BucketName;
}

var uri = URI.create(s3BucketName);

if (createBucketIfNotPresentYet) {
try (var fs = FileSystems.newFileSystem(uri,
Map.of("locationConstraint", awsS3Region))) {
log.info("Bucket created: " + fs.toString());
} catch (FileSystemAlreadyExistsException e) {
log.info("Bucket " + s3BucketName + " already exists: ", e);
} catch (IOException e) {
log.error("Failed to create bucket " + s3BucketName, e);
}
}

var uri = URI.create(awsS3BucketName);
fileSystem = getFileSystem(uri);
root = fileSystem.getPath(rootPath);

Expand All @@ -95,7 +129,7 @@ public S3FileSystemStorage(Vertx vertx, RestStorageExceptionFactory exceptionFac

// Cache string length of root without trailing slashes
int rootLen;
for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen) ;
for (rootLen = tmpRoot.length() - 1; tmpRoot.charAt(rootLen) == '/'; --rootLen);
this.rootLen = rootLen;
}

Expand Down
80 changes: 65 additions & 15 deletions src/main/java/org/swisspush/reststorage/util/ModuleConfiguration.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,15 @@ public enum StorageType {
private int maxRedisWaitingHandlers = 2048;
private int maxStorageExpandSubresources = 1000;

private String awsS3BucketName = null;
private String s3BucketName = null;
private String awsS3Region = null;
private String awsS3AccessKeyId = null;
private String awsS3SecretAccessKey = null;
private String s3AccessKeyId = null;
private String s3SecretAccessKey = null;
private boolean s3UseTlsConnection = true;
private boolean createBucketIfNotPresentYet = false;
private boolean localS3 = false;
private String localS3Endpoint = null;
private int localS3Port = 0;

public ModuleConfiguration root(String root) {
this.root = root;
Expand Down Expand Up @@ -301,18 +306,43 @@ public ModuleConfiguration awsS3Region(String awsS3Region) {
return this;
}

public ModuleConfiguration awsS3BucketName(String awsS3BucketName) {
this.awsS3BucketName = awsS3BucketName;
public ModuleConfiguration s3BucketName(String awsS3BucketName) {
this.s3BucketName = awsS3BucketName;
return this;
}

public ModuleConfiguration awsS3AccessKeyId(String awsS3AccessKeyId) {
this.awsS3AccessKeyId = awsS3AccessKeyId;
public ModuleConfiguration s3AccessKeyId(String awsS3AccessKeyId) {
this.s3AccessKeyId = awsS3AccessKeyId;
return this;
}

public ModuleConfiguration awsS3SecretAccessKey(String awsS3SecretAccessKey) {
this.awsS3SecretAccessKey = awsS3SecretAccessKey;
public ModuleConfiguration s3SecretAccessKey(String awsS3SecretAccessKey) {
this.s3SecretAccessKey = awsS3SecretAccessKey;
return this;
}

public ModuleConfiguration s3UseTlsConnection(boolean s3UseTlsConnection) {
this.s3UseTlsConnection = s3UseTlsConnection;
return this;
}

public ModuleConfiguration localS3Endpoint(String s3Endpoint) {
this.localS3Endpoint = s3Endpoint;
return this;
}

public ModuleConfiguration localS3Port(int s3Port) {
this.localS3Port = s3Port;
return this;
}

public ModuleConfiguration createBucketIfNotPresentYet(boolean createBucketIfNotExist) {
this.createBucketIfNotPresentYet = createBucketIfNotExist;
return this;
}

public ModuleConfiguration localS3(boolean localS3) {
this.localS3 = localS3;
return this;
}

Expand Down Expand Up @@ -493,16 +523,36 @@ public String getAwsS3Region() {
return awsS3Region;
}

public String getAwsS3BucketName() {
return awsS3BucketName;
public String getS3BucketName() {
return s3BucketName;
}

public String getS3AccessKeyId() {
return s3AccessKeyId;
}

public String getS3SecretAccessKey() {
return s3SecretAccessKey;
}

public boolean getS3UseTlsConnection() {
return s3UseTlsConnection;
}

public String getLocalS3Endpoint() {
return localS3Endpoint;
}

public int getLocalS3Port() {
return localS3Port;
}

public String getAwsS3AccessKeyId() {
return awsS3AccessKeyId;
public boolean getCreateBucketIfNotPresentYet() {
return createBucketIfNotPresentYet;
}

public String getAwsS3SecretAccessKey() {
return awsS3SecretAccessKey;
public boolean isLocalS3() {
return localS3;
}

public JsonObject asJsonObject() {
Expand Down

0 comments on commit c320063

Please sign in to comment.