Skip to content

Commit

Permalink
Add testMkdirs
Browse files Browse the repository at this point in the history
  • Loading branch information
@arielshaqed
arielshaqed committed Sep 12, 2023
1 parent a0ecb37 commit 3b497a4
Showing 1 changed file with 86 additions and 62 deletions.
148 changes: 86 additions & 62 deletions clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
import static org.mockserver.model.HttpResponse.response;
import static org.mockserver.model.JsonBody.json;

import static org.apache.commons.lang3.StringUtils.removeStart;
import com.google.common.base.Optional;
import org.immutables.value.Value;

Expand Down Expand Up @@ -147,6 +148,26 @@ protected String s3Url(String s3Path) {
return s3Base + s3Path;
}

protected String getS3Key(StagingLocation stagingLocation) {
return removeStart(stagingLocation.getPhysicalAddress(), s3Base + "/");
}

protected void assertS3Object(StagingLocation stagingLocation, String contents) {
String s3Key = getS3Key(stagingLocation);
List<String> actualFiles = ImmutableList.of();
try (S3Object obj =
s3Client.getObject(new GetObjectRequest(s3Bucket, "/" + s3Key))) {
actualFiles = getS3FilesByPrefix("/");
String actual = IOUtils.toString(obj.getObjectContent());
Assert.assertEquals(contents, actual);

Assert.assertEquals(ImmutableList.of(s3Key), actualFiles);
} catch (Exception e) {
throw new RuntimeException("Files " + actualFiles +
"; read key " + s3Key + " failed", e);
}
}

protected String objectLocToS3ObjKey(ObjectLocation objectLoc) {
return String.format("/%s/%s/%s",objectLoc.getRepository(), objectLoc.getRef(), objectLoc.getPath());
}
Expand Down Expand Up @@ -287,10 +308,12 @@ protected void expectUploadObject(String repo, String branch, String path) {
}

// Return a location under namespace for this getPhysicalAddress call.

// TODO(ariels): abstract, overload separately for direct and pre-signed.
protected StagingLocation expectGetPhysicalAddress(String repo, String branch, String path, String namespace) {
StagingLocation stagingLocation = new StagingLocation()
.token("token:foo:" + sessionId())
.physicalAddress(s3Url(String.format("/%s/%s/%s/%s/%s",
.physicalAddress(s3Url(String.format("/%s/%s/%s/%s/%s-object",
sessionId(), namespace, repo, branch, path)));
mockServerClient.when(request()
.withMethod("GET")
Expand Down Expand Up @@ -751,74 +774,75 @@ public void testCreate() throws ApiException, IOException {
out.close();

// Write succeeded, verify physical file on S3.
String s3Key = org.apache.commons.lang3.StringUtils.removeStart(stagingLocation.getPhysicalAddress(), s3Base + "/");
try (S3Object ret =
s3Client.getObject(new GetObjectRequest(s3Bucket, "/" + s3Key))) {
InputStream in = ret.getObjectContent();
String actual = IOUtils.toString(in);
Assert.assertEquals(contents, actual);

List<String> actualFiles = getS3FilesByPrefix("/");
Assert.assertEquals(ImmutableList.of(s3Key), actualFiles);
} catch (Exception e) {
throw new RuntimeException("Read key " + s3Key + " failed", e);
}
String s3Key = getS3Key(stagingLocation);
assertS3Object(stagingLocation, contents);
}

@Test
public void testCreateExistingDirectory() throws ApiException, IOException {
Path path = new Path("lakefs://repo/main/sub1/sub2/create.me");
// path is a directory -- so cannot be created as a file.
expectDirectoryMarker(ObjectLocation.pathToObjectLocation(path));
expectStatObject("repo", "main", "sub1/sub2/create.me/",
new ObjectStats()
.path("sub1/sub2/create.me")
.pathType(PathTypeEnum.OBJECT));
Assert.assertThrows(FileAlreadyExistsException.class, () -> fs.create(path, false));
}

// @Test(expected = FileAlreadyExistsException.class)
// public void testCreateExistingFile() throws ApiException, IOException {
// ObjectLocation dir = new ObjectLocation("lakefs", "repo", "main", "sub1/sub2");
// mockExistingDirPath(dir, ImmutableList.of(new ObjectLocation("lakefs", "repo", "main", "sub1/sub2/create.me")));
// fs.create(new Path("lakefs://repo/main/sub1/sub2/create.me"), false);
// }

// @Test
// public void testMkdirs() throws ApiException, IOException {
// // setup empty folder checks
// Path testPath = new Path("dir1/dir2/dir3");
// do {
// when(objectsApi.statObject("repo", "main", testPath.toString(), false, false))
// .thenThrow(noSuchFile);
// when(objectsApi.statObject("repo", "main", testPath+"/", false, false))
// .thenThrow(noSuchFile);
// when(objectsApi.listObjects(eq("repo"), eq("main"), eq(false), eq(false), eq(""), any(), eq(""), eq(testPath+"/")))
// .thenReturn(new ObjectStatsList().results(Collections.emptyList()).pagination(new Pagination().hasMore(false)));
// testPath = testPath.getParent();
// } while(testPath != null && !testPath.isRoot());

// // physical address to directory marker object
// StagingLocation stagingLocation = mockGetPhysicalAddress("repo", "main", "dir1/dir2/dir3/", "repo-base/emptyDir");

// // call mkdirs
// Path p = new Path("lakefs://repo/main/dir1/dir2/dir3");
// boolean mkdirs = fs.mkdirs(p);
// Assert.assertTrue("make dirs", mkdirs);

// // verify metadata
// ArgumentCaptor<StagingMetadata> metadataCapture = ArgumentCaptor.forClass(StagingMetadata.class);
// verify(stagingApi).linkPhysicalAddress(eq("repo"), eq("main"), eq("dir1/dir2/dir3/"),
// metadataCapture.capture());
// StagingMetadata actualMetadata = metadataCapture.getValue();
// Assert.assertEquals(stagingLocation, actualMetadata.getStaging());
// Assert.assertEquals(0, (long)actualMetadata.getSizeBytes());

// // verify file exists on s3
// S3Object ret = s3Client.getObject(new GetObjectRequest(s3Bucket, "/repo-base/emptyDir"));
// String actual = IOUtils.toString(ret.getObjectContent());
// Assert.assertEquals("", actual);
// }
expectStatObjectNotFound("repo", "main", "sub1/sub2/create.me");
ObjectStats stats = new ObjectStats()
.path("sub1/sub2/create.me/")
.physicalAddress(s3Url("/repo-base/sub1/sub2/create.me"));
expectStatObject("repo", "main", "sub1/sub2/create.me/", stats);

Exception e =
Assert.assertThrows(FileAlreadyExistsException.class, () -> fs.create(path, false));
Assert.assertThat(e.getMessage(), new StringContains("is a directory"));
}

@Test
public void testCreateExistingFile() throws ApiException, IOException {
Path path = new Path("lakefs://repo/main/sub1/sub2/create.me");

ObjectLocation dir = new ObjectLocation("lakefs", "repo", "main", "sub1/sub2");
expectStatObject("repo", "main", "sub1/sub2/create.me",
new ObjectStats().path("sub1/sub2/create.me"));
Exception e = Assert.assertThrows(FileAlreadyExistsException.class,
() -> fs.create(path, false));
Assert.assertThat(e.getMessage(), new StringContains("already exists"));
}

@Test
public void testMkdirs() throws ApiException, IOException {
// setup empty folder checks
Path path = new Path("dir1/dir2/dir3");
for (Path p = new Path(path.toString()); p != null && !p.isRoot(); p = p.getParent()) {
expectStatObjectNotFound("repo", "main", p.toString());
expectStatObjectNotFound("repo", "main", p+"/");
expectListing("repo", "main", ImmutablePagination.builder().prefix(p+"/").build(), false);
}

// physical address to directory marker object
StagingLocation stagingLocation =
expectGetPhysicalAddress("repo", "main", "dir1/dir2/dir3/", "repo-base/emptyDir");

ObjectStats newStats = new ObjectStats()
.path("dir1/dir2/dir3/")
.physicalAddress(s3Url("/repo-base/dir12"));
expectStatObject("repo", "main", "dir1/dir2/dir3/", newStats);

mockServerClient.when(request()
.withMethod("PUT")
.withPath("/repositories/repo/branches/main/staging/backing")
.withQueryStringParameter("path", "dir1/dir2/dir3/")
.withBody(json(gson.toJson(new StagingMetadata()
.staging(stagingLocation)
.sizeBytes(0L)),
MatchType.ONLY_MATCHING_FIELDS)))
.respond(response()
.withStatusCode(200)
.withBody(gson.toJson(newStats)));

// call mkdirs
Assert.assertTrue(fs.mkdirs(new Path("lakefs://repo/main/", path)));

// verify file exists on s3
assertS3Object(stagingLocation, "");
}

// @Test
// public void testOpen() throws IOException, ApiException {
Expand Down

0 comments on commit 3b497a4

Please sign in to comment.