From 3b497a4f068446f53f6f974e33e5aaf2fe1bf389 Mon Sep 17 00:00:00 2001 From: "Ariel Shaqed (Scolnicov)" Date: Tue, 12 Sep 2023 15:57:05 +0300 Subject: [PATCH] Add testMkdirs --- .../io/lakefs/LakeFSFileSystemServerTest.java | 148 ++++++++++-------- 1 file changed, 86 insertions(+), 62 deletions(-) diff --git a/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerTest.java b/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerTest.java index b641253d508..7d67c13672b 100644 --- a/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerTest.java +++ b/clients/hadoopfs/src/test/java/io/lakefs/LakeFSFileSystemServerTest.java @@ -54,6 +54,7 @@ import static org.mockserver.model.HttpResponse.response; import static org.mockserver.model.JsonBody.json; +import static org.apache.commons.lang3.StringUtils.removeStart; import com.google.common.base.Optional; import org.immutables.value.Value; @@ -147,6 +148,26 @@ protected String s3Url(String s3Path) { return s3Base + s3Path; } + protected String getS3Key(StagingLocation stagingLocation) { + return removeStart(stagingLocation.getPhysicalAddress(), s3Base + "/"); + } + + protected void assertS3Object(StagingLocation stagingLocation, String contents) { + String s3Key = getS3Key(stagingLocation); + List actualFiles = ImmutableList.of(); + try (S3Object obj = + s3Client.getObject(new GetObjectRequest(s3Bucket, "/" + s3Key))) { + actualFiles = getS3FilesByPrefix("/"); + String actual = IOUtils.toString(obj.getObjectContent()); + Assert.assertEquals(contents, actual); + + Assert.assertEquals(ImmutableList.of(s3Key), actualFiles); + } catch (Exception e) { + throw new RuntimeException("Files " + actualFiles + + "; read key " + s3Key + " failed", e); + } + } + protected String objectLocToS3ObjKey(ObjectLocation objectLoc) { return String.format("/%s/%s/%s",objectLoc.getRepository(), objectLoc.getRef(), objectLoc.getPath()); } @@ -287,10 +308,12 @@ protected void expectUploadObject(String repo, String branch, String path) { } // Return a location under namespace for this getPhysicalAddress call. + + // TODO(ariels): abstract, overload separately for direct and pre-signed. protected StagingLocation expectGetPhysicalAddress(String repo, String branch, String path, String namespace) { StagingLocation stagingLocation = new StagingLocation() .token("token:foo:" + sessionId()) - .physicalAddress(s3Url(String.format("/%s/%s/%s/%s/%s", + .physicalAddress(s3Url(String.format("/%s/%s/%s/%s/%s-object", sessionId(), namespace, repo, branch, path))); mockServerClient.when(request() .withMethod("GET") @@ -751,74 +774,75 @@ public void testCreate() throws ApiException, IOException { out.close(); // Write succeeded, verify physical file on S3. - String s3Key = org.apache.commons.lang3.StringUtils.removeStart(stagingLocation.getPhysicalAddress(), s3Base + "/"); - try (S3Object ret = - s3Client.getObject(new GetObjectRequest(s3Bucket, "/" + s3Key))) { - InputStream in = ret.getObjectContent(); - String actual = IOUtils.toString(in); - Assert.assertEquals(contents, actual); - - List actualFiles = getS3FilesByPrefix("/"); - Assert.assertEquals(ImmutableList.of(s3Key), actualFiles); - } catch (Exception e) { - throw new RuntimeException("Read key " + s3Key + " failed", e); - } + String s3Key = getS3Key(stagingLocation); + assertS3Object(stagingLocation, contents); } @Test public void testCreateExistingDirectory() throws ApiException, IOException { Path path = new Path("lakefs://repo/main/sub1/sub2/create.me"); // path is a directory -- so cannot be created as a file. - expectDirectoryMarker(ObjectLocation.pathToObjectLocation(path)); - expectStatObject("repo", "main", "sub1/sub2/create.me/", - new ObjectStats() - .path("sub1/sub2/create.me") - .pathType(PathTypeEnum.OBJECT)); - Assert.assertThrows(FileAlreadyExistsException.class, () -> fs.create(path, false)); - } - - // @Test(expected = FileAlreadyExistsException.class) - // public void testCreateExistingFile() throws ApiException, IOException { - // ObjectLocation dir = new ObjectLocation("lakefs", "repo", "main", "sub1/sub2"); - // mockExistingDirPath(dir, ImmutableList.of(new ObjectLocation("lakefs", "repo", "main", "sub1/sub2/create.me"))); - // fs.create(new Path("lakefs://repo/main/sub1/sub2/create.me"), false); - // } - // @Test - // public void testMkdirs() throws ApiException, IOException { - // // setup empty folder checks - // Path testPath = new Path("dir1/dir2/dir3"); - // do { - // when(objectsApi.statObject("repo", "main", testPath.toString(), false, false)) - // .thenThrow(noSuchFile); - // when(objectsApi.statObject("repo", "main", testPath+"/", false, false)) - // .thenThrow(noSuchFile); - // when(objectsApi.listObjects(eq("repo"), eq("main"), eq(false), eq(false), eq(""), any(), eq(""), eq(testPath+"/"))) - // .thenReturn(new ObjectStatsList().results(Collections.emptyList()).pagination(new Pagination().hasMore(false))); - // testPath = testPath.getParent(); - // } while(testPath != null && !testPath.isRoot()); - - // // physical address to directory marker object - // StagingLocation stagingLocation = mockGetPhysicalAddress("repo", "main", "dir1/dir2/dir3/", "repo-base/emptyDir"); - - // // call mkdirs - // Path p = new Path("lakefs://repo/main/dir1/dir2/dir3"); - // boolean mkdirs = fs.mkdirs(p); - // Assert.assertTrue("make dirs", mkdirs); - - // // verify metadata - // ArgumentCaptor metadataCapture = ArgumentCaptor.forClass(StagingMetadata.class); - // verify(stagingApi).linkPhysicalAddress(eq("repo"), eq("main"), eq("dir1/dir2/dir3/"), - // metadataCapture.capture()); - // StagingMetadata actualMetadata = metadataCapture.getValue(); - // Assert.assertEquals(stagingLocation, actualMetadata.getStaging()); - // Assert.assertEquals(0, (long)actualMetadata.getSizeBytes()); - - // // verify file exists on s3 - // S3Object ret = s3Client.getObject(new GetObjectRequest(s3Bucket, "/repo-base/emptyDir")); - // String actual = IOUtils.toString(ret.getObjectContent()); - // Assert.assertEquals("", actual); - // } + expectStatObjectNotFound("repo", "main", "sub1/sub2/create.me"); + ObjectStats stats = new ObjectStats() + .path("sub1/sub2/create.me/") + .physicalAddress(s3Url("/repo-base/sub1/sub2/create.me")); + expectStatObject("repo", "main", "sub1/sub2/create.me/", stats); + + Exception e = + Assert.assertThrows(FileAlreadyExistsException.class, () -> fs.create(path, false)); + Assert.assertThat(e.getMessage(), new StringContains("is a directory")); + } + + @Test + public void testCreateExistingFile() throws ApiException, IOException { + Path path = new Path("lakefs://repo/main/sub1/sub2/create.me"); + + ObjectLocation dir = new ObjectLocation("lakefs", "repo", "main", "sub1/sub2"); + expectStatObject("repo", "main", "sub1/sub2/create.me", + new ObjectStats().path("sub1/sub2/create.me")); + Exception e = Assert.assertThrows(FileAlreadyExistsException.class, + () -> fs.create(path, false)); + Assert.assertThat(e.getMessage(), new StringContains("already exists")); + } + + @Test + public void testMkdirs() throws ApiException, IOException { + // setup empty folder checks + Path path = new Path("dir1/dir2/dir3"); + for (Path p = new Path(path.toString()); p != null && !p.isRoot(); p = p.getParent()) { + expectStatObjectNotFound("repo", "main", p.toString()); + expectStatObjectNotFound("repo", "main", p+"/"); + expectListing("repo", "main", ImmutablePagination.builder().prefix(p+"/").build(), false); + } + + // physical address to directory marker object + StagingLocation stagingLocation = + expectGetPhysicalAddress("repo", "main", "dir1/dir2/dir3/", "repo-base/emptyDir"); + + ObjectStats newStats = new ObjectStats() + .path("dir1/dir2/dir3/") + .physicalAddress(s3Url("/repo-base/dir12")); + expectStatObject("repo", "main", "dir1/dir2/dir3/", newStats); + + mockServerClient.when(request() + .withMethod("PUT") + .withPath("/repositories/repo/branches/main/staging/backing") + .withQueryStringParameter("path", "dir1/dir2/dir3/") + .withBody(json(gson.toJson(new StagingMetadata() + .staging(stagingLocation) + .sizeBytes(0L)), + MatchType.ONLY_MATCHING_FIELDS))) + .respond(response() + .withStatusCode(200) + .withBody(gson.toJson(newStats))); + + // call mkdirs + Assert.assertTrue(fs.mkdirs(new Path("lakefs://repo/main/", path))); + + // verify file exists on s3 + assertS3Object(stagingLocation, ""); + } // @Test // public void testOpen() throws IOException, ApiException {