From ef6982871a8ae4a72a75b1d36f95db04f6f0bbd5 Mon Sep 17 00:00:00 2001 From: Harris Borawski Date: Mon, 4 Oct 2021 12:40:03 -0700 Subject: [PATCH 1/2] add recurse option to pkg_zip to maintain directory structure --- pkg/pkg.bzl | 3 +++ pkg/tests/BUILD | 17 +++++++++++++++++ pkg/tests/testdata/nested/nested.txt | 1 + pkg/tests/zip_test.py | 17 +++++++++++++++++ 4 files changed, 38 insertions(+) create mode 100644 pkg/tests/testdata/nested/nested.txt diff --git a/pkg/pkg.bzl b/pkg/pkg.bzl index f1d8e42f..48fad6e8 100644 --- a/pkg/pkg.bzl +++ b/pkg/pkg.bzl @@ -593,6 +593,8 @@ def _pkg_zip_impl(ctx): args.add("-d", ctx.attr.package_dir) args.add("-t", ctx.attr.timestamp) args.add("-m", ctx.attr.mode) + if ctx.attr.recurse: + args.add("-r") inputs = [] if ctx.attr.stamp == 1 or (ctx.attr.stamp == -1 and ctx.attr.private_stamp_detect): @@ -670,6 +672,7 @@ pkg_zip_impl = rule( "srcs": attr.label_list(allow_files = True), "strip_prefix": attr.string(), "timestamp": attr.int(default = 315532800), + "recurse": attr.bool(default = False), # Common attributes "out": attr.output(mandatory = True), diff --git a/pkg/tests/BUILD b/pkg/tests/BUILD index 5d9b252d..482a995f 100644 --- a/pkg/tests/BUILD +++ b/pkg/tests/BUILD @@ -322,6 +322,23 @@ filegroup( "/abc/def/", ])] +pkg_zip( + name = "test_zip_recurse", + srcs = glob( + ["testdata/**/*.txt"] + ), + recurse = True +) + +pkg_zip( + name = "test_zip_recurse_package_dir", + srcs = glob( + ["testdata/**/*.txt"] + ), + recurse = True, + package_dir = "abc/def" +) + py_test( name = "zip_test", srcs = [ diff --git a/pkg/tests/testdata/nested/nested.txt b/pkg/tests/testdata/nested/nested.txt new file mode 100644 index 00000000..4f5fb6a8 --- /dev/null +++ b/pkg/tests/testdata/nested/nested.txt @@ -0,0 +1 @@ +Nested Text \ No newline at end of file diff --git a/pkg/tests/zip_test.py b/pkg/tests/zip_test.py index adcd7fe9..3625709b 100644 --- a/pkg/tests/zip_test.py +++ b/pkg/tests/zip_test.py @@ -21,6 +21,7 @@ HELLO_CRC = 2069210904 LOREM_CRC = 2178844372 +NESTED_CRC = 143981220 EXECUTABLE_CRC = 342626072 # Unix dir bit and Windows dir bit. Magic from zip spec @@ -89,6 +90,22 @@ def test_basic(self): {"filename": "loremipsum.txt", "crc": LOREM_CRC}, ]) + def test_recurse(self): + self.assertZipFileContent("test_zip_recurse.zip", [ + {"filename": "nested/", "isdir": True, "attr": 0o711}, + {"filename": "nested/nested.txt", "crc": NESTED_CRC}, + {"filename": "hello.txt", "crc": HELLO_CRC}, + {"filename": "loremipsum.txt", "crc": LOREM_CRC}, + ]) + + def test_recurse_package_dir(self): + self.assertZipFileContent("test_zip_recurse.zip", [ + {"filename": "abc/def/nested/", "isdir": True, "attr": 0o711}, + {"filename": "abc/def/nested/nested.txt", "crc": NESTED_CRC}, + {"filename": "abc/def/hello.txt", "crc": HELLO_CRC}, + {"filename": "abc/def/loremipsum.txt", "crc": LOREM_CRC}, + ]) + def test_timestamp(self): self.assertZipFileContent("test_zip_timestamp.zip", [ {"filename": "hello.txt", "crc": HELLO_CRC, "timestamp": 1234567890}, From b540c7b04c89a56d7fd8bdedab623c8586889831 Mon Sep 17 00:00:00 2001 From: Harris Borawski Date: Mon, 4 Oct 2021 14:58:34 -0700 Subject: [PATCH 2/2] change strip_prefix to maintain original paths --- pkg/pkg.bzl | 4 +--- pkg/tests/BUILD | 18 ++++++++++++------ pkg/tests/zip_test.py | 16 +++++++--------- 3 files changed, 20 insertions(+), 18 deletions(-) diff --git a/pkg/pkg.bzl b/pkg/pkg.bzl index 48fad6e8..71363a03 100644 --- a/pkg/pkg.bzl +++ b/pkg/pkg.bzl @@ -593,8 +593,6 @@ def _pkg_zip_impl(ctx): args.add("-d", ctx.attr.package_dir) args.add("-t", ctx.attr.timestamp) args.add("-m", ctx.attr.mode) - if ctx.attr.recurse: - args.add("-r") inputs = [] if ctx.attr.stamp == 1 or (ctx.attr.stamp == -1 and ctx.attr.private_stamp_detect): @@ -622,7 +620,7 @@ def _pkg_zip_impl(ctx): ): # Add in the files of srcs which are not pkg_* types for f in src.files.to_list(): - d_path = dest_path(f, data_path, data_path_without_prefix) + d_path = dest_path(f, data_path, data_path_without_prefix) if not ctx.attr.recurse else dest_path(f, data_path_without_prefix, data_path_without_prefix) if f.is_directory: # Tree artifacts need a name, but the name is never really # the important part. The likely behavior people want is diff --git a/pkg/tests/BUILD b/pkg/tests/BUILD index 482a995f..2b98396c 100644 --- a/pkg/tests/BUILD +++ b/pkg/tests/BUILD @@ -324,17 +324,21 @@ filegroup( pkg_zip( name = "test_zip_recurse", - srcs = glob( - ["testdata/**/*.txt"] - ), + srcs = glob([ + "testdata/nested/*.txt", + "testdata/hello.txt", + "testdata/loremipsum.txt", + ]), recurse = True ) pkg_zip( name = "test_zip_recurse_package_dir", - srcs = glob( - ["testdata/**/*.txt"] - ), + srcs = glob([ + "testdata/nested/*.txt", + "testdata/hello.txt", + "testdata/loremipsum.txt", + ]), recurse = True, package_dir = "abc/def" ) @@ -354,6 +358,8 @@ py_test( "test-zip-strip_prefix-none.zip", "test-zip-strip_prefix-zipcontent.zip", "test-zip-strip_prefix-dot.zip", + "test_zip_recurse.zip", + "test_zip_recurse_package_dir.zip", # these could be replaced with diff_test() rules (from skylib) "test_zip_basic_timestamp_before_epoch.zip", diff --git a/pkg/tests/zip_test.py b/pkg/tests/zip_test.py index 3625709b..fa9db78b 100644 --- a/pkg/tests/zip_test.py +++ b/pkg/tests/zip_test.py @@ -92,18 +92,16 @@ def test_basic(self): def test_recurse(self): self.assertZipFileContent("test_zip_recurse.zip", [ - {"filename": "nested/", "isdir": True, "attr": 0o711}, - {"filename": "nested/nested.txt", "crc": NESTED_CRC}, - {"filename": "hello.txt", "crc": HELLO_CRC}, - {"filename": "loremipsum.txt", "crc": LOREM_CRC}, + {"filename": "testdata/hello.txt", "crc": HELLO_CRC}, + {"filename": "testdata/loremipsum.txt", "crc": LOREM_CRC}, + {"filename": "testdata/nested/nested.txt", "crc": NESTED_CRC}, ]) def test_recurse_package_dir(self): - self.assertZipFileContent("test_zip_recurse.zip", [ - {"filename": "abc/def/nested/", "isdir": True, "attr": 0o711}, - {"filename": "abc/def/nested/nested.txt", "crc": NESTED_CRC}, - {"filename": "abc/def/hello.txt", "crc": HELLO_CRC}, - {"filename": "abc/def/loremipsum.txt", "crc": LOREM_CRC}, + self.assertZipFileContent("test_zip_recurse_package_dir.zip", [ + {"filename": "abc/def/testdata/hello.txt", "crc": HELLO_CRC}, + {"filename": "abc/def/testdata/loremipsum.txt", "crc": LOREM_CRC}, + {"filename": "abc/def/testdata/nested/nested.txt", "crc": NESTED_CRC}, ]) def test_timestamp(self):