From 1c0f6cb2d2a80093530a06ae4361168f585c0c47 Mon Sep 17 00:00:00 2001 From: Lukas Larsson Date: Fri, 28 Jun 2024 09:00:49 +0200 Subject: [PATCH] fixup! zip: Cleanup testcases to be simpler and more stable --- lib/stdlib/test/zip_SUITE.erl | 134 +++++++++++++++++----------------- 1 file changed, 69 insertions(+), 65 deletions(-) diff --git a/lib/stdlib/test/zip_SUITE.erl b/lib/stdlib/test/zip_SUITE.erl index bac823946246..af5c03879404 100644 --- a/lib/stdlib/test/zip_SUITE.erl +++ b/lib/stdlib/test/zip_SUITE.erl @@ -37,6 +37,8 @@ basic_timestamp/1, extended_timestamp/1, uid_gid/1]). +-export([zip/5, unzip/3]). + -import(proplists,[get_value/2, get_value/3]). -include_lib("common_test/include/ct.hrl"). @@ -84,14 +86,6 @@ z64(Mode) when is_atom(Mode) -> z64(Modes) when is_list(Modes) -> [z64(M) || M <- Modes]. -noz64(Z64Mode) -> - case string:split(atom_to_list(Z64Mode), "_") of - ["z64",Mode] -> - list_to_atom(Mode); - [_Mode] -> - Z64Mode - end. - zip_testcases() -> [mode, basic_timestamp, extended_timestamp, uid_gid]. @@ -147,11 +141,11 @@ init_per_group(Group, Config) -> undefined -> Pdir = filename:join(get_value(priv_dir, Config),Group), ok = filelib:ensure_path(Pdir), - [{pdir, Pdir},{zip, noz64(Group)} | Config]; + [{pdir, Pdir},{zip, Group} | Config]; _Zip -> Pdir = filename:join(get_value(pdir, Config),Group), ok = filelib:ensure_path(Pdir), - [{pdir, Pdir},{unzip, noz64(Group)} | Config] + [{pdir, Pdir},{unzip, Group} | Config] end; false -> Config @@ -1569,57 +1563,63 @@ uid_gid(Config) -> zip(Config, Archive, ZipOpts, Filelist, Opts) when is_list(Config) -> zip(get_value(zip, Config), Archive, ZipOpts, Filelist, Opts); +zip(z64_zip, Archive, ZipOpts, Filelist, Opts) -> + zip(zip, Archive, ZipOpts, Filelist, Opts); zip(zip, Archive, ZipOpts, Filelist, Opts) -> cmd("cd "++get_value(cwd, Opts)++" && " "zip "++ZipOpts++" "++Archive++" "++lists:join($ ,Filelist)), {ok, Archive}; +zip(z64_ezip, Archive, _ZipOpts, Filelist, Opts) -> + zip(ezip, Archive, _ZipOpts, Filelist, Opts); zip(ezip, Archive, _ZipOpts, Filelist, Opts) -> ct:log("Creating zip:zip(~p,~n~p,~n~p)",[Archive, Filelist, Opts]), zip:zip(Archive, Filelist, Opts); -zip(emzip, Archive, _ZipOpts, Filelist, Opts) -> - ct:log("Creating emzip ~ts",[Archive]), - Cwd = get_value(cwd, Opts), - +zip(z64_emzip, Archive, _ZipOpts, Filelist, Opts) -> %% Run in peer node so that memory issues don't crash test node {ok, Peer, Node} = ?CT_PEER(#{ args => emzip_peer_args() }), try erpc:call( Node, fun() -> - %% For this not to use a huge amount of memory we re-use - %% the binary for files that are the same size as those are the same file. - %% This cuts memory usage from ~16GB to ~4GB. - - {Files,_Cache} = - lists:mapfoldl( - fun F(Fn, Cache) -> - AbsFn = filename:join(Cwd, Fn), - {ok, Fi} = file:read_file_info(AbsFn), - CacheKey = {Fi#file_info.type, Fi#file_info.size}, - {SubDirFiles, NewCache} = - if Fi#file_info.type == directory -> - {ok, Files} = file:list_dir(AbsFn), - lists:mapfoldl(F, Cache#{ CacheKey => <<>> }, - [filename:join(Fn, DirFn) || DirFn <- Files]); - Fi#file_info.type == regular -> - {[], - case maps:find(CacheKey, Cache) of - {ok, _} -> Cache; - error -> - {ok, Data} = read_file( - file:open(AbsFn, [read, raw, binary]), - Fi#file_info.size), - Cache#{ CacheKey => Data } - end} - end, - {[{Fn, maps:get(CacheKey, NewCache), Fi}|SubDirFiles], NewCache} - end, #{}, Filelist), - zip:zip(Archive, lists:flatten(Files), proplists:delete(cwd,Opts)) + ?MODULE:zip(emzip, Archive, _ZipOpts, Filelist, Opts) end) after peer:stop(Peer) - end. + end; +zip(emzip, Archive, _ZipOpts, Filelist, Opts) -> + ct:log("Creating emzip ~ts",[Archive]), + Cwd = get_value(cwd, Opts), + + %% For this not to use a huge amount of memory we re-use + %% the binary for files that are the same size as those are the same file. + %% This cuts memory usage from ~16GB to ~4GB. + + {Files,_Cache} = + lists:mapfoldl( + fun F(Fn, Cache) -> + AbsFn = filename:join(Cwd, Fn), + {ok, Fi} = file:read_file_info(AbsFn), + CacheKey = {Fi#file_info.type, Fi#file_info.size}, + {SubDirFiles, NewCache} = + if Fi#file_info.type == directory -> + {ok, Files} = file:list_dir(AbsFn), + lists:mapfoldl(F, Cache#{ CacheKey => <<>> }, + [filename:join(Fn, DirFn) || DirFn <- Files]); + Fi#file_info.type == regular -> + {[], + case maps:find(CacheKey, Cache) of + {ok, _} -> Cache; + error -> + {ok, Data} = read_file( + file:open(AbsFn, [read, raw, binary]), + Fi#file_info.size), + Cache#{ CacheKey => Data } + end} + end, + {[{Fn, maps:get(CacheKey, NewCache), Fi}|SubDirFiles], NewCache} + end, #{}, Filelist), + zip:zip(Archive, lists:flatten(Files), proplists:delete(cwd,Opts)). %% Special read_file that works on windows on > 4 GB files read_file({ok, D}, Size) -> @@ -1639,6 +1639,8 @@ read_file(D, Size) -> unzip(Config, Archive, Opts) when is_list(Config) -> unzip(get_value(unzip, Config), Archive, Opts); +unzip(z64_unzip, Archive, Opts) -> + unzip(unzip, Archive, Opts); unzip(unzip, Archive, Opts) -> UidGid = [" -X " || lists:member(uid_gid, get_value(extra, Opts, []))], Files = lists:join($ , get_value(file_list, Opts, [])), @@ -1652,41 +1654,43 @@ unzip(unzip, Archive, Opts) -> {match,Match} -> Match end end,string:split(Res,"\n",all)))}; +unzip(z64_unezip, Archive, Opts) -> + unzip(unezip, Archive, Opts); unzip(unezip, Archive, Opts) -> Cwd = get_value(cwd, Opts) ++ "/", {ok, Files} = zip:unzip(Archive, Opts), {ok, lists:sort([F -- Cwd || F <- Files])}; -unzip(unemzip, Archive, Opts) -> - Cwd = get_value(cwd, Opts) ++ "/", - +unzip(z64_unemzip, Archive, Opts) -> %% Run in peer node so that memory issues don't crash test node {ok, Peer, Node} = ?CT_PEER(#{ args => emzip_peer_args() }), try erpc:call( Node, fun() -> - {ok, Files} = zip:unzip(Archive, [memory | Opts]), - {ok, lists:sort( - [begin - case lists:last(F) of - $/ -> - filelib:ensure_path(F); - _ -> - filelib:ensure_dir(F), - file:write_file(F, B) - end, - F -- Cwd - end || {F, B} <- Files])} + unzip(unemzip, Archive, Opts) end) after peer:stop(Peer) - end. + end; +unzip(unemzip, Archive, Opts) -> + Cwd = get_value(cwd, Opts) ++ "/", + + {ok, Files} = zip:unzip(Archive, [memory | Opts]), + {ok, lists:sort( + [begin + case lists:last(F) of + $/ -> + filelib:ensure_path(F); + _ -> + filelib:ensure_dir(F), + file:write_file(F, B) + end, + F -- Cwd + end || {F, B} <- Files])}. emzip_peer_args() -> - case erlang:system_info(wordsize) of - 8 -> ["+MMscs","809600"]; %% Supercarrier only supported on 64-bit - 4 -> [] - end. + 8 = erlang:system_info(wordsize),%% Supercarrier only supported on 64-bit + ["+MMscs","8096"]. cmp(Source, Target) -> {ok, SrcInfo} = file:read_file_info(Source), @@ -1704,7 +1708,7 @@ cmp(Source, Target) -> %% Check if first 100 MB are the same cmp(Src, Tgt, Pos) when Pos < 100 bsl 20 -> - + erlang:garbage_collect(), case {file:read(Src, 20 bsl 20), file:read(Tgt, 20 bsl 20)} of {{ok, Data}, {ok, Data}} -> cmp(Src, Tgt, Pos + 20 bsl 20);