From 5a06c3ad03080dd5fd14402c7a69d4815fe7a86a Mon Sep 17 00:00:00 2001 From: MouriNaruto Date: Fri, 6 Sep 2024 23:38:18 +0800 Subject: [PATCH] Add NanaZip.Core.Console project. --- NanaZip.Core/NanaZip.Core.Console.manifest | 12 + NanaZip.Core/NanaZip.Core.Console.vcxproj | 283 + .../NanaZip.Core.Console.vcxproj.filters | 686 +++ .../CPP/7zip/Common/MultiOutStream.cpp | 855 +++ .../SevenZip/CPP/7zip/Common/MultiOutStream.h | 160 + .../CPP/7zip/UI/Common/ArchiveCommandLine.cpp | 1769 ++++++ .../SevenZip/CPP/7zip/UI/Common/Bench.cpp | 4941 +++++++++++++++++ .../SevenZip/CPP/7zip/UI/Common/Bench.h | 121 + .../CPP/7zip/UI/Common/EnumDirItems.cpp | 1654 ++++++ .../SevenZip/CPP/7zip/UI/Common/HashCalc.cpp | 2096 +++++++ .../CPP/7zip/UI/Common/LoadCodecs.cpp | 26 +- .../CPP/7zip/UI/Common/SetProperties.cpp | 88 + .../SevenZip/CPP/7zip/UI/Common/TempFiles.cpp | 19 + .../SevenZip/CPP/7zip/UI/Common/TempFiles.h | 16 + .../SevenZip/CPP/7zip/UI/Common/Update.cpp | 1880 +++++++ .../CPP/7zip/UI/Common/UpdateAction.cpp | 64 + .../CPP/7zip/UI/Common/UpdateCallback.cpp | 1029 ++++ .../CPP/7zip/UI/Common/UpdatePair.cpp | 302 + .../CPP/7zip/UI/Common/UpdateProduce.cpp | 74 + .../SevenZip/CPP/7zip/UI/Console/BenchCon.cpp | 41 + .../SevenZip/CPP/7zip/UI/Console/BenchCon.h | 14 + .../SevenZip/CPP/7zip/UI/Console/HashCon.cpp | 426 ++ .../SevenZip/CPP/7zip/UI/Console/HashCon.h | 58 + .../SevenZip/CPP/7zip/UI/Console/Main.cpp | 1643 ++++++ .../7zip/UI/Console/UpdateCallbackConsole.cpp | 885 +++ .../7zip/UI/Console/UpdateCallbackConsole.h | 138 + .../SevenZip/CPP/Windows/FileSystem.cpp | 187 + .../SevenZip/CPP/Windows/FileSystem.h | 31 + .../SevenZip/CPP/Windows/SystemInfo.cpp | 1170 ++++ .../SevenZip/CPP/Windows/SystemInfo.h | 19 + NanaZip.sln | 15 + 31 files changed, 20698 insertions(+), 4 deletions(-) create mode 100644 NanaZip.Core/NanaZip.Core.Console.manifest create mode 100644 NanaZip.Core/NanaZip.Core.Console.vcxproj create mode 100644 NanaZip.Core/NanaZip.Core.Console.vcxproj.filters create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.h create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.h create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/EnumDirItems.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/HashCalc.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/SetProperties.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.h create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Update.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateAction.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateCallback.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdatePair.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateProduce.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.h create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.h create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/Main.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.h create mode 100644 NanaZip.Core/SevenZip/CPP/Windows/FileSystem.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/Windows/FileSystem.h create mode 100644 NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.cpp create mode 100644 NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.h diff --git a/NanaZip.Core/NanaZip.Core.Console.manifest b/NanaZip.Core/NanaZip.Core.Console.manifest new file mode 100644 index 000000000..0bda7b619 --- /dev/null +++ b/NanaZip.Core/NanaZip.Core.Console.manifest @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/NanaZip.Core/NanaZip.Core.Console.vcxproj b/NanaZip.Core/NanaZip.Core.Console.vcxproj new file mode 100644 index 000000000..d485d62f5 --- /dev/null +++ b/NanaZip.Core/NanaZip.Core.Console.vcxproj @@ -0,0 +1,283 @@ + + + + {86E818B3-D657-4E03-9336-48EE242A79D1} + NanaZip.Core.Console + ConsoleApplication + 10.0.19041.0 + YY_Thunks_for_Vista.obj + NanaZip.Core.Console.manifest + true + true + M2-Team + NanaZip Command Line Interface (Console) + NanaZipConsole + © M2-Team and Contributors. All rights reserved. + NanaZipC.exe + NanaZip + 3.5.$([System.DateTime]::Today.Subtract($([System.DateTime]::Parse('2021-08-31'))).TotalDays).0 + Preview 1 + + + + + + + + + + + %(AdditionalOptions) /Wv:18 + Z7_DEVICE_FILE;Z7_EXTERNAL_CODECS;WINRT_NO_SOURCE_LOCATION;%(PreprocessorDefinitions) + + + true + + + + + + + + 1.0.645 + + + + + {5220420B-9A5C-44A7-BE69-97F25365BB26} + + + {96C0A1A0-D964-4725-AFDC-73EBF7FC1416} + Platform=Win32 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + MaxSpeed + + + + + + \ No newline at end of file diff --git a/NanaZip.Core/NanaZip.Core.Console.vcxproj.filters b/NanaZip.Core/NanaZip.Core.Console.vcxproj.filters new file mode 100644 index 000000000..2850973de --- /dev/null +++ b/NanaZip.Core/NanaZip.Core.Console.vcxproj.filters @@ -0,0 +1,686 @@ + + + + + + + + {8133bf4b-9163-4785-bf6d-7c3a28e7d068} + + + {55256e55-9ca8-4668-b1af-34a41efbbdb6} + + + {4fe3f75f-184d-4e23-ac76-94a75f3b0f85} + + + {061c9b33-f410-4ff4-be1d-13f382494139} + + + {09b5197c-cefb-46b7-a463-7994ccb2b92b} + + + {bfa15001-6486-4ac3-900c-429290e0ccc7} + + + {7c1b2fa5-b64f-4b30-81f6-af2d93780182} + + + {45d898b8-5962-46f9-a841-bcce1301abb7} + + + {4433c122-4a0c-48ca-82f8-e6f8731ad143} + + + {7f7932cc-3ad6-44f7-8ce8-9cbcd4370782} + + + {4c862da4-1e63-439c-8a92-c3887bc8c96a} + + + {88688397-460c-496c-876b-492e7c91956a} + + + {e5754918-8976-479f-ace8-91ff5e7f534c} + + + {764ac24d-a2a7-4b50-b0f4-fdc23700b641} + + + {ab36f77e-08ab-4c0b-8bab-f892701fc4ed} + + + + + SevenZip\ASM\x86 + + + SevenZip\ASM\x86 + + + SevenZip\C + + + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\7zip\Archive\Common + + + SevenZip\CPP\7zip\Archive\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Compress + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\C + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Common + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\7zip\Archive\Common + + + SevenZip\CPP\7zip\Archive\Common + + + SevenZip\CPP\7zip\Archive\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\Compress + + + SevenZip\CPP\7zip\Compress + + + SevenZip\CPP\Common + + + SevenZip\CPP\7zip + + + SevenZip\CPP\7zip + + + SevenZip\CPP\7zip + + + SevenZip\CPP\7zip + + + SevenZip\CPP\Common + + + SevenZip\CPP\7zip + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\Common + + + SevenZip\CPP\Common + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Windows + + + SevenZip\CPP\7zip\Archive + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\CPP\7zip\UI\Common + + + SevenZip\C + + + SevenZip\CPP\Windows + + + SevenZip\CPP\Common + + + SevenZip\CPP\Windows + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\7zip\UI\Console + + + SevenZip\CPP\Common + + + SevenZip\CPP\7zip + + + SevenZip\C + + + \ No newline at end of file diff --git a/NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.cpp b/NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.cpp new file mode 100644 index 000000000..82d10d480 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.cpp @@ -0,0 +1,855 @@ +// MultiOutStream.cpp + +#include "StdAfx.h" + +// #define DEBUG_VOLUMES + +#ifdef DEBUG_VOLUMES +#include + #define PRF(x) x; +#else + #define PRF(x) +#endif + +#include "../../Common/ComTry.h" + +#include "../../Windows/FileDir.h" +#include "../../Windows/FileFind.h" +#include "../../Windows/System.h" + +#include "MultiOutStream.h" + +using namespace NWindows; +using namespace NFile; +using namespace NDir; + +static const unsigned k_NumVols_MAX = k_VectorSizeMax - 1; + // 2; // for debug + +/* +#define UPDATE_HRES(hres, x) \ + { const HRESULT res2 = (x); if (hres == SZ_OK) hres = res2; } +*/ + +HRESULT CMultiOutStream::Destruct() +{ + COM_TRY_BEGIN + HRESULT hres = S_OK; + HRESULT hres3 = S_OK; + + while (!Streams.IsEmpty()) + { + try + { + HRESULT hres2; + if (NeedDelete) + { + /* we could call OptReOpen_and_SetSize() to test that we try to delete correct file, + but we cannot guarantee that (RealSize) will be correct after Write() or another failures. + And we still want to delete files even for such cases. + So we don't check for OptReOpen_and_SetSize() here: */ + // if (OptReOpen_and_SetSize(Streams.Size() - 1, 0) == S_OK) + hres2 = CloseStream_and_DeleteFile(Streams.Size() - 1); + } + else + { + hres2 = CloseStream(Streams.Size() - 1); + } + if (hres == S_OK) + hres = hres2; + } + catch(...) + { + hres3 = E_OUTOFMEMORY; + } + + { + /* Stream was released in CloseStream_*() above already, and it was removed from linked list + it's some unexpected case, if Stream is still attached here. + So the following code is optional: */ + CVolStream &s = Streams.Back(); + if (s.Stream) + { + if (hres3 == S_OK) + hres3 = E_FAIL; + s.Stream.Detach(); + /* it will be not failure, even if we call RemoveFromLinkedList() + twice for same CVolStream in this Destruct() function */ + RemoveFromLinkedList(Streams.Size() - 1); + } + } + Streams.DeleteBack(); + // Delete_LastStream_Records(); + } + + if (hres == S_OK) + hres = hres3; + if (hres == S_OK && NumListItems != 0) + hres = E_FAIL; + return hres; + COM_TRY_END +} + + +CMultiOutStream::~CMultiOutStream() +{ + // we try to avoid exception in destructors + Destruct(); +} + + +void CMultiOutStream::Init(const CRecordVector &sizes) +{ + Streams.Clear(); + InitLinkedList(); + Sizes = sizes; + NeedDelete = true; + MTime_Defined = false; + FinalVol_WasReopen = false; + NumOpenFiles_AllowedMax = NSystem::Get_File_OPEN_MAX_Reduced_for_3_tasks(); + + _streamIndex = 0; + _offsetPos = 0; + _absPos = 0; + _length = 0; + _absLimit = (UInt64)(Int64)-1; + + _restrict_Begin = 0; + _restrict_End = (UInt64)(Int64)-1; + _restrict_Global = 0; + + UInt64 sum = 0; + unsigned i = 0; + for (i = 0; i < Sizes.Size(); i++) + { + if (i >= k_NumVols_MAX) + { + _absLimit = sum; + break; + } + const UInt64 size = Sizes[i]; + const UInt64 next = sum + size; + if (next < sum) + break; + sum = next; + } + + // if (Sizes.IsEmpty()) throw "no volume sizes"; + const UInt64 size = Sizes.Back(); + if (size == 0) + throw "zero size last volume"; + + if (i == Sizes.Size()) + if ((_absLimit - sum) / size >= (k_NumVols_MAX - i)) + _absLimit = sum + (k_NumVols_MAX - i) * size; +} + + +/* IsRestricted(): + we must call only if volume is full (s.RealSize==VolSize) or finished. + the function doesn't use VolSize and it uses s.RealSize instead. + it returns true : if stream is restricted, and we can't close that stream + it returns false : if there is no restriction, and we can close that stream + Note: (RealSize == 0) (empty volume) on restriction bounds are supposed as non-restricted +*/ +bool CMultiOutStream::IsRestricted(const CVolStream &s) const +{ + if (s.Start < _restrict_Global) + return true; + if (_restrict_Begin == _restrict_End) + return false; + if (_restrict_Begin <= s.Start) + return _restrict_End > s.Start; + return _restrict_Begin < s.Start + s.RealSize; +} + +/* +// this function check also _length and volSize +bool CMultiOutStream::IsRestricted_for_Close(unsigned index) const +{ + const CVolStream &s = Streams[index]; + if (_length <= s.Start) // we don't close streams after the end, because we still can write them later + return true; + // (_length > s.Start) + const UInt64 volSize = GetVolSize_for_Stream(index); + if (volSize == 0) + return IsRestricted_Empty(s); + if (_length - s.Start < volSize) + return true; + return IsRestricted(s); +} +*/ + +FString CMultiOutStream::GetFilePath(unsigned index) +{ + FString name; + name.Add_UInt32((UInt32)(index + 1)); + while (name.Len() < 3) + name.InsertAtFront(FTEXT('0')); + name.Insert(0, Prefix); + return name; +} + + +// we close stream, but we still keep item in Streams[] vector +HRESULT CMultiOutStream::CloseStream(unsigned index) +{ + CVolStream &s = Streams[index]; + if (s.Stream) + { + RINOK(s.StreamSpec->Close()) + // the following two commands must be called together: + s.Stream.Release(); + RemoveFromLinkedList(index); + } + return S_OK; +} + + +// we close stream and delete file, but we still keep item in Streams[] vector +HRESULT CMultiOutStream::CloseStream_and_DeleteFile(unsigned index) +{ + PRF(printf("\n====== %u, CloseStream_AndDelete \n", index)) + RINOK(CloseStream(index)) + FString path = GetFilePath(index); + path += Streams[index].Postfix; + // we can checki that file exist + // if (NFind::DoesFileExist_Raw(path)) + if (!DeleteFileAlways(path)) + return GetLastError_noZero_HRESULT(); + return S_OK; +} + + +HRESULT CMultiOutStream::CloseStream_and_FinalRename(unsigned index) +{ + PRF(printf("\n====== %u, CloseStream_and_FinalRename \n", index)) + CVolStream &s = Streams[index]; + // HRESULT res = S_OK; + bool mtime_WasSet = false; + if (MTime_Defined && s.Stream) + { + if (s.StreamSpec->SetMTime(&MTime)) + mtime_WasSet = true; + // else res = GetLastError_noZero_HRESULT(); + } + + RINOK(CloseStream(index)) + if (s.Postfix.IsEmpty()) // if Postfix is empty, the path is already final + return S_OK; + const FString path = GetFilePath(index); + FString tempPath = path; + tempPath += s.Postfix; + + if (MTime_Defined && !mtime_WasSet) + { + if (!SetDirTime(tempPath, NULL, NULL, &MTime)) + { + // res = GetLastError_noZero_HRESULT(); + } + } + if (!MyMoveFile(tempPath, path)) + return GetLastError_noZero_HRESULT(); + /* we clear CVolStream::Postfix. So we will not use Temp path + anymore for this stream, and we will work only with final path */ + s.Postfix.Empty(); + // we can ignore set_mtime error or we can return it + return S_OK; + // return res; +} + + +HRESULT CMultiOutStream::PrepareToOpenNew() +{ + PRF(printf("PrepareToOpenNew NumListItems =%u, NumOpenFiles_AllowedMax = %u \n", NumListItems, NumOpenFiles_AllowedMax)) + + if (NumListItems < NumOpenFiles_AllowedMax) + return S_OK; + /* when we create zip archive: in most cases we need only starting + data of restricted region for rewriting zip's local header. + So here we close latest created volume (from Head), and we try to + keep oldest volumes that will be used for header rewriting later. */ + const int index = Head; + if (index == -1) + return E_FAIL; + PRF(printf("\n== %u, PrepareToOpenNew::CloseStream, NumListItems =%u \n", index, NumListItems)) + /* we don't expect non-restricted stream here in normal cases (if _restrict_Global was not changed). + if there was non-restricted stream, it should be closed before */ + // if (!IsRestricted_for_Close(index)) return CloseStream_and_FinalRename(index); + return CloseStream((unsigned)index); +} + + +HRESULT CMultiOutStream::CreateNewStream(UInt64 newSize) +{ + PRF(printf("\n== %u, CreateNewStream, size =%u \n", Streams.Size(), (unsigned)newSize)) + + if (Streams.Size() >= k_NumVols_MAX) + return E_INVALIDARG; // E_OUTOFMEMORY + + RINOK(PrepareToOpenNew()) + CVolStream s; + s.StreamSpec = new COutFileStream; + s.Stream = s.StreamSpec; + const FString path = GetFilePath(Streams.Size()); + + if (NFind::DoesFileExist_Raw(path)) + return HRESULT_FROM_WIN32(ERROR_ALREADY_EXISTS); + if (!CreateTempFile2(path, false, s.Postfix, &s.StreamSpec->File)) + return GetLastError_noZero_HRESULT(); + + s.Start = GetGlobalOffset_for_NewStream(); + s.Pos = 0; + s.RealSize = 0; + + const unsigned index = Streams.Add(s); + InsertToLinkedList(index); + + if (newSize != 0) + return s.SetSize2(newSize); + return S_OK; +} + + +HRESULT CMultiOutStream::CreateStreams_If_Required(unsigned streamIndex) +{ + // UInt64 lastStreamSize = 0; + for (;;) + { + const unsigned numStreamsBefore = Streams.Size(); + if (streamIndex < numStreamsBefore) + return S_OK; + UInt64 newSize; + if (streamIndex == numStreamsBefore) + { + // it's final volume that will be used for real writing. + /* SetSize(_offsetPos) is not required, + because the file Size will be set later by calling Seek() with Write() */ + newSize = 0; // lastStreamSize; + } + else + { + // it's intermediate volume. So we need full volume size + newSize = GetVolSize_for_Stream(numStreamsBefore); + } + + RINOK(CreateNewStream(newSize)) + + // optional check + if (numStreamsBefore + 1 != Streams.Size()) return E_FAIL; + + if (streamIndex != numStreamsBefore) + { + // it's intermediate volume. So we can close it, if it's non-restricted + bool isRestricted; + { + const CVolStream &s = Streams[numStreamsBefore]; + if (newSize == 0) + isRestricted = IsRestricted_Empty(s); + else + isRestricted = IsRestricted(s); + } + if (!isRestricted) + { + RINOK(CloseStream_and_FinalRename(numStreamsBefore)) + } + } + } +} + + +HRESULT CMultiOutStream::ReOpenStream(unsigned streamIndex) +{ + PRF(printf("\n====== %u, ReOpenStream \n", streamIndex)) + RINOK(PrepareToOpenNew()) + CVolStream &s = Streams[streamIndex]; + + FString path = GetFilePath(streamIndex); + path += s.Postfix; + + s.StreamSpec = new COutFileStream; + s.Stream = s.StreamSpec; + s.Pos = 0; + + HRESULT hres; + if (s.StreamSpec->Open_EXISTING(path)) + { + if (s.Postfix.IsEmpty()) + { + /* it's unexpected case that we open finished volume. + It can mean that the code for restriction is incorrect */ + FinalVol_WasReopen = true; + } + UInt64 realSize = 0; + hres = s.StreamSpec->GetSize(&realSize); + if (hres == S_OK) + { + if (realSize == s.RealSize) + { + PRF(printf("\n ReOpenStream OK realSize = %u\n", (unsigned)realSize)) + InsertToLinkedList(streamIndex); + return S_OK; + } + // file size was changed between Close() and ReOpen() + // we must release Stream to be consistent with linked list + hres = E_FAIL; + } + } + else + hres = GetLastError_noZero_HRESULT(); + s.Stream.Release(); + s.StreamSpec = NULL; + return hres; +} + + +/* Sets size of stream, if new size is not equal to old size (RealSize). + If stream was closed and size change is required, it reopens the stream. */ + +HRESULT CMultiOutStream::OptReOpen_and_SetSize(unsigned index, UInt64 size) +{ + CVolStream &s = Streams[index]; + if (size == s.RealSize) + return S_OK; + if (!s.Stream) + { + RINOK(ReOpenStream(index)) + } + PRF(printf("\n== %u, OptReOpen_and_SetSize, size =%u RealSize = %u\n", index, (unsigned)size, (unsigned)s.RealSize)) + // comment it to debug tail after data + return s.SetSize2(size); +} + + +/* +call Normalize_finalMode(false), if _length was changed. + for all streams starting after _length: + - it sets zero size + - it still keeps file open + Note: after _length reducing with CMultiOutStream::SetSize() we can + have very big number of empty streams at the end of Streams[] list. + And Normalize_finalMode() will runs all these empty streams of Streams[] vector. + So it can be ineffective, if we call Normalize_finalMode() many + times after big reducing of (_length). + +call Normalize_finalMode(true) to set final presentations of all streams + for all streams starting after _length: + - it sets zero size + - it removes file + - it removes CVolStream object from Streams[] vector + +Note: we don't remove zero sized first volume, if (_length == 0) +*/ + +HRESULT CMultiOutStream::Normalize_finalMode(bool finalMode) +{ + PRF(printf("\n== Normalize_finalMode: _length =%d \n", (unsigned)_length)) + + unsigned i = Streams.Size(); + + UInt64 offset = 0; + + /* At first we normalize (reduce or increase) the sizes of all existing + streams in Streams[] that can be affected by changed _length. + And we remove tailing zero-size streams, if (finalMode == true) */ + while (i != 0) + { + offset = Streams[--i].Start; // it's last item in Streams[] + // we don't want to remove first volume + if (offset < _length || i == 0) + { + const UInt64 volSize = GetVolSize_for_Stream(i); + UInt64 size = _length - offset; // (size != 0) here + if (size > volSize) + size = volSize; + RINOK(OptReOpen_and_SetSize(i, size)) + if (_length - offset <= volSize) + return S_OK; + // _length - offset > volSize + offset += volSize; + // _length > offset + break; + // UPDATE_HRES(res, OptReOpen_and_SetSize(i, size)); + } + + /* we Set Size of stream to zero even for (finalMode==true), although + that stream will be deleted in next commands */ + // UPDATE_HRES(res, OptReOpen_and_SetSize(i, 0)); + RINOK(OptReOpen_and_SetSize(i, 0)) + if (finalMode) + { + RINOK(CloseStream_and_DeleteFile(i)) + /* CVolStream::Stream was released above already, and it was + removed from linked list. So we don't need to update linked list + structure, when we delete last item in Streams[] */ + Streams.DeleteBack(); + // Delete_LastStream_Records(); + } + } + + /* now we create new zero-filled streams to cover all data up to _length */ + + if (_length == 0) + return S_OK; + + // (offset) is start offset of next stream after existing Streams[] + + for (;;) + { + // _length > offset + const UInt64 volSize = GetVolSize_for_Stream(Streams.Size()); + UInt64 size = _length - offset; // (size != 0) here + if (size > volSize) + size = volSize; + RINOK(CreateNewStream(size)) + if (_length - offset <= volSize) + return S_OK; + // _length - offset > volSize) + offset += volSize; + // _length > offset + } +} + + +HRESULT CMultiOutStream::FinalFlush_and_CloseFiles(unsigned &numTotalVolumesRes) +{ + // at first we remove unused zero-sized streams after _length + HRESULT res = Normalize_finalMode(true); + numTotalVolumesRes = Streams.Size(); + FOR_VECTOR (i, Streams) + { + const HRESULT res2 = CloseStream_and_FinalRename(i); + if (res == S_OK) + res = res2; + } + if (NumListItems != 0 && res == S_OK) + res = E_FAIL; + return res; +} + + +bool CMultiOutStream::SetMTime_Final(const CFiTime &mTime) +{ + // we will set mtime only if new value differs from previous + if (!FinalVol_WasReopen && MTime_Defined && Compare_FiTime(&MTime, &mTime) == 0) + return true; + bool res = true; + FOR_VECTOR (i, Streams) + { + CVolStream &s = Streams[i]; + if (s.Stream) + { + if (!s.StreamSpec->SetMTime(&mTime)) + res = false; + } + else + { + if (!SetDirTime(GetFilePath(i), NULL, NULL, &mTime)) + res = false; + } + } + return res; +} + + +Z7_COM7F_IMF(CMultiOutStream::SetSize(UInt64 newSize)) +{ + COM_TRY_BEGIN + if ((Int64)newSize < 0) + return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; + if (newSize > _absLimit) + { + /* big seek value was sent to SetSize() or to Seek()+Write(). + It can mean one of two situations: + 1) some incorrect code called it with big seek value. + 2) volume size was small, and we have too big number of volumes + */ + /* in Windows SetEndOfFile() can return: + ERROR_NEGATIVE_SEEK: for >= (1 << 63) + ERROR_INVALID_PARAMETER: for > (16 TiB - 64 KiB) + ERROR_DISK_FULL: for <= (16 TiB - 64 KiB) + */ + // return E_FAIL; + // return E_OUTOFMEMORY; + return E_INVALIDARG; + } + + if (newSize > _length) + { + // we don't expect such case. So we just define global restriction */ + _restrict_Global = newSize; + } + else if (newSize < _restrict_Global) + _restrict_Global = newSize; + + PRF(printf("\n== CMultiOutStream::SetSize, size =%u \n", (unsigned)newSize)) + + _length = newSize; + return Normalize_finalMode(false); + + COM_TRY_END +} + + +Z7_COM7F_IMF(CMultiOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + COM_TRY_BEGIN + if (processedSize) + *processedSize = 0; + if (size == 0) + return S_OK; + + PRF(printf("\n -- CMultiOutStream::Write() : _absPos = %6u, size =%6u \n", + (unsigned)_absPos, (unsigned)size)) + + if (_absPos > _length) + { + // it create data only up to _absPos. + // but we still can need additional new streams, if _absPos at range of volume + RINOK(SetSize(_absPos)) + } + + while (size != 0) + { + UInt64 volSize; + { + if (_streamIndex < Sizes.Size() - 1) + { + volSize = Sizes[_streamIndex]; + if (_offsetPos >= volSize) + { + _offsetPos -= volSize; + _streamIndex++; + continue; + } + } + else + { + volSize = Sizes[Sizes.Size() - 1]; + if (_offsetPos >= volSize) + { + const UInt64 v = _offsetPos / volSize; + if (v >= ((UInt32)(Int32)-1) - _streamIndex) + return E_INVALIDARG; + // throw 202208; + _streamIndex += (unsigned)v; + _offsetPos -= (unsigned)v * volSize; + } + if (_streamIndex >= k_NumVols_MAX) + return E_INVALIDARG; + } + } + + // (_offsetPos < volSize) here + + /* we can need to create one or more streams here, + vol_size for some streams is allowed to be 0. + Also we close some new created streams, if they are non-restricted */ + // file Size will be set later by calling Seek() with Write() + + /* the case (_absPos > _length) was processed above with SetSize(_absPos), + so here it's expected. that we can create optional zero-size streams and then _streamIndex */ + RINOK(CreateStreams_If_Required(_streamIndex)) + + CVolStream &s = Streams[_streamIndex]; + + PRF(printf("\n%d, == Write : Pos = %u, RealSize = %u size =%u \n", + _streamIndex, (unsigned)s.Pos, (unsigned)s.RealSize, size)) + + if (!s.Stream) + { + RINOK(ReOpenStream(_streamIndex)) + } + if (_offsetPos != s.Pos) + { + RINOK(s.Stream->Seek((Int64)_offsetPos, STREAM_SEEK_SET, NULL)) + s.Pos = _offsetPos; + } + + UInt32 curSize = size; + { + const UInt64 rem = volSize - _offsetPos; + if (curSize > rem) + curSize = (UInt32)rem; + } + // curSize != 0 + UInt32 realProcessed = 0; + + HRESULT hres = s.Stream->Write(data, curSize, &realProcessed); + + data = (const void *)((const Byte *)data + realProcessed); + size -= realProcessed; + s.Pos += realProcessed; + _offsetPos += realProcessed; + _absPos += realProcessed; + if (_length < _absPos) + _length = _absPos; + if (s.RealSize < _offsetPos) + s.RealSize = _offsetPos; + if (processedSize) + *processedSize += realProcessed; + + if (s.Pos == volSize) + { + bool isRestricted; + if (volSize == 0) + isRestricted = IsRestricted_Empty(s); + else + isRestricted = IsRestricted(s); + if (!isRestricted) + { + const HRESULT res2 = CloseStream_and_FinalRename(_streamIndex); + if (hres == S_OK) + hres = res2; + } + _streamIndex++; + _offsetPos = 0; + } + + RINOK(hres) + if (realProcessed == 0 && curSize != 0) + return E_FAIL; + // break; + } + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CMultiOutStream::Seek(Int64 offset, UInt32 seekOrigin, UInt64 *newPosition)) +{ + PRF(printf("\n-- CMultiOutStream::Seek seekOrigin=%u Seek =%u\n", seekOrigin, (unsigned)offset)) + + switch (seekOrigin) + { + case STREAM_SEEK_SET: break; + case STREAM_SEEK_CUR: offset += _absPos; break; + case STREAM_SEEK_END: offset += _length; break; + default: return STG_E_INVALIDFUNCTION; + } + if (offset < 0) + return HRESULT_WIN32_ERROR_NEGATIVE_SEEK; + if ((UInt64)offset != _absPos) + { + _absPos = (UInt64)offset; + _offsetPos = (UInt64)offset; + _streamIndex = 0; + } + if (newPosition) + *newPosition = (UInt64)offset; + return S_OK; +} + + +// result value will be saturated to (UInt32)(Int32)-1 + +unsigned CMultiOutStream::GetStreamIndex_for_Offset(UInt64 offset, UInt64 &relOffset) const +{ + const unsigned last = Sizes.Size() - 1; + for (unsigned i = 0; i < last; i++) + { + const UInt64 size = Sizes[i]; + if (offset < size) + { + relOffset = offset; + return i; + } + offset -= size; + } + const UInt64 size = Sizes[last]; + const UInt64 v = offset / size; + if (v >= ((UInt32)(Int32)-1) - last) + return (unsigned)(int)-1; // saturation + relOffset = offset - (unsigned)v * size; + return last + (unsigned)(v); +} + + +Z7_COM7F_IMF(CMultiOutStream::SetRestriction(UInt64 begin, UInt64 end)) +{ + COM_TRY_BEGIN + + // begin = end = 0; // for debug + + PRF(printf("\n==================== CMultiOutStream::SetRestriction %u, %u\n", (unsigned)begin, (unsigned)end)) + if (begin > end) + { + // these value are FAILED values. + return E_FAIL; + // return E_INVALIDARG; + /* + // or we can ignore error with 3 ways: no change, non-restricted, saturation: + end = begin; // non-restricted + end = (UInt64)(Int64)-1; // saturation: + return S_OK; + */ + } + UInt64 b = _restrict_Begin; + UInt64 e = _restrict_End; + _restrict_Begin = begin; + _restrict_End = end; + + if (b == e) // if there were no restriction before + return S_OK; // no work to derestrict now. + + /* [b, e) is previous restricted region. So all volumes that + intersect that [b, e) region are candidats for derestriction */ + + if (begin != end) // if there is new non-empty restricted region + { + /* Now we will try to reduce or change (b) and (e) bounds + to reduce main loop that checks volumes for derestriction. + We still use one big derestriction region in main loop, although + in some cases we could have two smaller derestriction regions. + Also usually restriction region cannot move back from previous start position, + so (b <= begin) is expected here for normal cases */ + if (b == begin) // if same low bounds + b = end; // we need to derestrict only after the end of new restricted region + if (e == end) // if same high bounds + e = begin; // we need to derestrict only before the begin of new restricted region + } + + if (b > e) // || b == (UInt64)(Int64)-1 + return S_OK; + + /* Here we close finished volumes that are not restricted anymore. + We close (low number) volumes at first. */ + + UInt64 offset; + unsigned index = GetStreamIndex_for_Offset(b, offset); + + for (; index < Streams.Size(); index++) + { + { + const CVolStream &s = Streams[index]; + if (_length <= s.Start) + break; // we don't close streams after _length + // (_length > s.Start) + const UInt64 volSize = GetVolSize_for_Stream(index); + if (volSize == 0) + { + if (e < s.Start) + break; + // we don't close empty stream, if next byte [s.Start, s.Start] is restricted + if (IsRestricted_Empty(s)) + continue; + } + else + { + if (e <= s.Start) + break; + // we don't close non full streams + if (_length - s.Start < volSize) + break; + // (volSize == s.RealSize) is expected here. So no need to check it + // if (volSize != s.RealSize) break; + if (IsRestricted(s)) + continue; + } + } + RINOK(CloseStream_and_FinalRename(index)) + } + + return S_OK; + COM_TRY_END +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.h b/NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.h new file mode 100644 index 000000000..47c9b3559 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/Common/MultiOutStream.h @@ -0,0 +1,160 @@ +// MultiOutStream.h + +#ifndef ZIP7_INC_MULTI_OUT_STREAM_H +#define ZIP7_INC_MULTI_OUT_STREAM_H + +#include "FileStreams.h" + +Z7_CLASS_IMP_COM_2( + CMultiOutStream + , IOutStream + , IStreamSetRestriction +) + Z7_IFACE_COM7_IMP(ISequentialOutStream) + + Z7_CLASS_NO_COPY(CMultiOutStream) + + struct CVolStream + { + COutFileStream *StreamSpec; + CMyComPtr Stream; + UInt64 Start; // start pos of current Stream in global stream + UInt64 Pos; // pos in current Stream + UInt64 RealSize; + int Next; // next older + int Prev; // prev newer + AString Postfix; + + HRESULT SetSize2(UInt64 size) + { + const HRESULT res = Stream->SetSize(size); + if (res == SZ_OK) + RealSize = size; + return res; + } + }; + + unsigned _streamIndex; // (_streamIndex >= Stream.Size()) is allowed in some internal code + UInt64 _offsetPos; // offset relative to Streams[_streamIndex] volume. (_offsetPos >= volSize is allowed) + UInt64 _absPos; + UInt64 _length; // virtual Length + UInt64 _absLimit; + + CObjectVector Streams; + CRecordVector Sizes; + + UInt64 _restrict_Begin; + UInt64 _restrict_End; + UInt64 _restrict_Global; + + unsigned NumOpenFiles_AllowedMax; + + // ----- Double Linked List ----- + + unsigned NumListItems; + int Head; // newest + int Tail; // oldest + + void InitLinkedList() + { + Head = -1; + Tail = -1; + NumListItems = 0; + } + + void InsertToLinkedList(unsigned index) + { + { + CVolStream &node = Streams[index]; + node.Next = Head; + node.Prev = -1; + } + if (Head != -1) + Streams[(unsigned)Head].Prev = (int)index; + else + { + // if (Tail != -1) throw 1; + Tail = (int)index; + } + Head = (int)index; + NumListItems++; + } + + void RemoveFromLinkedList(unsigned index) + { + CVolStream &s = Streams[index]; + if (s.Next != -1) Streams[(unsigned)s.Next].Prev = s.Prev; else Tail = s.Prev; + if (s.Prev != -1) Streams[(unsigned)s.Prev].Next = s.Next; else Head = s.Next; + s.Next = -1; // optional + s.Prev = -1; // optional + NumListItems--; + } + + /* + void Delete_LastStream_Records() + { + if (Streams.Back().Stream) + RemoveFromLinkedList(Streams.Size() - 1); + Streams.DeleteBack(); + } + */ + + UInt64 GetVolSize_for_Stream(unsigned i) const + { + const unsigned last = Sizes.Size() - 1; + return Sizes[i < last ? i : last]; + } + UInt64 GetGlobalOffset_for_NewStream() const + { + return Streams.Size() == 0 ? 0: + Streams.Back().Start + + GetVolSize_for_Stream(Streams.Size() - 1); + } + unsigned GetStreamIndex_for_Offset(UInt64 offset, UInt64 &relOffset) const; + bool IsRestricted(const CVolStream &s) const; + bool IsRestricted_Empty(const CVolStream &s) const + { + // (s) must be stream that has (VolSize == 0). + // we treat empty stream as restricted, if next byte is restricted. + if (s.Start < _restrict_Global) + return true; + return + (_restrict_Begin != _restrict_End) + && (_restrict_Begin <= s.Start) + && (_restrict_Begin == s.Start || _restrict_End > s.Start); + } + // bool IsRestricted_for_Close(unsigned index) const; + FString GetFilePath(unsigned index); + + HRESULT CloseStream(unsigned index); + HRESULT CloseStream_and_DeleteFile(unsigned index); + HRESULT CloseStream_and_FinalRename(unsigned index); + + HRESULT PrepareToOpenNew(); + HRESULT CreateNewStream(UInt64 newSize); + HRESULT CreateStreams_If_Required(unsigned streamIndex); + HRESULT ReOpenStream(unsigned streamIndex); + HRESULT OptReOpen_and_SetSize(unsigned index, UInt64 size); + + HRESULT Normalize_finalMode(bool finalMode); +public: + FString Prefix; + CFiTime MTime; + bool MTime_Defined; + bool FinalVol_WasReopen; + bool NeedDelete; + + CMultiOutStream() {} + ~CMultiOutStream(); + void Init(const CRecordVector &sizes); + bool SetMTime_Final(const CFiTime &mTime); + UInt64 GetSize() const { return _length; } + /* it makes final flushing, closes open files and renames to final name if required + but it still keeps Streams array of all closed files. + So we still can delete all files later, if required */ + HRESULT FinalFlush_and_CloseFiles(unsigned &numTotalVolumesRes); + // Destruct object without exceptions + HRESULT Destruct(); +}; + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp new file mode 100644 index 000000000..0c1b8d21b --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/ArchiveCommandLine.cpp @@ -0,0 +1,1769 @@ +// ArchiveCommandLine.cpp + +#include "StdAfx.h" +#undef printf +#undef sprintf + +#ifdef _WIN32 +#ifndef UNDER_CE +#include +#endif +#else +// for isatty() +#include +#endif + +#include + +#ifdef Z7_LARGE_PAGES +#include "../../../../C/Alloc.h" +#endif + +#include "../../../Common/IntToString.h" +#include "../../../Common/ListFileUtils.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/StringToInt.h" + +#include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/FileDir.h" +#include "../../../Windows/FileName.h" +#include "../../../Windows/PropVariantConv.h" +#include "../../../Windows/System.h" +#ifdef _WIN32 +#include "../../../Windows/FileMapping.h" +#include "../../../Windows/MemoryLock.h" +#include "../../../Windows/Synchronization.h" +#endif + +#include "ArchiveCommandLine.h" +#include "EnumDirItems.h" +#include "Update.h" +#include "UpdateAction.h" + +extern bool g_CaseSensitive; +extern bool g_PathTrailReplaceMode; + +#ifdef Z7_LARGE_PAGES +extern +bool g_LargePagesMode; +bool g_LargePagesMode = false; +#endif + +/* +#ifdef ENV_HAVE_LSTAT +EXTERN_C_BEGIN +extern int global_use_lstat; +EXTERN_C_END +#endif +*/ + +#ifdef UNDER_CE + +#define MY_IS_TERMINAL(x) false; + +#else + +// #define MY_isatty_fileno(x) (isatty(fileno(x))) +// #define MY_IS_TERMINAL(x) (MY_isatty_fileno(x) != 0); +static inline bool MY_IS_TERMINAL(FILE *x) +{ + return ( + #if defined(_MSC_VER) && (_MSC_VER >= 1400) + _isatty(_fileno(x)) + #else + isatty(fileno(x)) + #endif + != 0); +} + +#endif + +using namespace NCommandLineParser; +using namespace NWindows; +using namespace NFile; + +static bool StringToUInt32(const wchar_t *s, UInt32 &v) +{ + if (*s == 0) + return false; + const wchar_t *end; + v = ConvertStringToUInt32(s, &end); + return *end == 0; +} + + +namespace NKey { +enum Enum +{ + kHelp1 = 0, + kHelp2, + kHelp3, + + kDisableHeaders, + kDisablePercents, + kShowTime, + kLogLevel, + + kOutStream, + kErrStream, + kPercentStream, + + kYes, + + kShowDialog, + kOverwrite, + + kArchiveType, + kExcludedArcType, + + kProperty, + kOutputDir, + kWorkingDir, + + kInclude, + kExclude, + kArInclude, + kArExclude, + kNoArName, + + kUpdate, + kVolume, + kRecursed, + + kAffinity, + kSfx, + kEmail, + kHash, + // kHashGenFile, + kHashDir, + kExtractMemLimit, + + kStdIn, + kStdOut, + + kLargePages, + kListfileCharSet, + kConsoleCharSet, + kTechMode, + kListFields, + kListPathSlash, + kListTimestampUTC, + + kPreserveATime, + kShareForWrite, + kStopAfterOpenError, + kCaseSensitive, + kArcNameMode, + + kUseSlashMark, + kDisableWildcardParsing, + kElimDup, + kFullPathMode, + + kHardLinks, + kSymLinks_AllowDangerous, + kSymLinks, + kNtSecurity, + + kStoreOwnerId, + kStoreOwnerName, + + kZoneFile, + kAltStreams, + kReplaceColonForAltStream, + kWriteToAltStreamIfColon, + + kNameTrailReplace, + + kDeleteAfterCompressing, + kSetArcMTime + + #ifndef Z7_NO_CRYPTO + , kPassword + #endif +}; + +} + + +static const wchar_t kRecursedIDChar = 'r'; +static const char * const kRecursedPostCharSet = "0-"; + +static const char * const k_ArcNameMode_PostCharSet = "sea"; + +static const char * const k_Stream_PostCharSet = "012"; + +static inline EArcNameMode ParseArcNameMode(int postCharIndex) +{ + switch (postCharIndex) + { + case 1: return k_ArcNameMode_Exact; + case 2: return k_ArcNameMode_Add; + default: return k_ArcNameMode_Smart; + } +} + +namespace NRecursedPostCharIndex { + enum EEnum + { + kWildcardRecursionOnly = 0, + kNoRecursion = 1 + }; +} + +// static const char +#define kImmediateNameID '!' +#ifdef _WIN32 +#define kMapNameID '#' +#endif +#define kFileListID '@' + +static const Byte kSomeCludePostStringMinSize = 2; // at least <@|!>ame must be +static const Byte kSomeCludeAfterRecursedPostStringMinSize = 2; // at least <@|!>ame must be + +static const char * const kOverwritePostCharSet = "asut"; + +static const NExtract::NOverwriteMode::EEnum k_OverwriteModes[] = +{ + NExtract::NOverwriteMode::kOverwrite, + NExtract::NOverwriteMode::kSkip, + NExtract::NOverwriteMode::kRename, + NExtract::NOverwriteMode::kRenameExisting +}; + + + +#define SWFRM_3(t, mu, mi) t, mu, mi, NULL + +#define SWFRM_1(t) SWFRM_3(t, false, 0) +#define SWFRM_SIMPLE SWFRM_1(NSwitchType::kSimple) +#define SWFRM_MINUS SWFRM_1(NSwitchType::kMinus) +#define SWFRM_STRING SWFRM_1(NSwitchType::kString) + +#define SWFRM_STRING_SINGL(mi) SWFRM_3(NSwitchType::kString, false, mi) +#define SWFRM_STRING_MULT(mi) SWFRM_3(NSwitchType::kString, true, mi) + + +static const CSwitchForm kSwitchForms[] = +{ + { "?", SWFRM_SIMPLE }, + { "h", SWFRM_SIMPLE }, + { "-help", SWFRM_SIMPLE }, + + { "ba", SWFRM_SIMPLE }, + { "bd", SWFRM_SIMPLE }, + { "bt", SWFRM_SIMPLE }, + { "bb", SWFRM_STRING_SINGL(0) }, + + { "bso", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, + { "bse", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, + { "bsp", NSwitchType::kChar, false, 1, k_Stream_PostCharSet }, + + { "y", SWFRM_SIMPLE }, + + { "ad", SWFRM_SIMPLE }, + { "ao", NSwitchType::kChar, false, 1, kOverwritePostCharSet}, + + { "t", SWFRM_STRING_SINGL(1) }, + { "stx", SWFRM_STRING_MULT(1) }, + + { "m", SWFRM_STRING_MULT(1) }, + { "o", SWFRM_STRING_SINGL(1) }, + { "w", SWFRM_STRING }, + + { "i", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "x", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "ai", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "ax", SWFRM_STRING_MULT(kSomeCludePostStringMinSize) }, + { "an", SWFRM_SIMPLE }, + + { "u", SWFRM_STRING_MULT(1) }, + { "v", SWFRM_STRING_MULT(1) }, + { "r", NSwitchType::kChar, false, 0, kRecursedPostCharSet }, + + { "stm", SWFRM_STRING }, + { "sfx", SWFRM_STRING }, + { "seml", SWFRM_STRING_SINGL(0) }, + { "scrc", SWFRM_STRING_MULT(0) }, + // { "scrf", SWFRM_STRING_SINGL(1) }, + { "shd", SWFRM_STRING_SINGL(1) }, + { "smemx", SWFRM_STRING }, + + { "si", SWFRM_STRING }, + { "so", SWFRM_SIMPLE }, + + { "slp", SWFRM_STRING }, + { "scs", SWFRM_STRING }, + { "scc", SWFRM_STRING }, + { "slt", SWFRM_SIMPLE }, + { "slf", SWFRM_STRING_SINGL(1) }, + { "slsl", SWFRM_MINUS }, + { "slmu", SWFRM_MINUS }, + + { "ssp", SWFRM_SIMPLE }, + { "ssw", SWFRM_SIMPLE }, + { "sse", SWFRM_SIMPLE }, + { "ssc", SWFRM_MINUS }, + { "sa", NSwitchType::kChar, false, 1, k_ArcNameMode_PostCharSet }, + + { "spm", SWFRM_STRING_SINGL(0) }, + { "spd", SWFRM_SIMPLE }, + { "spe", SWFRM_MINUS }, + { "spf", SWFRM_STRING_SINGL(0) }, + + { "snh", SWFRM_MINUS }, + { "snld", SWFRM_MINUS }, + { "snl", SWFRM_MINUS }, + { "sni", SWFRM_SIMPLE }, + + { "snoi", SWFRM_MINUS }, + { "snon", SWFRM_MINUS }, + + { "snz", SWFRM_STRING_SINGL(0) }, + { "sns", SWFRM_MINUS }, + { "snr", SWFRM_SIMPLE }, + { "snc", SWFRM_SIMPLE }, + + { "snt", SWFRM_MINUS }, + + { "sdel", SWFRM_SIMPLE }, + { "stl", SWFRM_SIMPLE } + + #ifndef Z7_NO_CRYPTO + , { "p", SWFRM_STRING } + #endif +}; + +static const char * const kUniversalWildcard = "*"; +static const unsigned kMinNonSwitchWords = 1; +static const unsigned kCommandIndex = 0; + +// static const char * const kUserErrorMessage = "Incorrect command line"; +// static const char * const kCannotFindListFile = "Cannot find listfile"; +static const char * const kIncorrectListFile = "Incorrect item in listfile.\nCheck charset encoding and -scs switch."; +static const char * const kTerminalOutError = "I won't write compressed data to a terminal"; +static const char * const kSameTerminalError = "I won't write data and program's messages to same stream"; +static const char * const kEmptyFilePath = "Empty file path"; + +bool CArcCommand::IsFromExtractGroup() const +{ + switch ((int)CommandType) + { + case NCommandType::kTest: + case NCommandType::kExtract: + case NCommandType::kExtractFull: + return true; + default: + return false; + } +} + +NExtract::NPathMode::EEnum CArcCommand::GetPathMode() const +{ + switch ((int)CommandType) + { + case NCommandType::kTest: + case NCommandType::kExtractFull: + return NExtract::NPathMode::kFullPaths; + default: + return NExtract::NPathMode::kNoPaths; + } +} + +bool CArcCommand::IsFromUpdateGroup() const +{ + switch ((int)CommandType) + { + case NCommandType::kAdd: + case NCommandType::kUpdate: + case NCommandType::kDelete: + case NCommandType::kRename: + return true; + default: + return false; + } +} + +static NRecursedType::EEnum GetRecursedTypeFromIndex(int index) +{ + switch (index) + { + case NRecursedPostCharIndex::kWildcardRecursionOnly: + return NRecursedType::kWildcardOnlyRecursed; + case NRecursedPostCharIndex::kNoRecursion: + return NRecursedType::kNonRecursed; + default: + return NRecursedType::kRecursed; + } +} + +static const char *g_Commands = "audtexlbih"; + +static bool ParseArchiveCommand(const UString &commandString, CArcCommand &command) +{ + UString s (commandString); + s.MakeLower_Ascii(); + if (s.Len() == 1) + { + if (s[0] > 0x7F) + return false; + int index = FindCharPosInString(g_Commands, (char)s[0]); + if (index < 0) + return false; + command.CommandType = (NCommandType::EEnum)index; + return true; + } + if (s.Len() == 2 && s[0] == 'r' && s[1] == 'n') + { + command.CommandType = (NCommandType::kRename); + return true; + } + return false; +} + +// ------------------------------------------------------------------ +// filenames functions + +struct CNameOption +{ + bool Include; + bool WildcardMatching; + Byte MarkMode; + NRecursedType::EEnum RecursedType; + + CNameOption(): + Include(true), + WildcardMatching(true), + MarkMode(NWildcard::kMark_FileOrDir), + RecursedType(NRecursedType::kNonRecursed) + {} +}; + + +static void AddNameToCensor(NWildcard::CCensor &censor, + const CNameOption &nop, const UString &name) +{ + bool recursed = false; + + switch ((int)nop.RecursedType) + { + case NRecursedType::kWildcardOnlyRecursed: + recursed = DoesNameContainWildcard(name); + break; + case NRecursedType::kRecursed: + recursed = true; + break; + default: + break; + } + + NWildcard::CCensorPathProps props; + props.Recursive = recursed; + props.WildcardMatching = nop.WildcardMatching; + props.MarkMode = nop.MarkMode; + censor.AddPreItem(nop.Include, name, props); +} + +#ifndef Z7_EXTRACT_ONLY +static void AddRenamePair(CObjectVector *renamePairs, + const UString &oldName, const UString &newName, NRecursedType::EEnum type, + bool wildcardMatching) +{ + CRenamePair &pair = renamePairs->AddNew(); + pair.OldName = oldName; + pair.NewName = newName; + pair.RecursedType = type; + pair.WildcardParsing = wildcardMatching; + + if (!pair.Prepare()) + { + UString val; + val += pair.OldName; + val.Add_LF(); + val += pair.NewName; + val.Add_LF(); + if (type == NRecursedType::kRecursed) + val += "-r"; + else if (type == NRecursedType::kWildcardOnlyRecursed) + val += "-r0"; + throw CArcCmdLineException("Unsupported rename command:", val); + } +} +#endif + +static void AddToCensorFromListFile( + CObjectVector *renamePairs, + NWildcard::CCensor &censor, + const CNameOption &nop, LPCWSTR fileName, UInt32 codePage) +{ + UStringVector names; + /* + if (!NFind::DoesFileExist_FollowLink(us2fs(fileName))) + throw CArcCmdLineException(kCannotFindListFile, fileName); + */ + DWORD lastError = 0; + if (!ReadNamesFromListFile2(us2fs(fileName), names, codePage, lastError)) + { + if (lastError != 0) + { + UString m; + m = "The file operation error for listfile"; + m.Add_LF(); + m += NError::MyFormatMessage(lastError); + throw CArcCmdLineException(m, fileName); + } + throw CArcCmdLineException(kIncorrectListFile, fileName); + } + if (renamePairs) + { + #ifndef Z7_EXTRACT_ONLY + if ((names.Size() & 1) != 0) + throw CArcCmdLineException(kIncorrectListFile, fileName); + for (unsigned i = 0; i < names.Size(); i += 2) + { + // change type !!!! + AddRenamePair(renamePairs, names[i], names[i + 1], nop.RecursedType, nop.WildcardMatching); + } + #else + throw "not implemented"; + #endif + } + else + FOR_VECTOR (i, names) + AddNameToCensor(censor, nop, names[i]); +} + +static void AddToCensorFromNonSwitchesStrings( + CObjectVector *renamePairs, + unsigned startIndex, + NWildcard::CCensor &censor, + const UStringVector &nonSwitchStrings, + int stopSwitchIndex, + const CNameOption &nop, + bool thereAreSwitchIncludes, UInt32 codePage) +{ + // another default + if ((renamePairs || nonSwitchStrings.Size() == startIndex) && !thereAreSwitchIncludes) + { + /* for rename command: -i switch sets the mask for archive item reading. + if (thereAreSwitchIncludes), { we don't use UniversalWildcard. } + also for non-rename command: we set UniversalWildcard, only if there are no nonSwitches. */ + // we use default fileds in (CNameOption) for UniversalWildcard. + CNameOption nop2; + // recursive mode is not important for UniversalWildcard (*) + // nop2.RecursedType = nop.RecursedType; // we don't need it + /* + nop2.RecursedType = NRecursedType::kNonRecursed; + nop2.Include = true; + nop2.WildcardMatching = true; + nop2.MarkMode = NWildcard::kMark_FileOrDir; + */ + AddNameToCensor(censor, nop2, UString(kUniversalWildcard)); + } + + int oldIndex = -1; + + if (stopSwitchIndex < 0) + stopSwitchIndex = (int)nonSwitchStrings.Size(); + + for (unsigned i = startIndex; i < nonSwitchStrings.Size(); i++) + { + const UString &s = nonSwitchStrings[i]; + if (s.IsEmpty()) + throw CArcCmdLineException(kEmptyFilePath); + if (i < (unsigned)stopSwitchIndex && s[0] == kFileListID) + AddToCensorFromListFile(renamePairs, censor, nop, s.Ptr(1), codePage); + else if (renamePairs) + { + #ifndef Z7_EXTRACT_ONLY + if (oldIndex == -1) + oldIndex = (int)i; + else + { + // NRecursedType::EEnum type is used for global wildcard (-i! switches) + AddRenamePair(renamePairs, nonSwitchStrings[(unsigned)oldIndex], s, NRecursedType::kNonRecursed, nop.WildcardMatching); + // AddRenamePair(renamePairs, nonSwitchStrings[oldIndex], s, type); + oldIndex = -1; + } + #else + throw "not implemented"; + #endif + } + else + AddNameToCensor(censor, nop, s); + } + + if (oldIndex != -1) + { + throw CArcCmdLineException("There is no second file name for rename pair:", nonSwitchStrings[(unsigned)oldIndex]); + } +} + +#ifdef _WIN32 + +struct CEventSetEnd +{ + UString Name; + + CEventSetEnd(const wchar_t *name): Name(name) {} + ~CEventSetEnd() + { + NSynchronization::CManualResetEvent event; + if (event.Open(EVENT_MODIFY_STATE, false, GetSystemString(Name)) == 0) + event.Set(); + } +}; + +static const char * const k_IncorrectMapCommand = "Incorrect Map command"; + +static const char *ParseMapWithPaths( + NWildcard::CCensor &censor, + const UString &s2, + const CNameOption &nop) +{ + UString s (s2); + const int pos = s.Find(L':'); + if (pos < 0) + return k_IncorrectMapCommand; + const int pos2 = s.Find(L':', (unsigned)(pos + 1)); + if (pos2 < 0) + return k_IncorrectMapCommand; + + CEventSetEnd eventSetEnd((const wchar_t *)s + (unsigned)(pos2 + 1)); + s.DeleteFrom((unsigned)pos2); + UInt32 size; + if (!StringToUInt32(s.Ptr((unsigned)(pos + 1)), size) + || size < sizeof(wchar_t) + || size > ((UInt32)1 << 31) + || size % sizeof(wchar_t) != 0) + return "Unsupported Map data size"; + + s.DeleteFrom((unsigned)pos); + CFileMapping map; + if (map.Open(FILE_MAP_READ, GetSystemString(s)) != 0) + return "Cannot open mapping"; + const LPVOID data = map.Map(FILE_MAP_READ, 0, size); + if (!data) + return "MapViewOfFile error"; + CFileUnmapper unmapper(data); + + UString name; + const wchar_t *p = (const wchar_t *)data; + if (*p != 0) // data format marker + return "Unsupported Map data"; + const UInt32 numChars = size / sizeof(wchar_t); + for (UInt32 i = 1; i < numChars; i++) + { + const wchar_t c = p[i]; + if (c == 0) + { + // MessageBoxW(0, name, L"7-Zip", 0); + AddNameToCensor(censor, nop, name); + name.Empty(); + } + else + name += c; + } + if (!name.IsEmpty()) + return "Map data error"; + + return NULL; +} + +#endif + +static void AddSwitchWildcardsToCensor( + NWildcard::CCensor &censor, + const UStringVector &strings, + const CNameOption &nop, + UInt32 codePage) +{ + const char *errorMessage = NULL; + unsigned i; + for (i = 0; i < strings.Size(); i++) + { + const UString &name = strings[i]; + unsigned pos = 0; + + if (name.Len() < kSomeCludePostStringMinSize) + { + errorMessage = "Too short switch"; + break; + } + + if (!nop.Include) + { + if (name.IsEqualTo_Ascii_NoCase("td")) + { + censor.ExcludeDirItems = true; + continue; + } + if (name.IsEqualTo_Ascii_NoCase("tf")) + { + censor.ExcludeFileItems = true; + continue; + } + } + + CNameOption nop2 = nop; + + bool type_WasUsed = false; + bool recursed_WasUsed = false; + bool matching_WasUsed = false; + bool error = false; + + for (;;) + { + wchar_t c = ::MyCharLower_Ascii(name[pos]); + if (c == kRecursedIDChar) + { + if (recursed_WasUsed) + { + error = true; + break; + } + recursed_WasUsed = true; + pos++; + c = name[pos]; + int index = -1; + if (c <= 0x7F) + index = FindCharPosInString(kRecursedPostCharSet, (char)c); + nop2.RecursedType = GetRecursedTypeFromIndex(index); + if (index >= 0) + { + pos++; + continue; + } + } + + if (c == 'w') + { + if (matching_WasUsed) + { + error = true; + break; + } + matching_WasUsed = true; + nop2.WildcardMatching = true; + pos++; + if (name[pos] == '-') + { + nop2.WildcardMatching = false; + pos++; + } + } + else if (c == 'm') + { + if (type_WasUsed) + { + error = true; + break; + } + type_WasUsed = true; + pos++; + nop2.MarkMode = NWildcard::kMark_StrictFile; + c = name[pos]; + if (c == '-') + { + nop2.MarkMode = NWildcard::kMark_FileOrDir; + pos++; + } + else if (c == '2') + { + nop2.MarkMode = NWildcard::kMark_StrictFile_IfWildcard; + pos++; + } + } + else + break; + } + + if (error) + { + errorMessage = "inorrect switch"; + break; + } + + if (name.Len() < pos + kSomeCludeAfterRecursedPostStringMinSize) + { + errorMessage = "Too short switch"; + break; + } + + const UString tail = name.Ptr(pos + 1); + + const wchar_t c = name[pos]; + + if (c == kImmediateNameID) + AddNameToCensor(censor, nop2, tail); + else if (c == kFileListID) + AddToCensorFromListFile(NULL, censor, nop2, tail, codePage); + #ifdef _WIN32 + else if (c == kMapNameID) + { + errorMessage = ParseMapWithPaths(censor, tail, nop2); + if (errorMessage) + break; + } + #endif + else + { + errorMessage = "Incorrect wildcard type marker"; + break; + } + } + + if (i != strings.Size()) + throw CArcCmdLineException(errorMessage, strings[i]); +} + +/* +static NUpdateArchive::NPairAction::EEnum GetUpdatePairActionType(int i) +{ + switch (i) + { + case NUpdateArchive::NPairAction::kIgnore: return NUpdateArchive::NPairAction::kIgnore; + case NUpdateArchive::NPairAction::kCopy: return NUpdateArchive::NPairAction::kCopy; + case NUpdateArchive::NPairAction::kCompress: return NUpdateArchive::NPairAction::kCompress; + case NUpdateArchive::NPairAction::kCompressAsAnti: return NUpdateArchive::NPairAction::kCompressAsAnti; + } + throw 98111603; +} +*/ + +static const char * const kUpdatePairStateIDSet = "pqrxyzw"; +static const int kUpdatePairStateNotSupportedActions[] = {2, 2, 1, -1, -1, -1, -1}; + +static const unsigned kNumUpdatePairActions = 4; +static const char * const kUpdateIgnoreItselfPostStringID = "-"; +static const wchar_t kUpdateNewArchivePostCharID = '!'; + + +static bool ParseUpdateCommandString2(const UString &command, + NUpdateArchive::CActionSet &actionSet, UString &postString) +{ + for (unsigned i = 0; i < command.Len();) + { + wchar_t c = MyCharLower_Ascii(command[i]); + int statePos = FindCharPosInString(kUpdatePairStateIDSet, (char)c); + if (c > 0x7F || statePos < 0) + { + postString = command.Ptr(i); + return true; + } + i++; + if (i >= command.Len()) + return false; + c = command[i]; + if (c < '0' || c >= (wchar_t)('0' + kNumUpdatePairActions)) + return false; + unsigned actionPos = (unsigned)(c - '0'); + actionSet.StateActions[(unsigned)statePos] = (NUpdateArchive::NPairAction::EEnum)(actionPos); + if (kUpdatePairStateNotSupportedActions[(unsigned)statePos] == (int)actionPos) + return false; + i++; + } + postString.Empty(); + return true; +} + +static void ParseUpdateCommandString(CUpdateOptions &options, + const UStringVector &updatePostStrings, + const NUpdateArchive::CActionSet &defaultActionSet) +{ + const char *errorMessage = "incorrect update switch command"; + unsigned i; + for (i = 0; i < updatePostStrings.Size(); i++) + { + const UString &updateString = updatePostStrings[i]; + if (updateString.IsEqualTo(kUpdateIgnoreItselfPostStringID)) + { + if (options.UpdateArchiveItself) + { + options.UpdateArchiveItself = false; + options.Commands.Delete(0); + } + } + else + { + NUpdateArchive::CActionSet actionSet = defaultActionSet; + + UString postString; + if (!ParseUpdateCommandString2(updateString, actionSet, postString)) + break; + if (postString.IsEmpty()) + { + if (options.UpdateArchiveItself) + options.Commands[0].ActionSet = actionSet; + } + else + { + if (postString[0] != kUpdateNewArchivePostCharID) + break; + CUpdateArchiveCommand uc; + UString archivePath = postString.Ptr(1); + if (archivePath.IsEmpty()) + break; + uc.UserArchivePath = archivePath; + uc.ActionSet = actionSet; + options.Commands.Add(uc); + } + } + } + if (i != updatePostStrings.Size()) + throw CArcCmdLineException(errorMessage, updatePostStrings[i]); +} + +bool ParseComplexSize(const wchar_t *s, UInt64 &result); + +static void SetAddCommandOptions( + NCommandType::EEnum commandType, + const CParser &parser, + CUpdateOptions &options) +{ + NUpdateArchive::CActionSet defaultActionSet; + switch ((int)commandType) + { + case NCommandType::kAdd: + defaultActionSet = NUpdateArchive::k_ActionSet_Add; + break; + case NCommandType::kDelete: + defaultActionSet = NUpdateArchive::k_ActionSet_Delete; + break; + default: + defaultActionSet = NUpdateArchive::k_ActionSet_Update; + } + + options.UpdateArchiveItself = true; + + options.Commands.Clear(); + CUpdateArchiveCommand updateMainCommand; + updateMainCommand.ActionSet = defaultActionSet; + options.Commands.Add(updateMainCommand); + if (parser[NKey::kUpdate].ThereIs) + ParseUpdateCommandString(options, parser[NKey::kUpdate].PostStrings, + defaultActionSet); + if (parser[NKey::kWorkingDir].ThereIs) + { + const UString &postString = parser[NKey::kWorkingDir].PostStrings[0]; + if (postString.IsEmpty()) + NDir::MyGetTempPath(options.WorkingDir); + else + options.WorkingDir = us2fs(postString); + } + options.SfxMode = parser[NKey::kSfx].ThereIs; + if (options.SfxMode) + options.SfxModule = us2fs(parser[NKey::kSfx].PostStrings[0]); + + if (parser[NKey::kVolume].ThereIs) + { + const UStringVector &sv = parser[NKey::kVolume].PostStrings; + FOR_VECTOR (i, sv) + { + UInt64 size; + if (!ParseComplexSize(sv[i], size)) + throw CArcCmdLineException("Incorrect volume size:", sv[i]); + if (i == sv.Size() - 1 && size == 0) + throw CArcCmdLineException("zero size last volume is not allowed"); + options.VolumesSizes.Add(size); + } + } +} + +static void SetMethodOptions(const CParser &parser, CObjectVector &properties) +{ + if (parser[NKey::kProperty].ThereIs) + { + FOR_VECTOR (i, parser[NKey::kProperty].PostStrings) + { + CProperty prop; + prop.Name = parser[NKey::kProperty].PostStrings[i]; + int index = prop.Name.Find(L'='); + if (index >= 0) + { + prop.Value = prop.Name.Ptr((unsigned)(index + 1)); + prop.Name.DeleteFrom((unsigned)index); + } + properties.Add(prop); + } + } +} + + +static inline void SetStreamMode(const CSwitchResult &sw, unsigned &res) +{ + if (sw.ThereIs) + res = (unsigned)sw.PostCharIndex; +} + + +#if defined(_WIN32) && !defined(UNDER_CE) +static void PrintHex(UString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} +#endif + + +void CArcCmdLineParser::Parse1(const UStringVector &commandStrings, + CArcCmdLineOptions &options) +{ + Parse1Log.Empty(); + if (!parser.ParseStrings(kSwitchForms, Z7_ARRAY_SIZE(kSwitchForms), commandStrings)) + throw CArcCmdLineException(parser.ErrorMessage, parser.ErrorLine); + + options.IsInTerminal = MY_IS_TERMINAL(stdin); + options.IsStdOutTerminal = MY_IS_TERMINAL(stdout); + options.IsStdErrTerminal = MY_IS_TERMINAL(stderr); + + options.HelpMode = parser[NKey::kHelp1].ThereIs || parser[NKey::kHelp2].ThereIs || parser[NKey::kHelp3].ThereIs; + options.YesToAll = parser[NKey::kYes].ThereIs; + + options.StdInMode = parser[NKey::kStdIn].ThereIs; + options.StdOutMode = parser[NKey::kStdOut].ThereIs; + options.EnableHeaders = !parser[NKey::kDisableHeaders].ThereIs; + if (parser[NKey::kListFields].ThereIs) + { + const UString &s = parser[NKey::kListFields].PostStrings[0]; + options.ListFields = GetAnsiString(s); + } + if (parser[NKey::kListPathSlash].ThereIs) + { + options.ListPathSeparatorSlash.Val = !parser[NKey::kListPathSlash].WithMinus; + options.ListPathSeparatorSlash.Def = true; + } + if (parser[NKey::kListTimestampUTC].ThereIs) + g_Timestamp_Show_UTC = !parser[NKey::kListTimestampUTC].WithMinus; + options.TechMode = parser[NKey::kTechMode].ThereIs; + options.ShowTime = parser[NKey::kShowTime].ThereIs; + + if (parser[NKey::kDisablePercents].ThereIs) + options.DisablePercents = true; + + if (parser[NKey::kDisablePercents].ThereIs + || options.StdOutMode + || !options.IsStdOutTerminal) + options.Number_for_Percents = k_OutStream_disabled; + + if (options.StdOutMode) + options.Number_for_Out = k_OutStream_disabled; + + SetStreamMode(parser[NKey::kOutStream], options.Number_for_Out); + SetStreamMode(parser[NKey::kErrStream], options.Number_for_Errors); + SetStreamMode(parser[NKey::kPercentStream], options.Number_for_Percents); + + if (parser[NKey::kLogLevel].ThereIs) + { + const UString &s = parser[NKey::kLogLevel].PostStrings[0]; + if (s.IsEmpty()) + options.LogLevel = 1; + else + { + UInt32 v; + if (!StringToUInt32(s, v)) + throw CArcCmdLineException("Unsupported switch postfix -bb", s); + options.LogLevel = (unsigned)v; + } + } + + if (parser[NKey::kCaseSensitive].ThereIs) + { + options.CaseSensitive = + g_CaseSensitive = !parser[NKey::kCaseSensitive].WithMinus; + options.CaseSensitive_Change = true; + } + + + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_SymLink(); + #endif + + // options.LargePages = false; + + if (parser[NKey::kLargePages].ThereIs) + { + UInt32 slp = 0; + const UString &s = parser[NKey::kLargePages].PostStrings[0]; + if (s.IsEmpty()) + slp = 1; + else if (s != L"-") + { + if (!StringToUInt32(s, slp)) + throw CArcCmdLineException("Unsupported switch postfix for -slp", s); + } + + #ifdef Z7_LARGE_PAGES + if (slp > + #if defined(_WIN32) && !defined(UNDER_CE) + (unsigned)NSecurity::Get_LargePages_RiskLevel() + #else + 0 + #endif + ) + { + #ifdef _WIN32 // change it ! + SetLargePageSize(); + #endif + // note: this process also can inherit that Privilege from parent process + g_LargePagesMode = + #if defined(_WIN32) && !defined(UNDER_CE) + NSecurity::EnablePrivilege_LockMemory(); + #else + true; + #endif + } + #endif + } + + +#ifndef UNDER_CE + + if (parser[NKey::kAffinity].ThereIs) + { + const UString &s = parser[NKey::kAffinity].PostStrings[0]; + if (!s.IsEmpty()) + { + AString a; + a.SetFromWStr_if_Ascii(s); + Parse1Log += "Set process affinity mask: "; + + bool isError = false; + +#ifdef _WIN32 + + UInt64 v = 0; + { + const char *end; + v = ConvertHexStringToUInt64(a, &end); + if (*end != 0) + a.Empty(); + } + if (a.IsEmpty()) + isError = true; + else + { +#ifndef _WIN64 + if (v >= ((UInt64)1 << 32)) + throw CArcCmdLineException("unsupported value -stm", s); + else +#endif + { + PrintHex(Parse1Log, v); + if (!SetProcessAffinityMask(GetCurrentProcess(), (DWORD_PTR)v)) + { + const DWORD lastError = GetLastError(); + Parse1Log += " : ERROR : "; + Parse1Log += NError::MyFormatMessage(lastError); + } + } + } + +#else // _WIN32 + + if (a.Len() != s.Len()) + isError = true; + else + { + Parse1Log += a; + NSystem::CProcessAffinity aff; + aff.CpuZero(); + unsigned cpu = 0; + unsigned i = a.Len(); + while (i) + { + unsigned v = (Byte)a[--i]; + Z7_PARSE_HEX_DIGIT(v, { isError = true; break; }) + for (unsigned mask = 1; mask != 1u << 4; mask <<= 1, cpu++) + if (v & mask) + aff.CpuSet(cpu); + } + if (!isError) + if (!aff.SetProcAffinity()) + { + const DWORD lastError = GetLastError(); + Parse1Log += " : ERROR : "; + Parse1Log += NError::MyFormatMessage(lastError); + } + } +#endif // _WIN32 + + if (isError) + throw CArcCmdLineException("Unsupported switch postfix -stm", s); + + Parse1Log.Add_LF(); + } + } + +#endif +} + + + +struct CCodePagePair +{ + const char *Name; + UInt32 CodePage; +}; + +static const unsigned kNumByteOnlyCodePages = 3; + +static const CCodePagePair g_CodePagePairs[] = +{ + { "utf-8", CP_UTF8 }, + { "win", CP_ACP }, + { "dos", CP_OEMCP }, + { "utf-16le", Z7_WIN_CP_UTF16 }, + { "utf-16be", Z7_WIN_CP_UTF16BE } +}; + +static Int32 FindCharset(const NCommandLineParser::CParser &parser, unsigned keyIndex, + bool byteOnlyCodePages, Int32 defaultVal) +{ + if (!parser[keyIndex].ThereIs) + return defaultVal; + + UString name (parser[keyIndex].PostStrings.Back()); + UInt32 v; + if (StringToUInt32(name, v)) + if (v < ((UInt32)1 << 16)) + return (Int32)v; + name.MakeLower_Ascii(); + const unsigned num = byteOnlyCodePages ? kNumByteOnlyCodePages : Z7_ARRAY_SIZE(g_CodePagePairs); + for (unsigned i = 0;; i++) + { + if (i == num) // to disable warnings from different compilers + throw CArcCmdLineException("Unsupported charset:", name); + const CCodePagePair &pair = g_CodePagePairs[i]; + if (name.IsEqualTo(pair.Name)) + return (Int32)pair.CodePage; + } +} + + +static void SetBoolPair(NCommandLineParser::CParser &parser, unsigned switchID, CBoolPair &bp) +{ + bp.Def = parser[switchID].ThereIs; + if (bp.Def) + bp.Val = !parser[switchID].WithMinus; +} + + +static bool ParseSizeString(const wchar_t *s, UInt64 &res) +{ + const wchar_t *end; + const UInt64 v = ConvertStringToUInt64(s, &end); + if (s == end) + return false; + const wchar_t c = *end; + + if (c == 0) + { + res = v; + return true; + } + if (end[1] != 0) + return false; + + unsigned numBits; + switch (MyCharLower_Ascii(c)) + { + case 'b': numBits = 0; break; + case 'k': numBits = 10; break; + case 'm': numBits = 20; break; + case 'g': numBits = 30; break; + case 't': numBits = 40; break; + default: return false; + } + const UInt64 val2 = v << numBits; + if ((val2 >> numBits) != v) + return false; + res = val2; + return true; +} + +void CArcCmdLineParser::Parse2(CArcCmdLineOptions &options) +{ + const UStringVector &nonSwitchStrings = parser.NonSwitchStrings; + const unsigned numNonSwitchStrings = nonSwitchStrings.Size(); + if (numNonSwitchStrings < kMinNonSwitchWords) + throw CArcCmdLineException("The command must be specified"); + + if (!ParseArchiveCommand(nonSwitchStrings[kCommandIndex], options.Command)) + throw CArcCmdLineException("Unsupported command:", nonSwitchStrings[kCommandIndex]); + + if (parser[NKey::kHash].ThereIs) + options.HashMethods = parser[NKey::kHash].PostStrings; + + /* + if (parser[NKey::kHashGenFile].ThereIs) + { + const UString &s = parser[NKey::kHashGenFile].PostStrings[0]; + for (unsigned i = 0 ; i < s.Len();) + { + const wchar_t c = s[i++]; + if (!options.HashOptions.ParseFlagCharOption(c, true)) + { + if (c != '=') + throw CArcCmdLineException("Unsupported hash mode switch:", s); + options.HashOptions.HashFilePath = s.Ptr(i); + break; + } + } + } + */ + + if (parser[NKey::kHashDir].ThereIs) + options.ExtractOptions.HashDir = parser[NKey::kHashDir].PostStrings[0]; + + if (parser[NKey::kExtractMemLimit].ThereIs) + { + const UString &s = parser[NKey::kExtractMemLimit].PostStrings[0]; + if (!ParseSizeString(s, options.ExtractOptions.NtOptions.MemLimit)) + throw CArcCmdLineException("Unsupported -smemx:", s); + } + + if (parser[NKey::kElimDup].ThereIs) + { + options.ExtractOptions.ElimDup.Def = true; + options.ExtractOptions.ElimDup.Val = !parser[NKey::kElimDup].WithMinus; + } + + NWildcard::ECensorPathMode censorPathMode = NWildcard::k_RelatPath; + bool fullPathMode = parser[NKey::kFullPathMode].ThereIs; + if (fullPathMode) + { + censorPathMode = NWildcard::k_AbsPath; + const UString &s = parser[NKey::kFullPathMode].PostStrings[0]; + if (!s.IsEmpty()) + { + if (s == L"2") + censorPathMode = NWildcard::k_FullPath; + else + throw CArcCmdLineException("Unsupported -spf:", s); + } + } + + if (parser[NKey::kNameTrailReplace].ThereIs) + g_PathTrailReplaceMode = !parser[NKey::kNameTrailReplace].WithMinus; + + CNameOption nop; + + if (parser[NKey::kRecursed].ThereIs) + nop.RecursedType = GetRecursedTypeFromIndex(parser[NKey::kRecursed].PostCharIndex); + + if (parser[NKey::kDisableWildcardParsing].ThereIs) + nop.WildcardMatching = false; + + if (parser[NKey::kUseSlashMark].ThereIs) + { + const UString &s = parser[NKey::kUseSlashMark].PostStrings[0]; + if (s.IsEmpty()) + nop.MarkMode = NWildcard::kMark_StrictFile; + else if (s.IsEqualTo_Ascii_NoCase("-")) + nop.MarkMode = NWildcard::kMark_FileOrDir; + else if (s.IsEqualTo_Ascii_NoCase("2")) + nop.MarkMode = NWildcard::kMark_StrictFile_IfWildcard; + else + throw CArcCmdLineException("Unsupported -spm:", s); + } + + + options.ConsoleCodePage = FindCharset(parser, NKey::kConsoleCharSet, true, -1); + + const UInt32 codePage = (UInt32)FindCharset(parser, NKey::kListfileCharSet, false, CP_UTF8); + + bool thereAreSwitchIncludes = false; + + if (parser[NKey::kInclude].ThereIs) + { + thereAreSwitchIncludes = true; + nop.Include = true; + AddSwitchWildcardsToCensor(options.Censor, + parser[NKey::kInclude].PostStrings, nop, codePage); + } + + if (parser[NKey::kExclude].ThereIs) + { + nop.Include = false; + AddSwitchWildcardsToCensor(options.Censor, + parser[NKey::kExclude].PostStrings, nop, codePage); + } + + unsigned curCommandIndex = kCommandIndex + 1; + bool thereIsArchiveName = !parser[NKey::kNoArName].ThereIs && + options.Command.CommandType != NCommandType::kBenchmark && + options.Command.CommandType != NCommandType::kInfo && + options.Command.CommandType != NCommandType::kHash; + + const bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); + const bool isExtractOrList = isExtractGroupCommand || options.Command.CommandType == NCommandType::kList; + const bool isRename = options.Command.CommandType == NCommandType::kRename; + + if ((isExtractOrList || isRename) && options.StdInMode) + thereIsArchiveName = false; + + if (parser[NKey::kArcNameMode].ThereIs) + options.UpdateOptions.ArcNameMode = ParseArcNameMode(parser[NKey::kArcNameMode].PostCharIndex); + + if (thereIsArchiveName) + { + if (curCommandIndex >= numNonSwitchStrings) + throw CArcCmdLineException("Cannot find archive name"); + options.ArchiveName = nonSwitchStrings[curCommandIndex++]; + if (options.ArchiveName.IsEmpty()) + throw CArcCmdLineException("Archive name cannot by empty"); + #ifdef _WIN32 + // options.ArchiveName.Replace(L'/', WCHAR_PATH_SEPARATOR); + #endif + } + + nop.Include = true; + AddToCensorFromNonSwitchesStrings(isRename ? &options.UpdateOptions.RenamePairs : NULL, + curCommandIndex, options.Censor, + nonSwitchStrings, parser.StopSwitchIndex, + nop, + thereAreSwitchIncludes, codePage); + + #ifndef Z7_NO_CRYPTO + options.PasswordEnabled = parser[NKey::kPassword].ThereIs; + if (options.PasswordEnabled) + options.Password = parser[NKey::kPassword].PostStrings[0]; + #endif + + options.ShowDialog = parser[NKey::kShowDialog].ThereIs; + + if (parser[NKey::kArchiveType].ThereIs) + options.ArcType = parser[NKey::kArchiveType].PostStrings[0]; + + options.ExcludedArcTypes = parser[NKey::kExcludedArcType].PostStrings; + + SetMethodOptions(parser, options.Properties); + + if (parser[NKey::kNtSecurity].ThereIs) options.NtSecurity.SetTrueTrue(); + + SetBoolPair(parser, NKey::kAltStreams, options.AltStreams); + SetBoolPair(parser, NKey::kHardLinks, options.HardLinks); + SetBoolPair(parser, NKey::kSymLinks, options.SymLinks); + + SetBoolPair(parser, NKey::kStoreOwnerId, options.StoreOwnerId); + SetBoolPair(parser, NKey::kStoreOwnerName, options.StoreOwnerName); + + CBoolPair symLinks_AllowDangerous; + SetBoolPair(parser, NKey::kSymLinks_AllowDangerous, symLinks_AllowDangerous); + + + /* + bool supportSymLink = options.SymLinks.Val; + + if (!options.SymLinks.Def) + { + if (isExtractOrList) + supportSymLink = true; + else + supportSymLink = false; + } + + #ifdef ENV_HAVE_LSTAT + if (supportSymLink) + global_use_lstat = 1; + else + global_use_lstat = 0; + #endif + */ + + + if (isExtractOrList) + { + CExtractOptionsBase &eo = options.ExtractOptions; + + eo.ExcludeDirItems = options.Censor.ExcludeDirItems; + eo.ExcludeFileItems = options.Censor.ExcludeFileItems; + + { + CExtractNtOptions &nt = eo.NtOptions; + nt.NtSecurity = options.NtSecurity; + + nt.AltStreams = options.AltStreams; + if (!options.AltStreams.Def) + nt.AltStreams.Val = true; + + nt.HardLinks = options.HardLinks; + if (!options.HardLinks.Def) + nt.HardLinks.Val = true; + + nt.SymLinks = options.SymLinks; + if (!options.SymLinks.Def) + nt.SymLinks.Val = true; + + nt.SymLinks_AllowDangerous = symLinks_AllowDangerous; + + nt.ReplaceColonForAltStream = parser[NKey::kReplaceColonForAltStream].ThereIs; + nt.WriteToAltStreamIfColon = parser[NKey::kWriteToAltStreamIfColon].ThereIs; + + nt.ExtractOwner = options.StoreOwnerId.Val; // StoreOwnerName + + if (parser[NKey::kPreserveATime].ThereIs) + nt.PreserveATime = true; + if (parser[NKey::kShareForWrite].ThereIs) + nt.OpenShareForWrite = true; + } + + if (parser[NKey::kZoneFile].ThereIs) + { + eo.ZoneMode = NExtract::NZoneIdMode::kAll; + const UString &s = parser[NKey::kZoneFile].PostStrings[0]; + if (!s.IsEmpty()) + { + if (s == L"0") eo.ZoneMode = NExtract::NZoneIdMode::kNone; + else if (s == L"1") eo.ZoneMode = NExtract::NZoneIdMode::kAll; + else if (s == L"2") eo.ZoneMode = NExtract::NZoneIdMode::kOffice; + else + throw CArcCmdLineException("Unsupported -snz:", s); + } + } + + options.Censor.AddPathsToCensor(NWildcard::k_AbsPath); + options.Censor.ExtendExclude(); + + // are there paths that look as non-relative (!Prefix.IsEmpty()) + if (!options.Censor.AllAreRelative()) + throw CArcCmdLineException("Cannot use absolute pathnames for this command"); + + NWildcard::CCensor &arcCensor = options.arcCensor; + + CNameOption nopArc; + // nopArc.RecursedType = NRecursedType::kNonRecursed; // default: we don't want recursing for archives, if -r specified + // is it OK, external switches can disable WildcardMatching and MarcMode for arc. + nopArc.WildcardMatching = nop.WildcardMatching; + nopArc.MarkMode = nop.MarkMode; + + if (parser[NKey::kArInclude].ThereIs) + { + nopArc.Include = true; + AddSwitchWildcardsToCensor(arcCensor, parser[NKey::kArInclude].PostStrings, nopArc, codePage); + } + if (parser[NKey::kArExclude].ThereIs) + { + nopArc.Include = false; + AddSwitchWildcardsToCensor(arcCensor, parser[NKey::kArExclude].PostStrings, nopArc, codePage); + } + + if (thereIsArchiveName) + { + nopArc.Include = true; + AddNameToCensor(arcCensor, nopArc, options.ArchiveName); + } + + arcCensor.AddPathsToCensor(NWildcard::k_RelatPath); + + #ifdef _WIN32 + ConvertToLongNames(arcCensor); + #endif + + arcCensor.ExtendExclude(); + + if (options.StdInMode) + options.ArcName_for_StdInMode = parser[NKey::kStdIn].PostStrings.Front(); + + if (isExtractGroupCommand) + { + if (options.StdOutMode) + { + if ( + options.Number_for_Percents == k_OutStream_stdout + // || options.Number_for_Out == k_OutStream_stdout + // || options.Number_for_Errors == k_OutStream_stdout + || + ( + (options.IsStdOutTerminal && options.IsStdErrTerminal) + && + ( + options.Number_for_Percents != k_OutStream_disabled + // || options.Number_for_Out != k_OutStream_disabled + // || options.Number_for_Errors != k_OutStream_disabled + ) + ) + ) + throw CArcCmdLineException(kSameTerminalError); + } + + if (parser[NKey::kOutputDir].ThereIs) + { + eo.OutputDir = us2fs(parser[NKey::kOutputDir].PostStrings[0]); + #ifdef _WIN32 + NFile::NName::NormalizeDirSeparators(eo.OutputDir); + #endif + NFile::NName::NormalizeDirPathPrefix(eo.OutputDir); + } + + eo.OverwriteMode = NExtract::NOverwriteMode::kAsk; + if (parser[NKey::kOverwrite].ThereIs) + { + eo.OverwriteMode = k_OverwriteModes[(unsigned)parser[NKey::kOverwrite].PostCharIndex]; + eo.OverwriteMode_Force = true; + } + else if (options.YesToAll) + { + eo.OverwriteMode = NExtract::NOverwriteMode::kOverwrite; + eo.OverwriteMode_Force = true; + } + } + + eo.PathMode = options.Command.GetPathMode(); + if (censorPathMode == NWildcard::k_AbsPath) + { + eo.PathMode = NExtract::NPathMode::kAbsPaths; + eo.PathMode_Force = true; + } + else if (censorPathMode == NWildcard::k_FullPath) + { + eo.PathMode = NExtract::NPathMode::kFullPaths; + eo.PathMode_Force = true; + } + } + else if (options.Command.IsFromUpdateGroup()) + { + if (parser[NKey::kArInclude].ThereIs) + throw CArcCmdLineException("-ai switch is not supported for this command"); + + CUpdateOptions &updateOptions = options.UpdateOptions; + + SetAddCommandOptions(options.Command.CommandType, parser, updateOptions); + + updateOptions.MethodMode.Properties = options.Properties; + + if (parser[NKey::kPreserveATime].ThereIs) + updateOptions.PreserveATime = true; + if (parser[NKey::kShareForWrite].ThereIs) + updateOptions.OpenShareForWrite = true; + if (parser[NKey::kStopAfterOpenError].ThereIs) + updateOptions.StopAfterOpenError = true; + + updateOptions.PathMode = censorPathMode; + + updateOptions.AltStreams = options.AltStreams; + updateOptions.NtSecurity = options.NtSecurity; + updateOptions.HardLinks = options.HardLinks; + updateOptions.SymLinks = options.SymLinks; + + updateOptions.StoreOwnerId = options.StoreOwnerId; + updateOptions.StoreOwnerName = options.StoreOwnerName; + + updateOptions.EMailMode = parser[NKey::kEmail].ThereIs; + if (updateOptions.EMailMode) + { + updateOptions.EMailAddress = parser[NKey::kEmail].PostStrings.Front(); + if (updateOptions.EMailAddress.Len() > 0) + if (updateOptions.EMailAddress[0] == L'.') + { + updateOptions.EMailRemoveAfter = true; + updateOptions.EMailAddress.Delete(0); + } + } + + updateOptions.StdOutMode = options.StdOutMode; + updateOptions.StdInMode = options.StdInMode; + + updateOptions.DeleteAfterCompressing = parser[NKey::kDeleteAfterCompressing].ThereIs; + updateOptions.SetArcMTime = parser[NKey::kSetArcMTime].ThereIs; + + if (updateOptions.StdOutMode && updateOptions.EMailMode) + throw CArcCmdLineException("stdout mode and email mode cannot be combined"); + + if (updateOptions.StdOutMode) + { + if (options.IsStdOutTerminal) + throw CArcCmdLineException(kTerminalOutError); + + if (options.Number_for_Percents == k_OutStream_stdout + || options.Number_for_Out == k_OutStream_stdout + || options.Number_for_Errors == k_OutStream_stdout) + throw CArcCmdLineException(kSameTerminalError); + } + + if (updateOptions.StdInMode) + updateOptions.StdInFileName = parser[NKey::kStdIn].PostStrings.Front(); + + if (options.Command.CommandType == NCommandType::kRename) + if (updateOptions.Commands.Size() != 1) + throw CArcCmdLineException("Only one archive can be created with rename command"); + } + else if (options.Command.CommandType == NCommandType::kBenchmark) + { + options.NumIterations = 1; + options.NumIterations_Defined = false; + if (curCommandIndex < numNonSwitchStrings) + { + if (!StringToUInt32(nonSwitchStrings[curCommandIndex], options.NumIterations)) + throw CArcCmdLineException("Incorrect number of benchmark iterations", nonSwitchStrings[curCommandIndex]); + curCommandIndex++; + options.NumIterations_Defined = true; + } + } + else if (options.Command.CommandType == NCommandType::kHash) + { + options.Censor.AddPathsToCensor(censorPathMode); + options.Censor.ExtendExclude(); + + CHashOptions &hashOptions = options.HashOptions; + hashOptions.PathMode = censorPathMode; + hashOptions.Methods = options.HashMethods; + // hashOptions.HashFilePath = options.HashFilePath; + if (parser[NKey::kPreserveATime].ThereIs) + hashOptions.PreserveATime = true; + if (parser[NKey::kShareForWrite].ThereIs) + hashOptions.OpenShareForWrite = true; + hashOptions.StdInMode = options.StdInMode; + hashOptions.AltStreamsMode = options.AltStreams.Val; + hashOptions.SymLinks = options.SymLinks; + } + else if (options.Command.CommandType == NCommandType::kInfo) + { + } + else + throw 20150919; +} + + + +#ifndef _WIN32 + +static AString g_ModuleDirPrefix; + +void Set_ModuleDirPrefix_From_ProgArg0(const char *s); +void Set_ModuleDirPrefix_From_ProgArg0(const char *s) +{ + AString a (s); + int sep = a.ReverseFind_PathSepar(); + a.DeleteFrom((unsigned)(sep + 1)); + g_ModuleDirPrefix = a; +} + +namespace NWindows { +namespace NDLL { + +FString GetModuleDirPrefix(); +FString GetModuleDirPrefix() +{ + FString s; + + s = fas2fs(g_ModuleDirPrefix); + if (s.IsEmpty()) + s = FTEXT(".") FSTRING_PATH_SEPARATOR; + return s; + /* + setenv("_7ZIP_HOME_DIR", "/test/", 0); + const char *home = getenv("_7ZIP_HOME_DIR"); + if (home) + s = home; + else + s = FTEXT(".") FSTRING_PATH_SEPARATOR; + return s; + */ +} + +}} + +#endif // ! _WIN32 diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.cpp new file mode 100644 index 000000000..678d9cae6 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.cpp @@ -0,0 +1,4941 @@ +// Bench.cpp + +#include "StdAfx.h" + +// #include + +#ifndef _WIN32 +#define USE_POSIX_TIME +#define USE_POSIX_TIME2 +#endif // _WIN32 + +#ifdef USE_POSIX_TIME +#include +#include +#ifdef USE_POSIX_TIME2 +#include +#include +#endif +#endif // USE_POSIX_TIME + +#ifdef _WIN32 +#define USE_ALLOCA +#endif + +#ifdef USE_ALLOCA +#ifdef _WIN32 +#include +#else +#include +#endif +#define BENCH_ALLOCA_VALUE(index) (((index) * 64 * 21) & 0x7FF) +#endif + +#include "../../../../C/7zCrc.h" +#include "../../../../C/RotateDefs.h" +#include "../../../../C/CpuArch.h" + +#ifndef Z7_ST +#include "../../../Windows/Synchronization.h" +#include "../../../Windows/Thread.h" +#endif + +#include "../../../Windows/FileFind.h" +#include "../../../Windows/FileIO.h" +#include "../../../Windows/SystemInfo.h" + +#include "../../../Common/MyBuffer2.h" +#include "../../../Common/IntToString.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/StringToInt.h" +#include "../../../Common/Wildcard.h" + +#include "../../Common/MethodProps.h" +#include "../../Common/StreamObjects.h" +#include "../../Common/StreamUtils.h" + +#include "Bench.h" + +using namespace NWindows; + +#ifndef Z7_ST +static const UInt32 k_LZMA = 0x030101; +#endif + +static const UInt64 kComplexInCommands = (UInt64)1 << + #ifdef UNDER_CE + 31; + #else + 34; + #endif + +static const UInt32 kComplexInMs = 4000; + +static void SetComplexCommandsMs(UInt32 complexInMs, + bool isSpecifiedFreq, UInt64 cpuFreq, UInt64 &complexInCommands) +{ + complexInCommands = kComplexInCommands; + const UInt64 kMinFreq = (UInt64)1000000 * 4; + const UInt64 kMaxFreq = (UInt64)1000000 * 20000; + if (cpuFreq < kMinFreq && !isSpecifiedFreq) + cpuFreq = kMinFreq; + if (cpuFreq < kMaxFreq || isSpecifiedFreq) + { + if (complexInMs != 0) + complexInCommands = complexInMs * cpuFreq / 1000; + else + complexInCommands = cpuFreq >> 2; + } +} + +// const UInt64 kBenchmarkUsageMult = 1000000; // for debug +static const unsigned kBenchmarkUsageMultBits = 16; +static const UInt64 kBenchmarkUsageMult = 1 << kBenchmarkUsageMultBits; + +UInt64 Benchmark_GetUsage_Percents(UInt64 usage) +{ + return (100 * usage + kBenchmarkUsageMult / 2) / kBenchmarkUsageMult; +} + +static const unsigned kNumHashDictBits = 17; +static const UInt32 kFilterUnpackSize = (47 << 10); // + 5; // for test + +static const unsigned kOldLzmaDictBits = 32; + +// static const size_t kAdditionalSize = (size_t)1 << 32; // for debug +static const size_t kAdditionalSize = (size_t)1 << 16; +static const size_t kCompressedAdditionalSize = 1 << 10; + +static const UInt32 kMaxMethodPropSize = 1 << 6; + + +#define ALLOC_WITH_HRESULT(_buffer_, _size_) \ + { (_buffer_)->Alloc(_size_); \ + if (_size_ && !(_buffer_)->IsAllocated()) return E_OUTOFMEMORY; } + + +class CBaseRandomGenerator +{ + UInt32 A1; + UInt32 A2; + UInt32 Salt; +public: + CBaseRandomGenerator(UInt32 salt = 0): Salt(salt) { Init(); } + void Init() { A1 = 362436069; A2 = 521288629;} + Z7_FORCE_INLINE + UInt32 GetRnd() + { +#if 0 + // for debug: + return 0x0c080400; + // return 0; +#else + return Salt ^ + ( + ((A1 = 36969 * (A1 & 0xffff) + (A1 >> 16)) << 16) + + ((A2 = 18000 * (A2 & 0xffff) + (A2 >> 16)) ) + ); +#endif + } +}; + + +static const size_t k_RandBuf_AlignMask = 4 - 1; + +Z7_NO_INLINE +static void RandGen_BufAfterPad(Byte *buf, size_t size) +{ + CBaseRandomGenerator RG; + for (size_t i = 0; i < size; i += 4) + { + const UInt32 v = RG.GetRnd(); + SetUi32a(buf + i, v) + } + /* + UInt32 v = RG.GetRnd(); + for (; i < size; i++) + { + buf[i] = (Byte)v; + v >>= 8; + } + */ +} + + +class CBenchRandomGenerator: public CMidAlignedBuffer +{ + static UInt32 GetVal(UInt32 &res, unsigned numBits) + { + const UInt32 val = res & (((UInt32)1 << numBits) - 1); + res >>= numBits; + return val; + } + + static UInt32 GetLen(UInt32 &r) + { + const unsigned len = (unsigned)GetVal(r, 2); + return GetVal(r, 1 + len); + } + +public: + + void GenerateSimpleRandom(UInt32 salt) + { + CBaseRandomGenerator rg(salt); + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; + for (size_t i = 0; i < bufSize; i++) + buf[i] = (Byte)rg.GetRnd(); + } + + void GenerateLz(unsigned dictBits, UInt32 salt) + { + CBaseRandomGenerator rg(salt); + size_t pos = 0; + size_t rep0 = 1; + const size_t bufSize = Size(); + Byte *buf = (Byte *)*this; + unsigned posBits = 1; + + // printf("\n dictBits = %d\n", (UInt32)dictBits); + // printf("\n bufSize = 0x%p\n", (const void *)bufSize); + + while (pos < bufSize) + { + /* + if (pos >= ((UInt32)1 << 31)) + printf(" %x\n", pos); + */ + UInt32 r = rg.GetRnd(); + if (GetVal(r, 1) == 0 || pos < 1024) + buf[pos++] = (Byte)(r & 0xFF); + else + { + UInt32 len; + len = 1 + GetLen(r); + + if (GetVal(r, 3) != 0) + { + len += GetLen(r); + + while (((size_t)1 << posBits) < pos) + posBits++; + + unsigned numBitsMax = dictBits; + if (numBitsMax > posBits) + numBitsMax = posBits; + + const unsigned kAddBits = 6; + unsigned numLogBits = 5; + if (numBitsMax <= (1 << 4) - 1 + kAddBits) + numLogBits = 4; + + for (;;) + { + const UInt32 ppp = GetVal(r, numLogBits) + kAddBits; + r = rg.GetRnd(); + if (ppp > numBitsMax) + continue; + // rep0 = GetVal(r, ppp); + rep0 = r & (((size_t)1 << ppp) - 1); + if (rep0 < pos) + break; + r = rg.GetRnd(); + } + rep0++; + } + + // len *= 300; // for debug + { + const size_t rem = bufSize - pos; + if (len > rem) + len = (UInt32)rem; + } + Byte *dest = buf + pos; + const Byte *src = dest - rep0; + pos += len; + for (UInt32 i = 0; i < len; i++) + *dest++ = *src++; + } + } + // printf("\n CRC = %x\n", CrcCalc(buf, bufSize)); + } +}; + + +Z7_CLASS_IMP_NOQIB_1( + CBenchmarkInStream + , ISequentialInStream +) + const Byte *Data; + size_t Pos; + size_t Size; +public: + void Init(const Byte *data, size_t size) + { + Data = data; + Size = size; + Pos = 0; + } + bool WasFinished() const { return Pos == Size; } +}; + +Z7_COM7F_IMF(CBenchmarkInStream::Read(void *data, UInt32 size, UInt32 *processedSize)) +{ + const UInt32 kMaxBlockSize = (1 << 20); + if (size > kMaxBlockSize) + size = kMaxBlockSize; + const size_t remain = Size - Pos; + if (size > remain) + size = (UInt32)remain; + + if (size) + memcpy(data, Data + Pos, size); + + Pos += size; + if (processedSize) + *processedSize = size; + return S_OK; +} + + +class CBenchmarkOutStream Z7_final: + public ISequentialOutStream, + public CMyUnknownImp, + public CMidAlignedBuffer +{ + Z7_COM_UNKNOWN_IMP_0 + Z7_IFACE_COM7_IMP(ISequentialOutStream) + // bool _overflow; +public: + size_t Pos; + bool RealCopy; + bool CalcCrc; + UInt32 Crc; + + // CBenchmarkOutStream(): _overflow(false) {} + void Init(bool realCopy, bool calcCrc) + { + Crc = CRC_INIT_VAL; + RealCopy = realCopy; + CalcCrc = calcCrc; + // _overflow = false; + Pos = 0; + } + + void InitCrc() + { + Crc = CRC_INIT_VAL; + } + + void Calc(const void *data, size_t size) + { + Crc = CrcUpdate(Crc, data, size); + } + + size_t GetPos() const { return Pos; } + + // void Print() { printf("\n%8d %8d\n", (unsigned)BufferSize, (unsigned)Pos); } +}; + +Z7_COM7F_IMF(CBenchmarkOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + size_t curSize = Size() - Pos; + if (curSize > size) + curSize = size; + if (curSize != 0) + { + if (RealCopy) + memcpy(((Byte *)*this) + Pos, data, curSize); + if (CalcCrc) + Calc(data, curSize); + Pos += curSize; + } + if (processedSize) + *processedSize = (UInt32)curSize; + if (curSize != size) + { + // _overflow = true; + return E_FAIL; + } + return S_OK; +} + + +Z7_CLASS_IMP_NOQIB_1( + CCrcOutStream + , ISequentialOutStream +) +public: + bool CalcCrc; + UInt32 Crc; + UInt64 Pos; + + CCrcOutStream(): CalcCrc(true) {} + void Init() { Crc = CRC_INIT_VAL; Pos = 0; } + void Calc(const void *data, size_t size) + { + Crc = CrcUpdate(Crc, data, size); + } +}; + +Z7_COM7F_IMF(CCrcOutStream::Write(const void *data, UInt32 size, UInt32 *processedSize)) +{ + if (CalcCrc) + Calc(data, size); + Pos += size; + if (processedSize) + *processedSize = size; + return S_OK; +} + +// #include "../../../../C/My_sys_time.h" + +static UInt64 GetTimeCount() +{ + #ifdef USE_POSIX_TIME + #ifdef USE_POSIX_TIME2 + timeval v; + if (gettimeofday(&v, NULL) == 0) + return (UInt64)(v.tv_sec) * 1000000 + (UInt64)v.tv_usec; + return (UInt64)time(NULL) * 1000000; + #else + return time(NULL); + #endif + #else + LARGE_INTEGER value; + if (::QueryPerformanceCounter(&value)) + return (UInt64)value.QuadPart; + return GetTickCount(); + #endif +} + +static UInt64 GetFreq() +{ + #ifdef USE_POSIX_TIME + #ifdef USE_POSIX_TIME2 + return 1000000; + #else + return 1; + #endif + #else + LARGE_INTEGER value; + if (::QueryPerformanceFrequency(&value)) + return (UInt64)value.QuadPart; + return 1000; + #endif +} + + +#ifdef USE_POSIX_TIME + +struct CUserTime +{ + UInt64 Sum; + clock_t Prev; + + void Init() + { + // Prev = clock(); + Sum = 0; + Prev = 0; + Update(); + Sum = 0; + } + + void Update() + { + tms t; + /* clock_t res = */ times(&t); + clock_t newVal = t.tms_utime + t.tms_stime; + Sum += (UInt64)(newVal - Prev); + Prev = newVal; + + /* + clock_t v = clock(); + if (v != -1) + { + Sum += v - Prev; + Prev = v; + } + */ + } + UInt64 GetUserTime() + { + Update(); + return Sum; + } +}; + +#else + + +struct CUserTime +{ + bool UseTick; + DWORD Prev_Tick; + UInt64 Prev; + UInt64 Sum; + + void Init() + { + UseTick = false; + Prev_Tick = 0; + Prev = 0; + Sum = 0; + Update(); + Sum = 0; + } + UInt64 GetUserTime() + { + Update(); + return Sum; + } + void Update(); +}; + +static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } + +void CUserTime::Update() +{ + DWORD new_Tick = GetTickCount(); + FILETIME creationTime, exitTime, kernelTime, userTime; + if (!UseTick && + #ifdef UNDER_CE + ::GetThreadTimes(::GetCurrentThread() + #else + ::GetProcessTimes(::GetCurrentProcess() + #endif + , &creationTime, &exitTime, &kernelTime, &userTime)) + { + UInt64 newVal = GetTime64(userTime) + GetTime64(kernelTime); + Sum += newVal - Prev; + Prev = newVal; + } + else + { + UseTick = true; + Sum += (UInt64)(new_Tick - (DWORD)Prev_Tick) * 10000; + } + Prev_Tick = new_Tick; +} + + +#endif + +static UInt64 GetUserFreq() +{ + #ifdef USE_POSIX_TIME + // return CLOCKS_PER_SEC; + return (UInt64)sysconf(_SC_CLK_TCK); + #else + return 10000000; + #endif +} + +class CBenchProgressStatus Z7_final +{ + #ifndef Z7_ST + NSynchronization::CCriticalSection CS; + #endif +public: + HRESULT Res; + bool EncodeMode; + void SetResult(HRESULT res) + { + #ifndef Z7_ST + NSynchronization::CCriticalSectionLock lock(CS); + #endif + Res = res; + } + HRESULT GetResult() + { + #ifndef Z7_ST + NSynchronization::CCriticalSectionLock lock(CS); + #endif + return Res; + } +}; + +struct CBenchInfoCalc +{ + CBenchInfo BenchInfo; + CUserTime UserTime; + + void SetStartTime(); + void SetFinishTime(CBenchInfo &dest); +}; + +void CBenchInfoCalc::SetStartTime() +{ + BenchInfo.GlobalFreq = GetFreq(); + BenchInfo.UserFreq = GetUserFreq(); + BenchInfo.GlobalTime = ::GetTimeCount(); + BenchInfo.UserTime = 0; + UserTime.Init(); +} + +void CBenchInfoCalc::SetFinishTime(CBenchInfo &dest) +{ + dest = BenchInfo; + dest.GlobalTime = ::GetTimeCount() - BenchInfo.GlobalTime; + dest.UserTime = UserTime.GetUserTime(); +} + +class CBenchProgressInfo Z7_final: + public ICompressProgressInfo, + public CMyUnknownImp, + public CBenchInfoCalc +{ + Z7_COM_UNKNOWN_IMP_0 + Z7_IFACE_COM7_IMP(ICompressProgressInfo) +public: + CBenchProgressStatus *Status; + IBenchCallback *Callback; + + CBenchProgressInfo(): Callback(NULL) {} +}; + + +Z7_COM7F_IMF(CBenchProgressInfo::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize)) +{ + HRESULT res = Status->GetResult(); + if (res != S_OK) + return res; + if (!Callback) + return res; + + /* + static UInt64 inSizePrev = 0; + static UInt64 outSizePrev = 0; + UInt64 delta1 = 0, delta2 = 0, val1 = 0, val2 = 0; + if (inSize) { val1 = *inSize; delta1 = val1 - inSizePrev; inSizePrev = val1; } + if (outSize) { val2 = *outSize; delta2 = val2 - outSizePrev; outSizePrev = val2; } + UInt64 percents = delta2 * 1000; + if (delta1 != 0) + percents /= delta1; + printf("=== %7d %7d %7d %7d ratio = %4d\n", + (unsigned)(val1 >> 10), (unsigned)(delta1 >> 10), + (unsigned)(val2 >> 10), (unsigned)(delta2 >> 10), + (unsigned)percents); + */ + + CBenchInfo info; + SetFinishTime(info); + if (Status->EncodeMode) + { + info.UnpackSize = BenchInfo.UnpackSize + *inSize; + info.PackSize = BenchInfo.PackSize + *outSize; + res = Callback->SetEncodeResult(info, false); + } + else + { + info.PackSize = BenchInfo.PackSize + *inSize; + info.UnpackSize = BenchInfo.UnpackSize + *outSize; + res = Callback->SetDecodeResult(info, false); + } + if (res != S_OK) + Status->SetResult(res); + return res; +} + +static const unsigned kSubBits = 8; + +static unsigned GetLogSize(UInt64 size) +{ + unsigned i = 0; + for (;;) + { + i++; size >>= 1; if (size == 0) break; + } + return i; +} + + +static UInt32 GetLogSize_Sub(UInt64 size) +{ + if (size <= 1) + return 0; + const unsigned i = GetLogSize(size) - 1; + UInt32 v; + if (i <= kSubBits) + v = (UInt32)(size) << (kSubBits - i); + else + v = (UInt32)(size >> (i - kSubBits)); + return ((UInt32)i << kSubBits) + (v & (((UInt32)1 << kSubBits) - 1)); +} + + +static UInt64 Get_UInt64_from_double(double v) +{ + const UInt64 kMaxVal = (UInt64)1 << 62; + if (v > (double)(Int64)kMaxVal) + return kMaxVal; + return (UInt64)v; +} + +static UInt64 MyMultDiv64(UInt64 m1, UInt64 m2, UInt64 d) +{ + if (d == 0) + d = 1; + const double v = + (double)(Int64)m1 * + (double)(Int64)m2 / + (double)(Int64)d; + return Get_UInt64_from_double(v); + /* + unsigned n1 = GetLogSize(m1); + unsigned n2 = GetLogSize(m2); + while (n1 + n2 > 64) + { + if (n1 >= n2) + { + m1 >>= 1; + n1--; + } + else + { + m2 >>= 1; + n2--; + } + d >>= 1; + } + + if (d == 0) + d = 1; + return m1 * m2 / d; + */ +} + + +UInt64 CBenchInfo::GetUsage() const +{ + UInt64 userTime = UserTime; + UInt64 userFreq = UserFreq; + UInt64 globalTime = GlobalTime; + UInt64 globalFreq = GlobalFreq; + + if (userFreq == 0) + userFreq = 1; + if (globalTime == 0) + globalTime = 1; + + const double v = + ((double)(Int64)userTime / (double)(Int64)userFreq) + * ((double)(Int64)globalFreq / (double)(Int64)globalTime) + * (double)(Int64)kBenchmarkUsageMult; + return Get_UInt64_from_double(v); + /* + return MyMultDiv64( + MyMultDiv64(kBenchmarkUsageMult, userTime, userFreq), + globalFreq, globalTime); + */ +} + + +UInt64 CBenchInfo::GetRatingPerUsage(UInt64 rating) const +{ + if (UserTime == 0) + { + return 0; + // userTime = 1; + } + UInt64 globalFreq = GlobalFreq; + if (globalFreq == 0) + globalFreq = 1; + + const double v = + ((double)(Int64)GlobalTime / (double)(Int64)globalFreq) + * ((double)(Int64)UserFreq / (double)(Int64)UserTime) + * (double)(Int64)rating; + return Get_UInt64_from_double(v); + /* + return MyMultDiv64( + MyMultDiv64(rating, UserFreq, UserTime), + GlobalTime, globalFreq); + */ +} + + +UInt64 CBenchInfo::GetSpeed(UInt64 numUnits) const +{ + return MyMultDiv64(numUnits, GlobalFreq, GlobalTime); +} + +static UInt64 GetNumCommands_from_Size_and_Complexity(UInt64 size, Int32 complexity) +{ + return complexity >= 0 ? + size * (UInt32)complexity : + size / (UInt32)(-complexity); +} + +struct CBenchProps +{ + bool LzmaRatingMode; + + Int32 EncComplex; + Int32 DecComplexCompr; + Int32 DecComplexUnc; + + unsigned KeySize; + + CBenchProps(): + LzmaRatingMode(false), + KeySize(0) + {} + + void SetLzmaCompexity(); + + UInt64 GetNumCommands_Enc(UInt64 unpackSize) const + { + const UInt32 kMinSize = 100; + if (unpackSize < kMinSize) + unpackSize = kMinSize; + return GetNumCommands_from_Size_and_Complexity(unpackSize, EncComplex); + } + + UInt64 GetNumCommands_Dec(UInt64 packSize, UInt64 unpackSize) const + { + return + GetNumCommands_from_Size_and_Complexity(packSize, DecComplexCompr) + + GetNumCommands_from_Size_and_Complexity(unpackSize, DecComplexUnc); + } + + UInt64 GetRating_Enc(UInt64 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) const; + UInt64 GetRating_Dec(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations) const; +}; + +void CBenchProps::SetLzmaCompexity() +{ + EncComplex = 1200; + DecComplexUnc = 4; + DecComplexCompr = 190; + LzmaRatingMode = true; +} + +UInt64 CBenchProps::GetRating_Enc(UInt64 dictSize, UInt64 elapsedTime, UInt64 freq, UInt64 size) const +{ + if (dictSize < (1 << kBenchMinDicLogSize)) + dictSize = (1 << kBenchMinDicLogSize); + Int32 encComplex = EncComplex; + if (LzmaRatingMode) + { + /* + for (UInt64 uu = 0; uu < (UInt64)0xf << 60;) + { + unsigned rr = GetLogSize_Sub(uu); + printf("\n%16I64x , log = %4x", uu, rr); + uu += 1; + uu += uu / 50; + } + */ + // throw 1; + const UInt32 t = GetLogSize_Sub(dictSize) - (kBenchMinDicLogSize << kSubBits); + encComplex = 870 + ((t * t * 5) >> (2 * kSubBits)); + } + const UInt64 numCommands = GetNumCommands_from_Size_and_Complexity(size, encComplex); + return MyMultDiv64(numCommands, freq, elapsedTime); +} + +UInt64 CBenchProps::GetRating_Dec(UInt64 elapsedTime, UInt64 freq, UInt64 outSize, UInt64 inSize, UInt64 numIterations) const +{ + const UInt64 numCommands = GetNumCommands_Dec(inSize, outSize) * numIterations; + return MyMultDiv64(numCommands, freq, elapsedTime); +} + + + +UInt64 CBenchInfo::GetRating_LzmaEnc(UInt64 dictSize) const +{ + CBenchProps props; + props.SetLzmaCompexity(); + return props.GetRating_Enc(dictSize, GlobalTime, GlobalFreq, UnpackSize * NumIterations); +} + +UInt64 CBenchInfo::GetRating_LzmaDec() const +{ + CBenchProps props; + props.SetLzmaCompexity(); + return props.GetRating_Dec(GlobalTime, GlobalFreq, UnpackSize, PackSize, NumIterations); +} + + +#ifndef Z7_ST + +#define NUM_CPU_LEVELS_MAX 3 + +struct CAffinityMode +{ + unsigned NumBundleThreads; + unsigned NumLevels; + unsigned NumCoreThreads; + unsigned NumCores; + // unsigned DivideNum; + UInt32 Sizes[NUM_CPU_LEVELS_MAX]; + + void SetLevels(unsigned numCores, unsigned numCoreThreads); + DWORD_PTR GetAffinityMask(UInt32 bundleIndex, CCpuSet *cpuSet) const; + bool NeedAffinity() const { return NumBundleThreads != 0; } + + WRes CreateThread_WithAffinity(NWindows::CThread &thread, THREAD_FUNC_TYPE startAddress, LPVOID parameter, UInt32 bundleIndex) const + { + if (NeedAffinity()) + { + CCpuSet cpuSet; + GetAffinityMask(bundleIndex, &cpuSet); + return thread.Create_With_CpuSet(startAddress, parameter, &cpuSet); + } + return thread.Create(startAddress, parameter); + } + + CAffinityMode(): + NumBundleThreads(0), + NumLevels(0), + NumCoreThreads(1) + // DivideNum(1) + {} +}; + +void CAffinityMode::SetLevels(unsigned numCores, unsigned numCoreThreads) +{ + NumCores = numCores; + NumCoreThreads = numCoreThreads; + NumLevels = 0; + if (numCoreThreads == 0 || numCores == 0 || numCores % numCoreThreads != 0) + return; + UInt32 c = numCores / numCoreThreads; + UInt32 c2 = 1; + while ((c & 1) == 0) + { + c >>= 1; + c2 <<= 1; + } + if (c2 != 1) + Sizes[NumLevels++] = c2; + if (c != 1) + Sizes[NumLevels++] = c; + if (numCoreThreads != 1) + Sizes[NumLevels++] = numCoreThreads; + if (NumLevels == 0) + Sizes[NumLevels++] = 1; + + /* + printf("\n Cores:"); + for (unsigned i = 0; i < NumLevels; i++) + { + printf(" %d", Sizes[i]); + } + printf("\n"); + */ +} + + +DWORD_PTR CAffinityMode::GetAffinityMask(UInt32 bundleIndex, CCpuSet *cpuSet) const +{ + CpuSet_Zero(cpuSet); + + if (NumLevels == 0) + return 0; + + // printf("\n%2d", bundleIndex); + + /* + UInt32 low = 0; + if (DivideNum != 1) + { + low = bundleIndex % DivideNum; + bundleIndex /= DivideNum; + } + */ + + UInt32 numGroups = NumCores / NumBundleThreads; + UInt32 m = bundleIndex % numGroups; + UInt32 v = 0; + for (unsigned i = 0; i < NumLevels; i++) + { + UInt32 size = Sizes[i]; + while ((size & 1) == 0) + { + v *= 2; + v |= (m & 1); + m >>= 1; + size >>= 1; + } + v *= size; + v += m % size; + m /= size; + } + + // UInt32 nb = NumBundleThreads / DivideNum; + UInt32 nb = NumBundleThreads; + + DWORD_PTR mask = ((DWORD_PTR)1 << nb) - 1; + // v += low; + mask <<= v; + + // printf(" %2d %8x \n ", v, (unsigned)mask); + #ifdef _WIN32 + *cpuSet = mask; + #else + { + for (unsigned k = 0; k < nb; k++) + CpuSet_Set(cpuSet, v + k); + } + #endif + + return mask; +} + + +struct CBenchSyncCommon +{ + bool ExitMode; + NSynchronization::CManualResetEvent StartEvent; + + CBenchSyncCommon(): ExitMode(false) {} +}; + +#endif + + + +enum E_CheckCrcMode +{ + k_CheckCrcMode_Never = 0, + k_CheckCrcMode_Always = 1, + k_CheckCrcMode_FirstPass = 2 +}; + +class CEncoderInfo; + +class CEncoderInfo Z7_final +{ + Z7_CLASS_NO_COPY(CEncoderInfo) + +public: + + #ifndef Z7_ST + NWindows::CThread thread[2]; + NSynchronization::CManualResetEvent ReadyEvent; + UInt32 NumDecoderSubThreads; + CBenchSyncCommon *Common; + UInt32 EncoderIndex; + UInt32 NumEncoderInternalThreads; + CAffinityMode AffinityMode; + bool IsGlobalMtMode; // if more than one benchmark encoder threads + #endif + + CMyComPtr _encoder; + CMyComPtr _encoderFilter; + CBenchProgressInfo *progressInfoSpec[2]; + CMyComPtr progressInfo[2]; + UInt64 NumIterations; + + UInt32 Salt; + + #ifdef USE_ALLOCA + size_t AllocaSize; + #endif + + unsigned KeySize; + Byte _key[32]; + Byte _iv[16]; + + HRESULT Set_Key_and_IV(ICryptoProperties *cp) + { + RINOK(cp->SetKey(_key, KeySize)) + return cp->SetInitVector(_iv, sizeof(_iv)); + } + + Byte _psw[16]; + + bool CheckCrc_Enc; /* = 1, if we want to check packed data crcs after each pass + used for filter and usual coders */ + bool UseRealData_Enc; /* = 1, if we want to use only original data for each pass + used only for filter */ + E_CheckCrcMode CheckCrcMode_Dec; + + struct CDecoderInfo + { + CEncoderInfo *Encoder; + UInt32 DecoderIndex; + bool CallbackMode; + + #ifdef USE_ALLOCA + size_t AllocaSize; + #endif + }; + CDecoderInfo decodersInfo[2]; + + CMyComPtr _decoders[2]; + CMyComPtr _decoderFilter; + + HRESULT Results[2]; + CBenchmarkOutStream *outStreamSpec; + CMyComPtr outStream; + IBenchCallback *callback; + IBenchPrintCallback *printCallback; + UInt32 crc; + size_t kBufferSize; + size_t compressedSize; + const Byte *uncompressedDataPtr; + + const Byte *fileData; + CBenchRandomGenerator rg; + + CMidAlignedBuffer rgCopy; // it must be 16-byte aligned !!! + + // CBenchmarkOutStream *propStreamSpec; + Byte propsData[kMaxMethodPropSize]; + CBufPtrSeqOutStream *propStreamSpec; + CMyComPtr propStream; + + unsigned generateDictBits; + COneMethodInfo _method; + + // for decode + size_t _uncompressedDataSize; + + HRESULT Generate(); + HRESULT Encode(); + HRESULT Decode(UInt32 decoderIndex); + + CEncoderInfo(): + #ifndef Z7_ST + Common(NULL), + IsGlobalMtMode(true), + #endif + Salt(0), + KeySize(0), + CheckCrc_Enc(true), + UseRealData_Enc(true), + CheckCrcMode_Dec(k_CheckCrcMode_Always), + outStreamSpec(NULL), + callback(NULL), + printCallback(NULL), + fileData(NULL), + propStreamSpec(NULL) + {} + + #ifndef Z7_ST + + static THREAD_FUNC_DECL EncodeThreadFunction(void *param) + { + HRESULT res; + CEncoderInfo *encoder = (CEncoderInfo *)param; + try + { + #ifdef USE_ALLOCA + alloca(encoder->AllocaSize); + #endif + + res = encoder->Encode(); + } + catch(...) + { + res = E_FAIL; + } + encoder->Results[0] = res; + if (res != S_OK) + encoder->progressInfoSpec[0]->Status->SetResult(res); + encoder->ReadyEvent.Set(); + return THREAD_FUNC_RET_ZERO; + } + + static THREAD_FUNC_DECL DecodeThreadFunction(void *param) + { + CDecoderInfo *decoder = (CDecoderInfo *)param; + + #ifdef USE_ALLOCA + alloca(decoder->AllocaSize); + // printf("\nalloca=%d\n", (unsigned)decoder->AllocaSize); + #endif + + CEncoderInfo *encoder = decoder->Encoder; + encoder->Results[decoder->DecoderIndex] = encoder->Decode(decoder->DecoderIndex); + return THREAD_FUNC_RET_ZERO; + } + + HRESULT CreateEncoderThread() + { + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = AffinityMode.CreateThread_WithAffinity(thread[0], EncodeThreadFunction, this, + EncoderIndex); + return HRESULT_FROM_WIN32(res); + } + + HRESULT CreateDecoderThread(unsigned index, bool callbackMode + #ifdef USE_ALLOCA + , size_t allocaSize + #endif + ) + { + CDecoderInfo &decoder = decodersInfo[index]; + decoder.DecoderIndex = index; + decoder.Encoder = this; + + #ifdef USE_ALLOCA + decoder.AllocaSize = allocaSize; + #endif + + decoder.CallbackMode = callbackMode; + + WRes res = AffinityMode.CreateThread_WithAffinity(thread[index], DecodeThreadFunction, &decoder, + // EncoderIndex * NumEncoderInternalThreads + index + EncoderIndex + ); + + return HRESULT_FROM_WIN32(res); + } + + #endif +}; + + + + +static size_t GetBenchCompressedSize(size_t bufferSize) +{ + return kCompressedAdditionalSize + bufferSize + bufferSize / 16; + // kBufferSize / 2; +} + + +HRESULT CEncoderInfo::Generate() +{ + const COneMethodInfo &method = _method; + + // we need extra space, if input data is already compressed + const size_t kCompressedBufferSize = _encoderFilter ? + kBufferSize : + GetBenchCompressedSize(kBufferSize); + + if (kCompressedBufferSize < kBufferSize) + return E_FAIL; + + uncompressedDataPtr = fileData; + if (fileData) + { + #if !defined(Z7_ST) + if (IsGlobalMtMode) + { + /* we copy the data to local buffer of thread to eliminate + using of shared buffer by different threads */ + ALLOC_WITH_HRESULT(&rg, kBufferSize) + memcpy((Byte *)rg, fileData, kBufferSize); + uncompressedDataPtr = (const Byte *)rg; + } + #endif + } + else + { + ALLOC_WITH_HRESULT(&rg, kBufferSize) + // DWORD ttt = GetTickCount(); + if (generateDictBits == 0) + rg.GenerateSimpleRandom(Salt); + else + { + if (generateDictBits >= sizeof(size_t) * 8 + && kBufferSize > ((size_t)1 << (sizeof(size_t) * 8 - 1))) + return E_INVALIDARG; + rg.GenerateLz(generateDictBits, Salt); + // return E_ABORT; // for debug + } + // printf("\n%d\n ", GetTickCount() - ttt); + + crc = CrcCalc((const Byte *)rg, rg.Size()); + uncompressedDataPtr = (const Byte *)rg; + } + + if (!outStream) + { + outStreamSpec = new CBenchmarkOutStream; + outStream = outStreamSpec; + } + + ALLOC_WITH_HRESULT(outStreamSpec, kCompressedBufferSize) + + if (_encoderFilter) + { + /* we try to reduce the number of memcpy() in main encoding loop. + so we copy data to temp buffers here */ + ALLOC_WITH_HRESULT(&rgCopy, kBufferSize) + memcpy((Byte *)*outStreamSpec, uncompressedDataPtr, kBufferSize); + memcpy((Byte *)rgCopy, uncompressedDataPtr, kBufferSize); + } + + if (!propStream) + { + propStreamSpec = new CBufPtrSeqOutStream; // CBenchmarkOutStream; + propStream = propStreamSpec; + } + // ALLOC_WITH_HRESULT_2(propStreamSpec, kMaxMethodPropSize); + // propStreamSpec->Init(true, false); + propStreamSpec->Init(propsData, sizeof(propsData)); + + + CMyComPtr coder; + if (_encoderFilter) + coder = _encoderFilter; + else + coder = _encoder; + { + CMyComPtr scp; + coder.QueryInterface(IID_ICompressSetCoderProperties, &scp); + if (scp) + { + const UInt64 reduceSize = kBufferSize; + + /* in posix new thread uses same affinity as parent thread, + so we don't need to send affinity to coder in posix */ + UInt64 affMask; + #if !defined(Z7_ST) && defined(_WIN32) + { + CCpuSet cpuSet; + affMask = AffinityMode.GetAffinityMask(EncoderIndex, &cpuSet); + } + #else + affMask = 0; + #endif + // affMask <<= 3; // debug line: to test no affinity in coder; + // affMask = 0; + + RINOK(method.SetCoderProps_DSReduce_Aff(scp, &reduceSize, (affMask != 0 ? &affMask : NULL))) + } + else + { + if (method.AreThereNonOptionalProps()) + return E_INVALIDARG; + } + + CMyComPtr writeCoderProps; + coder.QueryInterface(IID_ICompressWriteCoderProperties, &writeCoderProps); + if (writeCoderProps) + { + RINOK(writeCoderProps->WriteCoderProperties(propStream)) + } + + { + CMyComPtr sp; + coder.QueryInterface(IID_ICryptoSetPassword, &sp); + if (sp) + { + RINOK(sp->CryptoSetPassword(_psw, sizeof(_psw))) + + // we must call encoding one time to calculate password key for key cache. + // it must be after WriteCoderProperties! + Byte temp[16]; + memset(temp, 0, sizeof(temp)); + + if (_encoderFilter) + { + _encoderFilter->Init(); + _encoderFilter->Filter(temp, sizeof(temp)); + } + else + { + CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream; + CMyComPtr inStream = inStreamSpec; + inStreamSpec->Init(temp, sizeof(temp)); + + CCrcOutStream *crcStreamSpec = new CCrcOutStream; + CMyComPtr crcStream = crcStreamSpec; + crcStreamSpec->Init(); + + RINOK(_encoder->Code(inStream, crcStream, NULL, NULL, NULL)) + } + } + } + } + + return S_OK; +} + + +static void My_FilterBench(ICompressFilter *filter, Byte *data, size_t size, UInt32 *crc) +{ + while (size != 0) + { + UInt32 cur = crc ? 1 << 17 : 1 << 24; + if (cur > size) + cur = (UInt32)size; + UInt32 processed = filter->Filter(data, cur); + /* if (processed > size) (in AES filter), we must fill last block with zeros. + but it is not important for benchmark. So we just copy that data without filtering. + if (processed == 0) then filter can't process more */ + if (processed > size || processed == 0) + processed = (UInt32)size; + if (crc) + *crc = CrcUpdate(*crc, data, processed); + data += processed; + size -= processed; + } +} + + +HRESULT CEncoderInfo::Encode() +{ + // printf("\nCEncoderInfo::Generate\n"); + + RINOK(Generate()) + + // printf("\n2222\n"); + + #ifndef Z7_ST + if (Common) + { + Results[0] = S_OK; + WRes wres = ReadyEvent.Set(); + if (wres == 0) + wres = Common->StartEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + if (Common->ExitMode) + return S_OK; + } + else + #endif + { + CBenchProgressInfo *bpi = progressInfoSpec[0]; + bpi->SetStartTime(); + } + + + CBenchInfo &bi = progressInfoSpec[0]->BenchInfo; + bi.UnpackSize = 0; + bi.PackSize = 0; + CMyComPtr cp; + CMyComPtr coder; + if (_encoderFilter) + coder = _encoderFilter; + else + coder = _encoder; + coder.QueryInterface(IID_ICryptoProperties, &cp); + CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream; + CMyComPtr inStream = inStreamSpec; + + if (cp) + { + RINOK(Set_Key_and_IV(cp)) + } + + compressedSize = 0; + if (_encoderFilter) + compressedSize = kBufferSize; + + // CBenchmarkOutStream *outStreamSpec = this->outStreamSpec; + UInt64 prev = 0; + + const UInt32 mask = (CheckCrc_Enc ? 0 : 0xFFFF); + const bool useCrc = (mask < NumIterations); + bool crcPrev_defined = false; + UInt32 crcPrev = 0; + + bool useRealData_Enc = UseRealData_Enc; + bool data_Was_Changed = false; + if (useRealData_Enc) + { + /* we want memcpy() for each iteration including first iteration. + So results will be equal for different number of iterations */ + data_Was_Changed = true; + } + + const UInt64 numIterations = NumIterations; + UInt64 i = numIterations; + // printCallback->NewLine(); + + while (i != 0) + { + i--; + if (printCallback && bi.UnpackSize - prev >= (1 << 26)) + { + prev = bi.UnpackSize; + RINOK(printCallback->CheckBreak()) + } + + /* + CBenchInfo info; + progressInfoSpec[0]->SetStartTime(); + */ + + bool calcCrc = false; + if (useCrc) + calcCrc = (((UInt32)i & mask) == 0); + + if (_encoderFilter) + { + Byte *filterData = rgCopy; + if (i == numIterations - 1 || calcCrc || useRealData_Enc) + { + // printf("\nfilterData = (Byte *)*outStreamSpec;\n"); + filterData = (Byte *)*outStreamSpec; + if (data_Was_Changed) + { + // printf("\nmemcpy(filterData, uncompressedDataPtr\n"); + memcpy(filterData, uncompressedDataPtr, kBufferSize); + } + data_Was_Changed = true; + } + _encoderFilter->Init(); + if (calcCrc) + { + // printf("\nInitCrc\n"); + outStreamSpec->InitCrc(); + } + // printf("\nMy_FilterBench\n"); + My_FilterBench(_encoderFilter, filterData, kBufferSize, + calcCrc ? &outStreamSpec->Crc : NULL); + } + else + { + outStreamSpec->Init(true, calcCrc); // write real data for speed consistency at any number of iterations + inStreamSpec->Init(uncompressedDataPtr, kBufferSize); + RINOK(_encoder->Code(inStream, outStream, NULL, NULL, progressInfo[0])) + if (!inStreamSpec->WasFinished()) + return E_FAIL; + if (compressedSize != outStreamSpec->Pos) + { + if (compressedSize != 0) + return E_FAIL; + compressedSize = outStreamSpec->Pos; + } + } + + // outStreamSpec->Print(); + + if (calcCrc) + { + const UInt32 crc2 = CRC_GET_DIGEST(outStreamSpec->Crc); + if (crcPrev_defined && crcPrev != crc2) + return E_FAIL; + crcPrev = crc2; + crcPrev_defined = true; + } + + bi.UnpackSize += kBufferSize; + bi.PackSize += compressedSize; + + /* + { + progressInfoSpec[0]->SetFinishTime(info); + info.UnpackSize = 0; + info.PackSize = 0; + info.NumIterations = 1; + + info.UnpackSize = kBufferSize; + info.PackSize = compressedSize; + // printf("\n%7d\n", encoder.compressedSize); + + RINOK(callback->SetEncodeResult(info, true)) + printCallback->NewLine(); + } + */ + + } + + _encoder.Release(); + _encoderFilter.Release(); + return S_OK; +} + + +HRESULT CEncoderInfo::Decode(UInt32 decoderIndex) +{ + CBenchmarkInStream *inStreamSpec = new CBenchmarkInStream; + CMyComPtr inStream = inStreamSpec; + CMyComPtr &decoder = _decoders[decoderIndex]; + CMyComPtr coder; + if (_decoderFilter) + { + if (decoderIndex != 0) + return E_FAIL; + coder = _decoderFilter; + } + else + coder = decoder; + + // printf("\ndecoderIndex = %d, stack = %p", decoderIndex, &coder); + + CMyComPtr setDecProps; + coder.QueryInterface(IID_ICompressSetDecoderProperties2, &setDecProps); + if (!setDecProps && propStreamSpec->GetPos() != 0) + return E_FAIL; + + CCrcOutStream *crcOutStreamSpec = new CCrcOutStream; + CMyComPtr crcOutStream = crcOutStreamSpec; + + CBenchProgressInfo *pi = progressInfoSpec[decoderIndex]; + pi->BenchInfo.UnpackSize = 0; + pi->BenchInfo.PackSize = 0; + + #ifndef Z7_ST + { + CMyComPtr setCoderMt; + coder.QueryInterface(IID_ICompressSetCoderMt, &setCoderMt); + if (setCoderMt) + { + RINOK(setCoderMt->SetNumberOfThreads(NumDecoderSubThreads)) + } + } + #endif + + CMyComPtr scp; + coder.QueryInterface(IID_ICompressSetCoderProperties, &scp); + if (scp) + { + const UInt64 reduceSize = _uncompressedDataSize; + RINOK(_method.SetCoderProps(scp, &reduceSize)) + } + + CMyComPtr cp; + coder.QueryInterface(IID_ICryptoProperties, &cp); + + if (setDecProps) + { + RINOK(setDecProps->SetDecoderProperties2( + /* (const Byte *)*propStreamSpec, */ + propsData, + (UInt32)propStreamSpec->GetPos())) + } + + { + CMyComPtr sp; + coder.QueryInterface(IID_ICryptoSetPassword, &sp); + if (sp) + { + RINOK(sp->CryptoSetPassword(_psw, sizeof(_psw))) + } + } + + UInt64 prev = 0; + + if (cp) + { + RINOK(Set_Key_and_IV(cp)) + } + + CMyComPtr setFinishMode; + + if (_decoderFilter) + { + if (compressedSize > rgCopy.Size()) + return E_FAIL; + } + else + { + decoder->QueryInterface(IID_ICompressSetFinishMode, (void **)&setFinishMode); + } + + const UInt64 numIterations = NumIterations; + const E_CheckCrcMode checkCrcMode = CheckCrcMode_Dec; + + for (UInt64 i = 0; i < numIterations; i++) + { + if (printCallback && pi->BenchInfo.UnpackSize - prev >= (1 << 26)) + { + RINOK(printCallback->CheckBreak()) + prev = pi->BenchInfo.UnpackSize; + } + + const UInt64 outSize = kBufferSize; + bool calcCrc = (checkCrcMode != k_CheckCrcMode_Never); + + crcOutStreamSpec->Init(); + + if (_decoderFilter) + { + Byte *filterData = (Byte *)*outStreamSpec; + if (calcCrc) + { + calcCrc = (i == 0); + if (checkCrcMode == k_CheckCrcMode_Always) + { + calcCrc = true; + memcpy((Byte *)rgCopy, (const Byte *)*outStreamSpec, compressedSize); + filterData = rgCopy; + } + } + _decoderFilter->Init(); + My_FilterBench(_decoderFilter, filterData, compressedSize, + calcCrc ? &crcOutStreamSpec->Crc : NULL); + } + else + { + crcOutStreamSpec->CalcCrc = calcCrc; + inStreamSpec->Init((const Byte *)*outStreamSpec, compressedSize); + + if (setFinishMode) + { + RINOK(setFinishMode->SetFinishMode(BoolToUInt(true))) + } + + RINOK(decoder->Code(inStream, crcOutStream, NULL, &outSize, progressInfo[decoderIndex])) + + if (setFinishMode) + { + if (!inStreamSpec->WasFinished()) + return S_FALSE; + + CMyComPtr getInStreamProcessedSize; + decoder.QueryInterface(IID_ICompressGetInStreamProcessedSize, (void **)&getInStreamProcessedSize); + + if (getInStreamProcessedSize) + { + UInt64 processed; + RINOK(getInStreamProcessedSize->GetInStreamProcessedSize(&processed)) + if (processed != compressedSize) + return S_FALSE; + } + } + + if (crcOutStreamSpec->Pos != outSize) + return S_FALSE; + } + + if (calcCrc && CRC_GET_DIGEST(crcOutStreamSpec->Crc) != crc) + return S_FALSE; + + pi->BenchInfo.UnpackSize += kBufferSize; + pi->BenchInfo.PackSize += compressedSize; + } + + decoder.Release(); + _decoderFilter.Release(); + return S_OK; +} + + +static const UInt32 kNumThreadsMax = (1 << 12); + +struct CBenchEncoders +{ + CEncoderInfo *encoders; + CBenchEncoders(UInt32 num): encoders(NULL) { encoders = new CEncoderInfo[num]; } + ~CBenchEncoders() { delete []encoders; } +}; + + +static UInt64 GetNumIterations(UInt64 numCommands, UInt64 complexInCommands) +{ + if (numCommands < (1 << 4)) + numCommands = (1 << 4); + UInt64 res = complexInCommands / numCommands; + return (res == 0 ? 1 : res); +} + + + +#ifndef Z7_ST + +// ---------- CBenchThreadsFlusher ---------- + +struct CBenchThreadsFlusher +{ + CBenchEncoders *EncodersSpec; + CBenchSyncCommon Common; + unsigned NumThreads; + bool NeedClose; + + CBenchThreadsFlusher(): NumThreads(0), NeedClose(false) {} + + ~CBenchThreadsFlusher() + { + StartAndWait(true); + } + + WRes StartAndWait(bool exitMode = false); +}; + + +WRes CBenchThreadsFlusher::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes res = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + NWindows::CThread &t = EncodersSpec->encoders[i].thread[0]; + if (t.IsCreated()) + { + WRes res2 = t.Wait_Close(); + if (res == 0) + res = res2; + } + } + NeedClose = false; + return res; +} + +#endif // Z7_ST + + + +static void SetPseudoRand(Byte *data, size_t size, UInt32 startValue) +{ + for (size_t i = 0; i < size; i++) + { + data[i] = (Byte)startValue; + startValue++; + } +} + + + +static HRESULT MethodBench( + DECL_EXTERNAL_CODECS_LOC_VARS + UInt64 complexInCommands, + #ifndef Z7_ST + bool oldLzmaBenchMode, + UInt32 numThreads, + const CAffinityMode *affinityMode, + #endif + const COneMethodInfo &method2, + size_t uncompressedDataSize, + const Byte *fileData, + unsigned generateDictBits, + + IBenchPrintCallback *printCallback, + IBenchCallback *callback, + CBenchProps *benchProps) +{ + COneMethodInfo method = method2; + UInt64 methodId; + UInt32 numStreams; + bool isFilter; + const int codecIndex = FindMethod_Index( + EXTERNAL_CODECS_LOC_VARS + method.MethodName, true, + methodId, numStreams, isFilter); + if (codecIndex < 0) + return E_NOTIMPL; + if (numStreams != 1) + return E_INVALIDARG; + + UInt32 numEncoderThreads = 1; + UInt32 numSubDecoderThreads = 1; + + #ifndef Z7_ST + numEncoderThreads = numThreads; + + if (oldLzmaBenchMode) + if (methodId == k_LZMA) + { + if (numThreads == 1 && method.Get_NumThreads() < 0) + method.AddProp_NumThreads(1); + const UInt32 numLzmaThreads = method.Get_Lzma_NumThreads(); + if (numThreads > 1 && numLzmaThreads > 1) + { + numEncoderThreads = (numThreads + 1) / 2; // 20.03 + numSubDecoderThreads = 2; + } + } + + const bool mtEncMode = (numEncoderThreads > 1) || affinityMode->NeedAffinity(); + + #endif + + CBenchEncoders encodersSpec(numEncoderThreads); + CEncoderInfo *encoders = encodersSpec.encoders; + + UInt32 i; + + for (i = 0; i < numEncoderThreads; i++) + { + CEncoderInfo &encoder = encoders[i]; + encoder.callback = (i == 0) ? callback : NULL; + encoder.printCallback = printCallback; + + #ifndef Z7_ST + encoder.EncoderIndex = i; + encoder.NumEncoderInternalThreads = numSubDecoderThreads; + encoder.AffinityMode = *affinityMode; + + /* + if (numSubDecoderThreads > 1) + if (encoder.AffinityMode.NeedAffinity() + && encoder.AffinityMode.NumBundleThreads == 1) + { + // if old LZMA benchmark uses two threads in coder, we increase (NumBundleThreads) for old LZMA benchmark uses two threads instead of one + if (encoder.AffinityMode.NumBundleThreads * 2 <= encoder.AffinityMode.NumCores) + encoder.AffinityMode.NumBundleThreads *= 2; + } + */ + + #endif + + { + CCreatedCoder cod; + RINOK(CreateCoder_Index(EXTERNAL_CODECS_LOC_VARS (unsigned)codecIndex, true, encoder._encoderFilter, cod)) + encoder._encoder = cod.Coder; + if (!encoder._encoder && !encoder._encoderFilter) + return E_NOTIMPL; + } + + SetPseudoRand(encoder._iv, sizeof(encoder._iv), 17); + SetPseudoRand(encoder._key, sizeof(encoder._key), 51); + SetPseudoRand(encoder._psw, sizeof(encoder._psw), 123); + + for (UInt32 j = 0; j < numSubDecoderThreads; j++) + { + CCreatedCoder cod; + CMyComPtr &decoder = encoder._decoders[j]; + RINOK(CreateCoder_Id(EXTERNAL_CODECS_LOC_VARS methodId, false, encoder._decoderFilter, cod)) + decoder = cod.Coder; + if (!encoder._decoderFilter && !decoder) + return E_NOTIMPL; + } + + encoder.UseRealData_Enc = + encoder.CheckCrc_Enc = (benchProps->EncComplex) > 30; + + encoder.CheckCrcMode_Dec = k_CheckCrcMode_Always; + if (benchProps->DecComplexCompr + + benchProps->DecComplexUnc <= 30) + encoder.CheckCrcMode_Dec = + k_CheckCrcMode_FirstPass; // for filters + // k_CheckCrcMode_Never; // for debug + // k_CheckCrcMode_Always; // for debug + if (fileData) + { + encoder.UseRealData_Enc = true; + encoder.CheckCrcMode_Dec = k_CheckCrcMode_Always; + } + } + + UInt32 crc = 0; + if (fileData) + crc = CrcCalc(fileData, uncompressedDataSize); + + for (i = 0; i < numEncoderThreads; i++) + { + CEncoderInfo &encoder = encoders[i]; + encoder._method = method; + encoder.generateDictBits = generateDictBits; + encoder._uncompressedDataSize = uncompressedDataSize; + encoder.kBufferSize = uncompressedDataSize; + encoder.fileData = fileData; + encoder.crc = crc; + } + + CBenchProgressStatus status; + status.Res = S_OK; + status.EncodeMode = true; + + #ifndef Z7_ST + CBenchThreadsFlusher encoderFlusher; + if (mtEncMode) + { + WRes wres = encoderFlusher.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + encoderFlusher.NumThreads = numEncoderThreads; + encoderFlusher.EncodersSpec = &encodersSpec; + encoderFlusher.NeedClose = true; + } + #endif + + for (i = 0; i < numEncoderThreads; i++) + { + CEncoderInfo &encoder = encoders[i]; + encoder.NumIterations = GetNumIterations(benchProps->GetNumCommands_Enc(uncompressedDataSize), complexInCommands); + // encoder.NumIterations = 3; + { +#if 0 + #define kCrcPoly 0xEDB88320 + UInt32 r = i; + unsigned num = numEncoderThreads < 256 ? 8 : 16; + do + r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1))); + while (--num); + encoder.Salt = r; +#else + UInt32 salt0 = g_CrcTable[(Byte)i]; + UInt32 salt1 = g_CrcTable[(Byte)(i >> 8)]; + encoder.Salt = salt0 ^ (salt1 << 3); +#endif + } + + // (g_CrcTable[0] == 0), and (encoder.Salt == 0) for first thread + // printf("\n encoder index = %d, Salt = %8x\n", i, encoder.Salt); + + encoder.KeySize = benchProps->KeySize; + + for (int j = 0; j < 2; j++) + { + CBenchProgressInfo *spec = new CBenchProgressInfo; + encoder.progressInfoSpec[j] = spec; + encoder.progressInfo[j] = spec; + spec->Status = &status; + } + + if (i == 0) + { + CBenchProgressInfo *bpi = encoder.progressInfoSpec[0]; + bpi->Callback = callback; + bpi->BenchInfo.NumIterations = numEncoderThreads; + } + + #ifndef Z7_ST + if (mtEncMode) + { + #ifdef USE_ALLOCA + encoder.AllocaSize = BENCH_ALLOCA_VALUE(i); + #endif + + encoder.Common = &encoderFlusher.Common; + encoder.IsGlobalMtMode = numEncoderThreads > 1; + RINOK(encoder.CreateEncoderThread()) + } + #endif + } + + if (printCallback) + { + RINOK(printCallback->CheckBreak()) + } + + #ifndef Z7_ST + if (mtEncMode) + { + for (i = 0; i < numEncoderThreads; i++) + { + CEncoderInfo &encoder = encoders[i]; + const WRes wres = encoder.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(encoder.Results[0]) + } + + CBenchProgressInfo *bpi = encoders[0].progressInfoSpec[0]; + bpi->SetStartTime(); + + const WRes wres = encoderFlusher.StartAndWait(); + if (status.Res == 0 && wres != 0) + return HRESULT_FROM_WIN32(wres); + } + else + #endif + { + RINOK(encoders[0].Encode()) + } + + RINOK(status.Res) + + CBenchInfo info; + + encoders[0].progressInfoSpec[0]->SetFinishTime(info); + info.UnpackSize = 0; + info.PackSize = 0; + info.NumIterations = encoders[0].NumIterations; + + for (i = 0; i < numEncoderThreads; i++) + { + const CEncoderInfo &encoder = encoders[i]; + info.UnpackSize += encoder.kBufferSize; + info.PackSize += encoder.compressedSize; + // printf("\n%7d\n", encoder.compressedSize); + } + + RINOK(callback->SetEncodeResult(info, true)) + + + + + // ---------- Decode ---------- + + status.Res = S_OK; + status.EncodeMode = false; + + const UInt32 numDecoderThreads = numEncoderThreads * numSubDecoderThreads; + #ifndef Z7_ST + const bool mtDecoderMode = (numDecoderThreads > 1) || affinityMode->NeedAffinity(); + #endif + + for (i = 0; i < numEncoderThreads; i++) + { + CEncoderInfo &encoder = encoders[i]; + + /* + #ifndef Z7_ST + // encoder.affinityMode = *affinityMode; + if (encoder.NumEncoderInternalThreads != 1) + encoder.AffinityMode.DivideNum = encoder.NumEncoderInternalThreads; + #endif + */ + + + if (i == 0) + { + encoder.NumIterations = GetNumIterations( + benchProps->GetNumCommands_Dec( + encoder.compressedSize, + encoder.kBufferSize), + complexInCommands); + CBenchProgressInfo *bpi = encoder.progressInfoSpec[0]; + bpi->Callback = callback; + bpi->BenchInfo.NumIterations = numDecoderThreads; + bpi->SetStartTime(); + } + else + encoder.NumIterations = encoders[0].NumIterations; + + #ifndef Z7_ST + { + const int numSubThreads = method.Get_NumThreads(); + encoder.NumDecoderSubThreads = (numSubThreads <= 0) ? 1 : (unsigned)numSubThreads; + } + if (mtDecoderMode) + { + for (UInt32 j = 0; j < numSubDecoderThreads; j++) + { + const HRESULT res = encoder.CreateDecoderThread(j, (i == 0 && j == 0) + #ifdef USE_ALLOCA + , BENCH_ALLOCA_VALUE(i * numSubDecoderThreads + j) + #endif + ); + RINOK(res) + } + } + else + #endif + { + RINOK(encoder.Decode(0)) + } + } + + #ifndef Z7_ST + if (mtDecoderMode) + { + WRes wres = 0; + HRESULT res = S_OK; + for (i = 0; i < numEncoderThreads; i++) + for (UInt32 j = 0; j < numSubDecoderThreads; j++) + { + CEncoderInfo &encoder = encoders[i]; + const WRes wres2 = encoder.thread[j]. + // Wait(); // later we can get thread times from thread in UNDER_CE + Wait_Close(); + if (wres == 0 && wres2 != 0) + wres = wres2; + const HRESULT res2 = encoder.Results[j]; + if (res == 0 && res2 != 0) + res = res2; + } + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(res) + } + #endif // Z7_ST + + RINOK(status.Res) + encoders[0].progressInfoSpec[0]->SetFinishTime(info); + + /* + #ifndef Z7_ST + #ifdef UNDER_CE + if (mtDecoderMode) + for (i = 0; i < numEncoderThreads; i++) + for (UInt32 j = 0; j < numSubDecoderThreads; j++) + { + FILETIME creationTime, exitTime, kernelTime, userTime; + if (::GetThreadTimes(encoders[i].thread[j], &creationTime, &exitTime, &kernelTime, &userTime) != 0) + info.UserTime += GetTime64(userTime) + GetTime64(kernelTime); + } + #endif + #endif + */ + + info.UnpackSize = 0; + info.PackSize = 0; + info.NumIterations = numSubDecoderThreads * encoders[0].NumIterations; + + for (i = 0; i < numEncoderThreads; i++) + { + const CEncoderInfo &encoder = encoders[i]; + info.UnpackSize += encoder.kBufferSize; + info.PackSize += encoder.compressedSize; + } + + // RINOK(callback->SetDecodeResult(info, false)) // why we called before 21.03 ?? + RINOK(callback->SetDecodeResult(info, true)) + + return S_OK; +} + + + +static inline UInt64 GetDictSizeFromLog(unsigned dictSizeLog) +{ + /* + if (dictSizeLog < 32) + return (UInt32)1 << dictSizeLog; + else + return (UInt32)(Int32)-1; + */ + return (UInt64)1 << dictSizeLog; +} + + +// it's limit of current LZMA implementation that can be changed later +#define kLzmaMaxDictSize ((UInt32)15 << 28) + +static inline UInt64 GetLZMAUsage(bool multiThread, int btMode, UInt64 dict) +{ + if (dict == 0) + dict = 1; + if (dict > kLzmaMaxDictSize) + dict = kLzmaMaxDictSize; + UInt32 hs = (UInt32)dict - 1; + hs |= (hs >> 1); + hs |= (hs >> 2); + hs |= (hs >> 4); + hs |= (hs >> 8); + hs >>= 1; + hs |= 0xFFFF; + if (hs > (1 << 24)) + hs >>= 1; + hs++; + hs += (1 << 16); + + const UInt32 kBlockSizeMax = (UInt32)0 - (UInt32)(1 << 16); + UInt64 blockSize = (UInt64)dict + (1 << 16) + + (multiThread ? (1 << 20) : 0); + blockSize += (blockSize >> (blockSize < ((UInt32)1 << 30) ? 1 : 2)); + if (blockSize >= kBlockSizeMax) + blockSize = kBlockSizeMax; + + UInt64 son = (UInt64)dict; + if (btMode) + son *= 2; + const UInt64 v = (hs + son) * 4 + blockSize + + (1 << 20) + (multiThread ? (6 << 20) : 0); + + // printf("\nGetLZMAUsage = %d\n", (UInt32)(v >> 20)); + // printf("\nblockSize = %d\n", (UInt32)(blockSize >> 20)); + return v; +} + + +UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool totalBench) +{ + const size_t kBufferSize = (size_t)dictionary + kAdditionalSize; + const UInt64 kCompressedBufferSize = GetBenchCompressedSize(kBufferSize); // / 2; + if (level < 0) + level = 5; + const int algo = (level < 5 ? 0 : 1); + const int btMode = (algo == 0 ? 0 : 1); + + UInt32 numBigThreads = numThreads; + const bool lzmaMt = (totalBench || (numThreads > 1 && btMode)); + if (btMode) + { + if (!totalBench && lzmaMt) + numBigThreads /= 2; + } + return ((UInt64)kBufferSize + kCompressedBufferSize + + GetLZMAUsage(lzmaMt, btMode, dictionary) + (2 << 20)) * numBigThreads; +} + +static UInt64 GetBenchMemoryUsage_Hash(UInt32 numThreads, UInt64 dictionary) +{ + // dictionary += (dictionary >> 9); // for page tables (virtual memory) + return (UInt64)(dictionary + (1 << 15)) * numThreads + (2 << 20); +} + + +// ---------- CRC and HASH ---------- + +struct CCrcInfo_Base +{ + CMidAlignedBuffer Buffer; + const Byte *Data; + size_t Size; + bool CreateLocalBuf; + UInt32 CheckSum_Res; + + CCrcInfo_Base(): CreateLocalBuf(true), CheckSum_Res(0) {} + + HRESULT Generate(const Byte *data, size_t size); + HRESULT CrcProcess(UInt64 numIterations, + const UInt32 *checkSum, IHasher *hf, + IBenchPrintCallback *callback); +}; + + +// for debug: define it to test hash calling with unaligned data +// #define Z7_BENCH_HASH_ALIGN_BUF_OFFSET 3 + +HRESULT CCrcInfo_Base::Generate(const Byte *data, size_t size) +{ + Size = size; + Data = data; + if (!data || CreateLocalBuf) + { + Byte *buf; + const size_t size2 = (size + k_RandBuf_AlignMask) & ~(size_t)k_RandBuf_AlignMask; + if (size2 < size) + return E_OUTOFMEMORY; +#ifdef Z7_BENCH_HASH_ALIGN_BUF_OFFSET + ALLOC_WITH_HRESULT(&Buffer, size2 + Z7_BENCH_HASH_ALIGN_BUF_OFFSET) + buf = Buffer + Z7_BENCH_HASH_ALIGN_BUF_OFFSET; +#else + ALLOC_WITH_HRESULT(&Buffer, size2) + buf = Buffer; +#endif + Data = buf; + if (!data) + RandGen_BufAfterPad(buf, size); + else if (size != 0) // (CreateLocalBuf == true) + memcpy(buf, data, size); + } + return S_OK; +} + + +HRESULT CCrcInfo_Base::CrcProcess(UInt64 numIterations, + const UInt32 *checkSum, IHasher *hf, + IBenchPrintCallback *callback) +{ + MY_ALIGN(16) + UInt32 hash32[64 / 4]; + memset(hash32, 0, sizeof(hash32)); + + CheckSum_Res = 0; + + const UInt32 hashSize = hf->GetDigestSize(); + if (hashSize > sizeof(hash32)) + return S_FALSE; + + const Byte *buf = Data; + const size_t size = Size; + UInt32 checkSum_Prev = 0; + + UInt64 prev = 0; + UInt64 cur = 0; + + do + { + hf->Init(); + size_t pos = 0; + do + { + const size_t rem = size - pos; + const UInt32 kStep = ((UInt32)1 << 31); + const UInt32 curSize = (rem < kStep) ? (UInt32)rem : kStep; + hf->Update(buf + pos, curSize); + pos += curSize; + } + while (pos != size); + + hf->Final((Byte *)(void *)hash32); + UInt32 sum = 0; + for (UInt32 j = 0; j < hashSize; j += 4) + { + sum = rotlFixed(sum, 11); + sum += GetUi32((const Byte *)(const void *)hash32 + j); + } + if (checkSum) + { + if (sum != *checkSum) + return S_FALSE; + } + else + { + checkSum_Prev = sum; + checkSum = &checkSum_Prev; + } + if (callback) + { + cur += size; + if (cur - prev >= ((UInt32)1 << 30)) + { + prev = cur; + RINOK(callback->CheckBreak()) + } + } + } + while (--numIterations); + + CheckSum_Res = checkSum_Prev; + return S_OK; +} + +extern +UInt32 g_BenchCpuFreqTemp; // we need non-static variavble to disable compiler optimization +UInt32 g_BenchCpuFreqTemp = 1; + +#define YY1 sum += val; sum ^= val; +#define YY3 YY1 YY1 YY1 YY1 +#define YY5 YY3 YY3 YY3 YY3 +#define YY7 YY5 YY5 YY5 YY5 +static const UInt32 kNumFreqCommands = 128; + +EXTERN_C_BEGIN + +static UInt32 CountCpuFreq(UInt32 sum, UInt32 num, UInt32 val) +{ + for (UInt32 i = 0; i < num; i++) + { + YY7 + } + return sum; +} + +EXTERN_C_END + + +#ifndef Z7_ST + +struct CBaseThreadInfo +{ + NWindows::CThread Thread; + IBenchPrintCallback *Callback; + HRESULT CallbackRes; + + WRes Wait_If_Created() + { + if (!Thread.IsCreated()) + return 0; + return Thread.Wait_Close(); + } +}; + +struct CFreqInfo: public CBaseThreadInfo +{ + UInt32 ValRes; + UInt32 Size; + UInt64 NumIterations; +}; + +static THREAD_FUNC_DECL FreqThreadFunction(void *param) +{ + CFreqInfo *p = (CFreqInfo *)param; + + UInt32 sum = g_BenchCpuFreqTemp; + for (UInt64 k = p->NumIterations; k > 0; k--) + { + if (p->Callback) + { + p->CallbackRes = p->Callback->CheckBreak(); + if (p->CallbackRes != S_OK) + break; + } + sum = CountCpuFreq(sum, p->Size, g_BenchCpuFreqTemp); + } + p->ValRes = sum; + return THREAD_FUNC_RET_ZERO; +} + +struct CFreqThreads +{ + CFreqInfo *Items; + UInt32 NumThreads; + + CFreqThreads(): Items(NULL), NumThreads(0) {} + + WRes WaitAll() + { + WRes wres = 0; + for (UInt32 i = 0; i < NumThreads; i++) + { + WRes wres2 = Items[i].Wait_If_Created(); + if (wres == 0 && wres2 != 0) + wres = wres2; + } + NumThreads = 0; + return wres; + } + + ~CFreqThreads() + { + WaitAll(); + delete []Items; + } +}; + + +static THREAD_FUNC_DECL CrcThreadFunction(void *param); + +struct CCrcInfo: public CBaseThreadInfo +{ + const Byte *Data; + size_t Size; + UInt64 NumIterations; + bool CheckSumDefined; + UInt32 CheckSum; + CMyComPtr Hasher; + HRESULT Res; + UInt32 CheckSum_Res; + + #ifndef Z7_ST + NSynchronization::CManualResetEvent ReadyEvent; + UInt32 ThreadIndex; + CBenchSyncCommon *Common; + CAffinityMode AffinityMode; + #endif + + // we want to call CCrcInfo_Base::Buffer.Free() in main thread. + // so we uses non-local CCrcInfo_Base. + CCrcInfo_Base crcib; + + HRESULT CreateThread() + { + WRes res = 0; + if (!ReadyEvent.IsCreated()) + res = ReadyEvent.Create(); + if (res == 0) + res = AffinityMode.CreateThread_WithAffinity(Thread, CrcThreadFunction, this, + ThreadIndex); + return HRESULT_FROM_WIN32(res); + } + + #ifdef USE_ALLOCA + size_t AllocaSize; + #endif + + void Process(); + + CCrcInfo(): Res(E_FAIL) {} +}; + +static const bool k_Crc_CreateLocalBuf_For_File = true; // for total BW test +// static const bool k_Crc_CreateLocalBuf_For_File = false; // for shared memory read test + +void CCrcInfo::Process() +{ + crcib.CreateLocalBuf = k_Crc_CreateLocalBuf_For_File; + // we can use additional Generate() passes to reduce some time effects for new page allocation + // for (unsigned y = 0; y < 10; y++) + Res = crcib.Generate(Data, Size); + + // if (Common) + { + WRes wres = ReadyEvent.Set(); + if (wres != 0) + { + if (Res == 0) + Res = HRESULT_FROM_WIN32(wres); + return; + } + if (Res != 0) + return; + + wres = Common->StartEvent.Lock(); + + if (wres != 0) + { + Res = HRESULT_FROM_WIN32(wres); + return; + } + if (Common->ExitMode) + return; + } + + Res = crcib.CrcProcess(NumIterations, + CheckSumDefined ? &CheckSum : NULL, Hasher, + Callback); + CheckSum_Res = crcib.CheckSum_Res; + /* + We don't want to include the time of slow CCrcInfo_Base::Buffer.Free() + to time of benchmark. So we don't free Buffer here + */ + // crcib.Buffer.Free(); +} + + +static THREAD_FUNC_DECL CrcThreadFunction(void *param) +{ + CCrcInfo *p = (CCrcInfo *)param; + + #ifdef USE_ALLOCA + alloca(p->AllocaSize); + #endif + p->Process(); + return THREAD_FUNC_RET_ZERO; +} + + +struct CCrcThreads +{ + CCrcInfo *Items; + unsigned NumThreads; + CBenchSyncCommon Common; + bool NeedClose; + + CCrcThreads(): Items(NULL), NumThreads(0), NeedClose(false) {} + + WRes StartAndWait(bool exitMode = false); + + ~CCrcThreads() + { + StartAndWait(true); + delete []Items; + } +}; + + +WRes CCrcThreads::StartAndWait(bool exitMode) +{ + if (!NeedClose) + return 0; + + Common.ExitMode = exitMode; + WRes wres = Common.StartEvent.Set(); + + for (unsigned i = 0; i < NumThreads; i++) + { + WRes wres2 = Items[i].Wait_If_Created(); + if (wres == 0 && wres2 != 0) + wres = wres2; + } + NumThreads = 0; + NeedClose = false; + return wres; +} + +#endif + + +/* +static UInt32 CrcCalc1(const Byte *buf, size_t size) +{ + UInt32 crc = CRC_INIT_VAL; + for (size_t i = 0; i < size; i++) + crc = CRC_UPDATE_BYTE(crc, buf[i]); + return CRC_GET_DIGEST(crc); +} +*/ + +/* +static UInt32 RandGenCrc(Byte *buf, size_t size, CBaseRandomGenerator &RG) +{ + RandGen(buf, size, RG); + return CrcCalc1(buf, size); +} +*/ + +static bool CrcInternalTest() +{ + CAlignedBuffer buffer; + const size_t kBufSize = 1 << 11; + const size_t kCheckSize = 1 << 6; + buffer.Alloc(kBufSize); + if (!buffer.IsAllocated()) + return false; + Byte *buf = (Byte *)buffer; + RandGen_BufAfterPad(buf, kBufSize); + UInt32 sum = 0; + for (size_t i = 0; i < kBufSize - kCheckSize * 2; i += kCheckSize - 1) + for (size_t j = 0; j < kCheckSize; j++) + { + sum = rotlFixed(sum, 11); + sum += CrcCalc(buf + i + j, j); + } + return sum == 0x28462c7c; +} + +struct CBenchMethod +{ + unsigned Weight; + unsigned DictBits; + Int32 EncComplex; + Int32 DecComplexCompr; + Int32 DecComplexUnc; + const char *Name; + // unsigned KeySize; +}; + +// #define USE_SW_CMPLX + +#ifdef USE_SW_CMPLX +#define CMPLX(x) ((x) * 1000) +#else +#define CMPLX(x) (x) +#endif + +static const CBenchMethod g_Bench[] = +{ + // { 40, 17, 357, 145, 20, "LZMA:x1" }, + // { 20, 18, 360, 145, 20, "LZMA2:x1:mt2" }, + + { 20, 18, 360, 145, 20, "LZMA:x1" }, + { 20, 22, 600, 145, 20, "LZMA:x3" }, + + { 80, 24, 1220, 145, 20, "LZMA:x5:mt1" }, + { 80, 24, 1220, 145, 20, "LZMA:x5:mt2" }, + + { 10, 16, 124, 40, 14, "Deflate:x1" }, + { 20, 16, 376, 40, 14, "Deflate:x5" }, + { 10, 16, 1082, 40, 14, "Deflate:x7" }, + { 10, 17, 422, 40, 14, "Deflate64:x5" }, + + { 10, 15, 590, 69, 69, "BZip2:x1" }, + { 20, 19, 815, 122, 122, "BZip2:x5" }, + { 10, 19, 815, 122, 122, "BZip2:x5:mt2" }, + { 10, 19, 2530, 122, 122, "BZip2:x7" }, + + // { 10, 18, 1010, 0, 1150, "PPMDZip:x1" }, + { 10, 18, 1010, 0, 1150, "PPMD:x1" }, + // { 10, 22, 1655, 0, 1830, "PPMDZip:x5" }, + { 10, 22, 1655, 0, 1830, "PPMD:x5" }, + + // { 2, 0, -16, 0, -16, "Swap2" }, + { 2, 0, -16, 0, -16, "Swap4" }, + + // { 2, 0, 3, 0, 4, "Delta:1" }, + // { 2, 0, 3, 0, 4, "Delta:2" }, + // { 2, 0, 3, 0, 4, "Delta:3" }, + { 2, 0, 3, 0, 4, "Delta:4" }, + // { 2, 0, 3, 0, 4, "Delta:8" }, + // { 2, 0, 3, 0, 4, "Delta:32" }, + + { 2, 0, 2, 0, 2, "BCJ" }, + { 2, 0, 1, 0, 1, "ARM64" }, + { 2, 0, 1, 0, 1, "RISCV" }, + + // { 10, 0, 18, 0, 18, "AES128CBC:1" }, + // { 10, 0, 21, 0, 21, "AES192CBC:1" }, + { 10, 0, 24, 0, 24, "AES256CBC:1" }, + + // { 10, 0, 18, 0, 18, "AES128CTR:1" }, + // { 10, 0, 21, 0, 21, "AES192CTR:1" }, + // { 10, 0, 24, 0, 24, "AES256CTR:1" }, + // { 2, 0, CMPLX(6), 0, CMPLX(1), "AES128CBC:2" }, + // { 2, 0, CMPLX(7), 0, CMPLX(1), "AES192CBC:2" }, + { 2, 0, CMPLX(8), 0, CMPLX(1), "AES256CBC:2" }, + + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES128CTR:2" }, + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES192CTR:2" }, + // { 2, 0, CMPLX(1), 0, CMPLX(1), "AES256CTR:2" }, + + // { 1, 0, CMPLX(6), 0, -2, "AES128CBC:3" }, + // { 1, 0, CMPLX(7), 0, -2, "AES192CBC:3" }, + { 1, 0, CMPLX(8), 0, -2, "AES256CBC:3" } + + // { 1, 0, CMPLX(1), 0, -2, "AES128CTR:3" }, + // { 1, 0, CMPLX(1), 0, -2, "AES192CTR:3" }, + // { 1, 0, CMPLX(1), 0, -2, "AES256CTR:3" }, +}; + +struct CBenchHash +{ + unsigned Weight; + UInt32 Complex; + UInt32 CheckSum; + const char *Name; +}; + +// #define ARM_CRC_MUL 100 +#define ARM_CRC_MUL 1 + +#define k_Hash_Complex_Mult 256 + +static const CBenchHash g_Hash[] = +{ + { 20, 256, 0x21e207bb, "CRC32:12" } , + { 2, 128 *ARM_CRC_MUL, 0x21e207bb, "CRC32:32" }, + { 2, 64 *ARM_CRC_MUL, 0x21e207bb, "CRC32:64" }, + { 10, 256, 0x41b901d1, "CRC64" }, + { 10, 64, 0x43eac94f, "XXH64" }, + + { 10, 5100, 0x7913ba03, "SHA256:1" }, + { 2, CMPLX((32 * 4 + 1) * 4 + 4), 0x7913ba03, "SHA256:2" }, + + { 10, 2340, 0xff769021, "SHA1:1" }, + { 2, CMPLX((20 * 6 + 1) * 4 + 4), 0xff769021, "SHA1:2" }, + + { 2, 4096, 0x85189d02, "BLAKE2sp:1" }, + { 2, 1024, 0x85189d02, "BLAKE2sp:2" }, // sse2-way4-fast + { 2, 512, 0x85189d02, "BLAKE2sp:3" } // avx2-way8-fast +#if 0 + , { 2, 2048, 0x85189d02, "BLAKE2sp:4" } // sse2-way1 + , { 2, 1024, 0x85189d02, "BLAKE2sp:5" } // sse2-way2 + , { 2, 1024, 0x85189d02, "BLAKE2sp:6" } // avx2-way2 + , { 2, 1024, 0x85189d02, "BLAKE2sp:7" } // avx2-way4 +#endif +}; + +static void PrintNumber(IBenchPrintCallback &f, UInt64 value, unsigned size) +{ + char s[128]; + unsigned startPos = (unsigned)sizeof(s) - 32; + memset(s, ' ', startPos); + ConvertUInt64ToString(value, s + startPos); + // if (withSpace) + { + startPos--; + size++; + } + unsigned len = (unsigned)strlen(s + startPos); + if (size > len) + { + size -= len; + if (startPos < size) + startPos = 0; + else + startPos -= size; + } + f.Print(s + startPos); +} + +static const unsigned kFieldSize_Name = 12; +static const unsigned kFieldSize_SmallName = 4; +static const unsigned kFieldSize_Speed = 9; +static const unsigned kFieldSize_Usage = 5; +static const unsigned kFieldSize_RU = 6; +static const unsigned kFieldSize_Rating = 6; +static const unsigned kFieldSize_EU = 5; +static const unsigned kFieldSize_Effec = 5; +static const unsigned kFieldSize_CrcSpeed = 8; + + +static const unsigned kFieldSize_TotalSize = 4 + kFieldSize_Speed + kFieldSize_Usage + kFieldSize_RU + kFieldSize_Rating; +static const unsigned kFieldSize_EUAndEffec = 2 + kFieldSize_EU + kFieldSize_Effec; + + +static void PrintRating(IBenchPrintCallback &f, UInt64 rating, unsigned size) +{ + PrintNumber(f, (rating + 500000) / 1000000, size); +} + + +static void PrintPercents(IBenchPrintCallback &f, UInt64 val, UInt64 divider, unsigned size) +{ + UInt64 v = 0; + if (divider != 0) + v = (val * 100 + divider / 2) / divider; + PrintNumber(f, v, size); +} + +static void PrintChars(IBenchPrintCallback &f, char c, unsigned size) +{ + char s[256]; + memset(s, (Byte)c, size); + s[size] = 0; + f.Print(s); +} + +static void PrintSpaces(IBenchPrintCallback &f, unsigned size) +{ + PrintChars(f, ' ', size); +} + +static void PrintUsage(IBenchPrintCallback &f, UInt64 usage, unsigned size) +{ + PrintNumber(f, Benchmark_GetUsage_Percents(usage), size); +} + +static void PrintResults(IBenchPrintCallback &f, UInt64 usage, UInt64 rpu, UInt64 rating, bool showFreq, UInt64 cpuFreq) +{ + PrintUsage(f, usage, kFieldSize_Usage); + PrintRating(f, rpu, kFieldSize_RU); + PrintRating(f, rating, kFieldSize_Rating); + if (showFreq) + { + if (cpuFreq == 0) + PrintSpaces(f, kFieldSize_EUAndEffec); + else + { + PrintPercents(f, rating, cpuFreq * usage / kBenchmarkUsageMult, kFieldSize_EU); + PrintPercents(f, rating, cpuFreq, kFieldSize_Effec); + } + } +} + + +void CTotalBenchRes::Generate_From_BenchInfo(const CBenchInfo &info) +{ + Speed = info.GetUnpackSizeSpeed(); + Usage = info.GetUsage(); + RPU = info.GetRatingPerUsage(Rating); +} + +void CTotalBenchRes::Mult_For_Weight(unsigned weight) +{ + NumIterations2 *= weight; + RPU *= weight; + Rating *= weight; + Usage *= weight; + Speed *= weight; +} + +void CTotalBenchRes::Update_With_Res(const CTotalBenchRes &r) +{ + Rating += r.Rating; + Usage += r.Usage; + RPU += r.RPU; + Speed += r.Speed; + // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); + NumIterations2 += r.NumIterations2; +} + +static void PrintResults(IBenchPrintCallback *f, + const CBenchInfo &info, + unsigned weight, + UInt64 rating, + bool showFreq, UInt64 cpuFreq, + CTotalBenchRes *res) +{ + CTotalBenchRes t; + t.Rating = rating; + t.NumIterations2 = 1; + t.Generate_From_BenchInfo(info); + + if (f) + { + if (t.Speed != 0) + PrintNumber(*f, t.Speed / 1024, kFieldSize_Speed); + else + PrintSpaces(*f, 1 + kFieldSize_Speed); + } + if (f) + { + PrintResults(*f, t.Usage, t.RPU, rating, showFreq, cpuFreq); + } + + if (res) + { + // res->NumIterations1++; + t.Mult_For_Weight(weight); + res->Update_With_Res(t); + } +} + +static void PrintTotals(IBenchPrintCallback &f, + bool showFreq, UInt64 cpuFreq, bool showSpeed, const CTotalBenchRes &res) +{ + const UInt64 numIterations2 = res.NumIterations2 ? res.NumIterations2 : 1; + const UInt64 speed = res.Speed / numIterations2; + if (showSpeed && speed != 0) + PrintNumber(f, speed / 1024, kFieldSize_Speed); + else + PrintSpaces(f, 1 + kFieldSize_Speed); + + // PrintSpaces(f, 1 + kFieldSize_Speed); + // UInt64 numIterations1 = res.NumIterations1; if (numIterations1 == 0) numIterations1 = 1; + PrintResults(f, res.Usage / numIterations2, res.RPU / numIterations2, res.Rating / numIterations2, showFreq, cpuFreq); +} + + +static void PrintHex(AString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} + +AString GetProcessThreadsInfo(const NSystem::CProcessAffinity &ti) +{ + AString s; + // s.Add_UInt32(ti.numProcessThreads); + unsigned numSysThreads = ti.GetNumSystemThreads(); + if (ti.GetNumProcessThreads() != numSysThreads) + { + // if (ti.numProcessThreads != ti.numSysThreads) + { + s += " / "; + s.Add_UInt32(numSysThreads); + } + s += " : "; + #ifdef _WIN32 + PrintHex(s, ti.processAffinityMask); + s += " / "; + PrintHex(s, ti.systemAffinityMask); + #else + unsigned i = (numSysThreads + 3) & ~(unsigned)3; + if (i == 0) + i = 4; + for (; i >= 4; ) + { + i -= 4; + unsigned val = 0; + for (unsigned k = 0; k < 4; k++) + { + const unsigned bit = (ti.IsCpuSet(i + k) ? 1 : 0); + val += (bit << k); + } + PrintHex(s, val); + } + #endif + } + return s; +} + + +#ifdef Z7_LARGE_PAGES + +#ifdef _WIN32 +extern bool g_LargePagesMode; +extern "C" +{ + extern SIZE_T g_LargePageSize; +} +#endif + +void Add_LargePages_String(AString &s) +{ + #ifdef _WIN32 + if (g_LargePagesMode || g_LargePageSize != 0) + { + s.Add_OptSpaced("(LP-"); + PrintSize_KMGT_Or_Hex(s, g_LargePageSize); + #ifdef MY_CPU_X86_OR_AMD64 + if (CPU_IsSupported_PageGB()) + s += "-1G"; + #endif + if (!g_LargePagesMode) + s += "-NA"; + s += ")"; + } + #else + s += ""; + #endif +} + +#endif + + + +static void PrintRequirements(IBenchPrintCallback &f, const char *sizeString, + bool size_Defined, UInt64 size, const char *threadsString, UInt32 numThreads) +{ + f.Print("RAM "); + f.Print(sizeString); + if (size_Defined) + PrintNumber(f, (size >> 20), 6); + else + f.Print(" ?"); + f.Print(" MB"); + + #ifdef Z7_LARGE_PAGES + { + AString s; + Add_LargePages_String(s); + f.Print(s); + } + #endif + + f.Print(", # "); + f.Print(threadsString); + PrintNumber(f, numThreads, 3); +} + + + +struct CBenchCallbackToPrint Z7_final: public IBenchCallback +{ + bool NeedPrint; + bool Use2Columns; + bool ShowFreq; + unsigned NameFieldSize; + + unsigned EncodeWeight; + unsigned DecodeWeight; + + UInt64 CpuFreq; + UInt64 DictSize; + + IBenchPrintCallback *_file; + CBenchProps BenchProps; + CTotalBenchRes EncodeRes; + CTotalBenchRes DecodeRes; + + CBenchInfo BenchInfo_Results[2]; + + CBenchCallbackToPrint(): + NeedPrint(true), + Use2Columns(false), + ShowFreq(false), + NameFieldSize(0), + EncodeWeight(1), + DecodeWeight(1), + CpuFreq(0) + {} + + void Init() { EncodeRes.Init(); DecodeRes.Init(); } + void Print(const char *s); + void NewLine(); + + HRESULT SetFreq(bool showFreq, UInt64 cpuFreq); + HRESULT SetEncodeResult(const CBenchInfo &info, bool final) Z7_override; + HRESULT SetDecodeResult(const CBenchInfo &info, bool final) Z7_override; +}; + +HRESULT CBenchCallbackToPrint::SetFreq(bool showFreq, UInt64 cpuFreq) +{ + ShowFreq = showFreq; + CpuFreq = cpuFreq; + return S_OK; +} + +HRESULT CBenchCallbackToPrint::SetEncodeResult(const CBenchInfo &info, bool final) +{ + RINOK(_file->CheckBreak()) + if (final) + BenchInfo_Results[0] = info; + if (final) + if (NeedPrint) + { + const UInt64 rating = BenchProps.GetRating_Enc(DictSize, info.GlobalTime, info.GlobalFreq, info.UnpackSize * info.NumIterations); + PrintResults(_file, info, + EncodeWeight, rating, + ShowFreq, CpuFreq, &EncodeRes); + if (!Use2Columns) + _file->NewLine(); + } + return S_OK; +} + +static const char * const kSep = " | "; + +HRESULT CBenchCallbackToPrint::SetDecodeResult(const CBenchInfo &info, bool final) +{ + RINOK(_file->CheckBreak()) + if (final) + BenchInfo_Results[1] = info; + if (final) + if (NeedPrint) + { + const UInt64 rating = BenchProps.GetRating_Dec(info.GlobalTime, info.GlobalFreq, info.UnpackSize, info.PackSize, info.NumIterations); + if (Use2Columns) + _file->Print(kSep); + else + PrintSpaces(*_file, NameFieldSize); + CBenchInfo info2 = info; + info2.UnpackSize *= info2.NumIterations; + info2.PackSize *= info2.NumIterations; + info2.NumIterations = 1; + PrintResults(_file, info2, + DecodeWeight, rating, + ShowFreq, CpuFreq, &DecodeRes); + } + return S_OK; +} + +void CBenchCallbackToPrint::Print(const char *s) +{ + _file->Print(s); +} + +void CBenchCallbackToPrint::NewLine() +{ + _file->NewLine(); +} + +static void PrintLeft(IBenchPrintCallback &f, const char *s, unsigned size) +{ + f.Print(s); + int numSpaces = (int)size - (int)MyStringLen(s); + if (numSpaces > 0) + PrintSpaces(f, (unsigned)numSpaces); +} + +static void PrintRight(IBenchPrintCallback &f, const char *s, unsigned size) +{ + int numSpaces = (int)size - (int)MyStringLen(s); + if (numSpaces > 0) + PrintSpaces(f, (unsigned)numSpaces); + f.Print(s); +} + + +static bool DoesWildcardMatchName_NoCase(const AString &mask, const char *name) +{ + UString wildc = GetUnicodeString(mask); + UString bname = GetUnicodeString(name); + wildc.MakeLower_Ascii(); + bname.MakeLower_Ascii(); + return DoesWildcardMatchName(wildc, bname); +} + + +static HRESULT TotalBench( + DECL_EXTERNAL_CODECS_LOC_VARS + const COneMethodInfo &methodMask, + UInt64 complexInCommands, + #ifndef Z7_ST + UInt32 numThreads, + const CAffinityMode *affinityMode, + #endif + bool forceUnpackSize, + size_t unpackSize, + const Byte *fileData, + IBenchPrintCallback *printCallback, CBenchCallbackToPrint *callback) +{ + for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_Bench); i++) + { + const CBenchMethod &bench = g_Bench[i]; + if (!DoesWildcardMatchName_NoCase(methodMask.MethodName, bench.Name)) + continue; + PrintLeft(*callback->_file, bench.Name, kFieldSize_Name); + { + unsigned keySize = 32; + if (IsString1PrefixedByString2(bench.Name, "AES128")) keySize = 16; + else if (IsString1PrefixedByString2(bench.Name, "AES192")) keySize = 24; + callback->BenchProps.KeySize = keySize; + } + callback->BenchProps.DecComplexUnc = bench.DecComplexUnc; + callback->BenchProps.DecComplexCompr = bench.DecComplexCompr; + callback->BenchProps.EncComplex = bench.EncComplex; + + COneMethodInfo method; + NCOM::CPropVariant propVariant; + propVariant = bench.Name; + RINOK(method.ParseMethodFromPROPVARIANT(UString(), propVariant)) + + size_t unpackSize2 = unpackSize; + if (!forceUnpackSize && bench.DictBits == 0) + unpackSize2 = kFilterUnpackSize; + + callback->EncodeWeight = bench.Weight; + callback->DecodeWeight = bench.Weight; + + const HRESULT res = MethodBench( + EXTERNAL_CODECS_LOC_VARS + complexInCommands, + #ifndef Z7_ST + false, numThreads, affinityMode, + #endif + method, + unpackSize2, fileData, + bench.DictBits, + printCallback, callback, &callback->BenchProps); + + if (res == E_NOTIMPL) + { + // callback->Print(" ---"); + // we need additional empty line as line for decompression results + if (!callback->Use2Columns) + callback->NewLine(); + } + else + { + RINOK(res) + } + + callback->NewLine(); + } + return S_OK; +} + + +struct CFreqBench +{ + // in: + UInt64 complexInCommands; + UInt32 numThreads; + bool showFreq; + UInt64 specifiedFreq; + + // out: + UInt64 CpuFreqRes; + UInt64 UsageRes; + UInt32 res; + + CFreqBench() + {} + + HRESULT FreqBench(IBenchPrintCallback *_file + #ifndef Z7_ST + , const CAffinityMode *affinityMode + #endif + ); +}; + + +HRESULT CFreqBench::FreqBench(IBenchPrintCallback *_file + #ifndef Z7_ST + , const CAffinityMode *affinityMode + #endif + ) +{ + res = 0; + CpuFreqRes = 0; + UsageRes = 0; + + if (numThreads == 0) + numThreads = 1; + + #ifdef Z7_ST + numThreads = 1; + #endif + + const UInt32 complexity = kNumFreqCommands; + UInt64 numIterations = complexInCommands / complexity; + UInt32 numIterations2 = 1 << 30; + if (numIterations > numIterations2) + numIterations /= numIterations2; + else + { + numIterations2 = (UInt32)numIterations; + numIterations = 1; + } + + CBenchInfoCalc progressInfoSpec; + + #ifndef Z7_ST + + bool mtMode = (numThreads > 1) || affinityMode->NeedAffinity(); + + if (mtMode) + { + CFreqThreads threads; + threads.Items = new CFreqInfo[numThreads]; + UInt32 i; + for (i = 0; i < numThreads; i++) + { + CFreqInfo &info = threads.Items[i]; + info.Callback = _file; + info.CallbackRes = S_OK; + info.NumIterations = numIterations; + info.Size = numIterations2; + } + progressInfoSpec.SetStartTime(); + for (i = 0; i < numThreads; i++) + { + // Sleep(10); + CFreqInfo &info = threads.Items[i]; + WRes wres = affinityMode->CreateThread_WithAffinity(info.Thread, FreqThreadFunction, &info, i); + if (info.Thread.IsCreated()) + threads.NumThreads++; + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + } + WRes wres = threads.WaitAll(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + for (i = 0; i < numThreads; i++) + { + RINOK(threads.Items[i].CallbackRes) + } + } + else + #endif + { + progressInfoSpec.SetStartTime(); + UInt32 sum = g_BenchCpuFreqTemp; + UInt64 k = numIterations; + do + { + sum = CountCpuFreq(sum, numIterations2, g_BenchCpuFreqTemp); + if (_file) + { + RINOK(_file->CheckBreak()) + } + } + while (--k); + res += sum; + } + + if (res == 0x12345678) + if (_file) + { + RINOK(_file->CheckBreak()) + } + + CBenchInfo info; + progressInfoSpec.SetFinishTime(info); + + info.UnpackSize = 0; + info.PackSize = 0; + info.NumIterations = 1; + + const UInt64 numCommands = (UInt64)numIterations * numIterations2 * numThreads * complexity; + const UInt64 rating = info.GetSpeed(numCommands); + CpuFreqRes = rating / numThreads; + UsageRes = info.GetUsage(); + + if (_file) + { + PrintResults(_file, info, + 0, // weight + rating, + showFreq, showFreq ? (specifiedFreq != 0 ? specifiedFreq : CpuFreqRes) : 0, NULL); + RINOK(_file->CheckBreak()) + } + + return S_OK; +} + + + +static HRESULT CrcBench( + DECL_EXTERNAL_CODECS_LOC_VARS + UInt64 complexInCommands, + UInt32 numThreads, + const size_t bufferSize, + const Byte *fileData, + + UInt64 &speed, + UInt64 &usage, + + UInt32 complexity, unsigned benchWeight, + const UInt32 *checkSum, + const COneMethodInfo &method, + IBenchPrintCallback *_file, + #ifndef Z7_ST + const CAffinityMode *affinityMode, + #endif + bool showRating, + CTotalBenchRes *encodeRes, + bool showFreq, UInt64 cpuFreq) +{ + if (numThreads == 0) + numThreads = 1; + + #ifdef Z7_ST + numThreads = 1; + #endif + + const AString &methodName = method.MethodName; + // methodName.RemoveChar(L'-'); + CMethodId hashID; + if (!FindHashMethod( + EXTERNAL_CODECS_LOC_VARS + methodName, hashID)) + return E_NOTIMPL; + + /* + // if will generate random data in each thread, instead of global data + CMidAlignedBuffer buffer; + if (!fileData) + { + ALLOC_WITH_HRESULT(&buffer, bufferSize) + RandGen(buffer, bufferSize); + fileData = buffer; + } + */ + + const size_t bsize = (bufferSize == 0 ? 1 : bufferSize); + UInt64 numIterations = complexInCommands * k_Hash_Complex_Mult / complexity / bsize; + if (numIterations == 0) + numIterations = 1; + + CBenchInfoCalc progressInfoSpec; + CBenchInfo info; + + #ifndef Z7_ST + bool mtEncMode = (numThreads > 1) || affinityMode->NeedAffinity(); + + if (mtEncMode) + { + CCrcThreads threads; + threads.Items = new CCrcInfo[numThreads]; + { + WRes wres = threads.Common.StartEvent.Create(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + threads.NeedClose = true; + } + + UInt32 i; + for (i = 0; i < numThreads; i++) + { + CCrcInfo &ci = threads.Items[i]; + AString name; + RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, ci.Hasher)) + if (!ci.Hasher) + return E_NOTIMPL; + CMyComPtr scp; + ci.Hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); + if (scp) + { + RINOK(method.SetCoderProps(scp)) + } + + ci.Callback = _file; + ci.Data = fileData; + ci.NumIterations = numIterations; + ci.Size = bufferSize; + ci.CheckSumDefined = false; + if (checkSum) + { + ci.CheckSum = *checkSum; + ci.CheckSumDefined = true; + } + + #ifdef USE_ALLOCA + ci.AllocaSize = BENCH_ALLOCA_VALUE(i); + #endif + } + + for (i = 0; i < numThreads; i++) + { + CCrcInfo &ci = threads.Items[i]; + ci.ThreadIndex = i; + ci.Common = &threads.Common; + ci.AffinityMode = *affinityMode; + HRESULT hres = ci.CreateThread(); + if (ci.Thread.IsCreated()) + threads.NumThreads++; + if (hres != 0) + return hres; + } + + for (i = 0; i < numThreads; i++) + { + CCrcInfo &ci = threads.Items[i]; + WRes wres = ci.ReadyEvent.Lock(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + RINOK(ci.Res) + } + + progressInfoSpec.SetStartTime(); + + WRes wres = threads.StartAndWait(); + if (wres != 0) + return HRESULT_FROM_WIN32(wres); + + progressInfoSpec.SetFinishTime(info); + + for (i = 0; i < numThreads; i++) + { + RINOK(threads.Items[i].Res) + if (i != 0) + if (threads.Items[i].CheckSum_Res != + threads.Items[i - 1].CheckSum_Res) + return S_FALSE; + } + } + else + #endif + { + CMyComPtr hasher; + AString name; + RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS hashID, name, hasher)) + if (!hasher) + return E_NOTIMPL; + CMyComPtr scp; + hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); + if (scp) + { + RINOK(method.SetCoderProps(scp)) + } + CCrcInfo_Base crcib; + crcib.CreateLocalBuf = false; + RINOK(crcib.Generate(fileData, bufferSize)) + progressInfoSpec.SetStartTime(); + RINOK(crcib.CrcProcess(numIterations, checkSum, hasher, _file)) + progressInfoSpec.SetFinishTime(info); + } + + + UInt64 unpSize = numIterations * bufferSize; + UInt64 unpSizeThreads = unpSize * numThreads; + info.UnpackSize = unpSizeThreads; + info.PackSize = unpSizeThreads; + info.NumIterations = 1; + + if (_file) + { + if (showRating) + { + UInt64 unpSizeThreads2 = unpSizeThreads; + if (unpSizeThreads2 == 0) + unpSizeThreads2 = numIterations * 1 * numThreads; + const UInt64 numCommands = unpSizeThreads2 * complexity / 256; + const UInt64 rating = info.GetSpeed(numCommands); + PrintResults(_file, info, + benchWeight, rating, + showFreq, cpuFreq, encodeRes); + } + RINOK(_file->CheckBreak()) + } + + speed = info.GetSpeed(unpSizeThreads); + usage = info.GetUsage(); + + return S_OK; +} + + + +static HRESULT TotalBench_Hash( + DECL_EXTERNAL_CODECS_LOC_VARS + const COneMethodInfo &methodMask, + UInt64 complexInCommands, + UInt32 numThreads, + size_t bufSize, + const Byte *fileData, + IBenchPrintCallback *printCallback, CBenchCallbackToPrint *callback, + #ifndef Z7_ST + const CAffinityMode *affinityMode, + #endif + CTotalBenchRes *encodeRes, + bool showFreq, UInt64 cpuFreq) +{ + for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_Hash); i++) + { + const CBenchHash &bench = g_Hash[i]; + if (!DoesWildcardMatchName_NoCase(methodMask.MethodName, bench.Name)) + continue; + PrintLeft(*callback->_file, bench.Name, kFieldSize_Name); + // callback->BenchProps.DecComplexUnc = bench.DecComplexUnc; + // callback->BenchProps.DecComplexCompr = bench.DecComplexCompr; + // callback->BenchProps.EncComplex = bench.EncComplex; + + COneMethodInfo method; + NCOM::CPropVariant propVariant; + propVariant = bench.Name; + RINOK(method.ParseMethodFromPROPVARIANT(UString(), propVariant)) + + UInt64 speed, usage; + + const HRESULT res = CrcBench( + EXTERNAL_CODECS_LOC_VARS + complexInCommands, + numThreads, bufSize, fileData, + speed, usage, + bench.Complex, bench.Weight, + (!fileData && bufSize == (1 << kNumHashDictBits)) ? &bench.CheckSum : NULL, + method, + printCallback, + #ifndef Z7_ST + affinityMode, + #endif + true, // showRating + encodeRes, showFreq, cpuFreq); + if (res == E_NOTIMPL) + { + // callback->Print(" ---"); + } + else + { + RINOK(res) + } + callback->NewLine(); + } + return S_OK; +} + +struct CTempValues +{ + UInt64 *Values; + CTempValues(): Values(NULL) {} + void Alloc(UInt32 num) { Values = new UInt64[num]; } + ~CTempValues() { delete []Values; } +}; + +static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) +{ + const wchar_t *end; + UInt64 result = ConvertStringToUInt64(s, &end); + if (*end != 0 || s.IsEmpty()) + prop = s; + else if (result <= (UInt32)0xFFFFFFFF) + prop = (UInt32)result; + else + prop = result; +} + + +static bool AreSameMethodNames(const char *fullName, const char *shortName) +{ + return StringsAreEqualNoCase_Ascii(fullName, shortName); +} + + + + +static void Print_Usage_and_Threads(IBenchPrintCallback &f, UInt64 usage, UInt32 threads) +{ + PrintRequirements(f, "usage:", true, usage, "Benchmark threads: ", threads); +} + + +static void Print_Delimiter(IBenchPrintCallback &f) +{ + f.Print(" |"); +} + +static void Print_Pow(IBenchPrintCallback &f, unsigned pow) +{ + char s[16]; + ConvertUInt32ToString(pow, s); + unsigned pos = MyStringLen(s); + s[pos++] = ':'; + s[pos] = 0; + PrintLeft(f, s, kFieldSize_SmallName); // 4 +} + +static void Bench_BW_Print_Usage_Speed(IBenchPrintCallback &f, + UInt64 usage, UInt64 speed) +{ + PrintUsage(f, usage, kFieldSize_Usage); + PrintNumber(f, speed / 1000000, kFieldSize_CrcSpeed); +} + + +HRESULT Bench( + DECL_EXTERNAL_CODECS_LOC_VARS + IBenchPrintCallback *printCallback, + IBenchCallback *benchCallback, + const CObjectVector &props, + UInt32 numIterations, + bool multiDict, + IBenchFreqCallback *freqCallback) +{ + // for (int y = 0; y < 10000; y++) + if (!CrcInternalTest()) + return E_FAIL; + + UInt32 numCPUs = 1; + UInt64 ramSize = (UInt64)(sizeof(size_t)) << 29; + + NSystem::CProcessAffinity threadsInfo; + threadsInfo.InitST(); + + #ifndef Z7_ST + + if (threadsInfo.Get() && threadsInfo.GetNumProcessThreads() != 0) + numCPUs = threadsInfo.GetNumProcessThreads(); + else + numCPUs = NSystem::GetNumberOfProcessors(); + + #endif + + // numCPUs = 24; + /* + { + DWORD_PTR mask = (1 << 0); + DWORD_PTR old = SetThreadAffinityMask(GetCurrentThread(), mask); + old = old; + DWORD_PTR old2 = SetThreadAffinityMask(GetCurrentThread(), mask); + old2 = old2; + return 0; + } + */ + + const bool ramSize_Defined = NSystem::GetRamSize(ramSize); + + UInt32 numThreadsSpecified = numCPUs; + bool needSetComplexity = false; + UInt32 testTimeMs = kComplexInMs; + UInt32 startDicLog = 22; + bool startDicLog_Defined = false; + UInt64 specifiedFreq = 0; + bool multiThreadTests = false; + UInt64 complexInCommands = kComplexInCommands; + UInt32 numThreads_Start = 1; + + #ifndef Z7_ST + CAffinityMode affinityMode; + #endif + + + COneMethodInfo method; + + CMidAlignedBuffer fileDataBuffer; + bool use_fileData = false; + bool isFixedDict = false; + + { + unsigned i; + + if (printCallback) + { + for (i = 0; i < props.Size(); i++) + { + const CProperty &property = props[i]; + printCallback->Print(" "); + printCallback->Print(GetAnsiString(property.Name)); + if (!property.Value.IsEmpty()) + { + printCallback->Print("="); + printCallback->Print(GetAnsiString(property.Value)); + } + } + if (!props.IsEmpty()) + printCallback->NewLine(); + } + + + for (i = 0; i < props.Size(); i++) + { + const CProperty &property = props[i]; + UString name (property.Name); + name.MakeLower_Ascii(); + + if (name.IsEqualTo("file")) + { + if (property.Value.IsEmpty()) + return E_INVALIDARG; + + NFile::NIO::CInFile file; + if (!file.Open(us2fs(property.Value))) + return GetLastError_noZero_HRESULT(); + size_t len; + { + UInt64 len64; + if (!file.GetLength(len64)) + return GetLastError_noZero_HRESULT(); + if (printCallback) + { + printCallback->Print("file size ="); + PrintNumber(*printCallback, len64, 0); + printCallback->NewLine(); + } + len = (size_t)len64; + if (len != len64) + return E_INVALIDARG; + } + + // (len == 0) is allowed. Also it's allowed if Alloc(0) returns NULL here + + ALLOC_WITH_HRESULT(&fileDataBuffer, len) + use_fileData = true; + + { + size_t processed; + if (!file.ReadFull((Byte *)fileDataBuffer, len, processed)) + return GetLastError_noZero_HRESULT(); + if (processed != len) + return E_FAIL; + } + continue; + } + + NCOM::CPropVariant propVariant; + if (!property.Value.IsEmpty()) + ParseNumberString(property.Value, propVariant); + + if (name.IsEqualTo("time")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, testTimeMs)) + needSetComplexity = true; + testTimeMs *= 1000; + continue; + } + + if (name.IsEqualTo("timems")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, testTimeMs)) + needSetComplexity = true; + continue; + } + + if (name.IsEqualTo("tic")) + { + UInt32 v; + RINOK(ParsePropToUInt32(UString(), propVariant, v)) + if (v >= 64) + return E_INVALIDARG; + complexInCommands = (UInt64)1 << v; + continue; + } + + const bool isCurrent_fixedDict = name.IsEqualTo("df"); + if (isCurrent_fixedDict) + isFixedDict = true; + if (isCurrent_fixedDict || name.IsEqualTo("ds")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, startDicLog)) + if (startDicLog > 32) + return E_INVALIDARG; + startDicLog_Defined = true; + continue; + } + + if (name.IsEqualTo("mts")) + { + RINOK(ParsePropToUInt32(UString(), propVariant, numThreads_Start)) + continue; + } + + if (name.IsEqualTo("af")) + { + UInt32 bundle; + RINOK(ParsePropToUInt32(UString(), propVariant, bundle)) + if (bundle > 0 && bundle < numCPUs) + { + #ifndef Z7_ST + affinityMode.SetLevels(numCPUs, 2); + affinityMode.NumBundleThreads = bundle; + #endif + } + continue; + } + + if (name.IsEqualTo("freq")) + { + UInt32 freq32 = 0; + RINOK(ParsePropToUInt32(UString(), propVariant, freq32)) + if (freq32 == 0) + return E_INVALIDARG; + specifiedFreq = (UInt64)freq32 * 1000000; + + if (printCallback) + { + printCallback->Print("freq="); + PrintNumber(*printCallback, freq32, 0); + printCallback->NewLine(); + } + + continue; + } + + if (name.IsPrefixedBy_Ascii_NoCase("mt")) + { + const UString s = name.Ptr(2); + if (s.IsEqualTo("*") + || (s.IsEmpty() + && propVariant.vt == VT_BSTR + && StringsAreEqual_Ascii(propVariant.bstrVal, "*"))) + { + multiThreadTests = true; + continue; + } + #ifndef Z7_ST + RINOK(ParseMtProp(s, propVariant, numCPUs, numThreadsSpecified)) + #endif + continue; + } + + RINOK(method.ParseMethodFromPROPVARIANT(name, propVariant)) + } + } + + if (printCallback) + { + AString s; + +#if 1 || !defined(Z7_MSC_VER_ORIGINAL) || (Z7_MSC_VER_ORIGINAL >= 1900) + s += "Compiler: "; + GetCompiler(s); + printCallback->Print(s); + printCallback->NewLine(); + s.Empty(); +#endif + + GetSystemInfoText(s); + printCallback->Print(s); + printCallback->NewLine(); + } + + if (printCallback) + { + printCallback->Print("1T CPU Freq (MHz):"); + } + + if (printCallback || freqCallback) + { + UInt64 numMilCommands = 1 << 6; + if (specifiedFreq != 0) + { + while (numMilCommands > 1 && specifiedFreq < (numMilCommands * 1000000)) + numMilCommands >>= 1; + } + + for (int jj = 0;; jj++) + { + if (printCallback) + RINOK(printCallback->CheckBreak()) + + UInt64 start = ::GetTimeCount(); + UInt32 sum = (UInt32)start; + sum = CountCpuFreq(sum, (UInt32)(numMilCommands * 1000000 / kNumFreqCommands), g_BenchCpuFreqTemp); + if (sum == 0xF1541213) + if (printCallback) + printCallback->Print(""); + const UInt64 realDelta = ::GetTimeCount() - start; + start = realDelta; + if (start == 0) + start = 1; + if (start > (UInt64)1 << 61) + start = 1; + const UInt64 freq = GetFreq(); + // mips is constant in some compilers + const UInt64 hzVal = MyMultDiv64(numMilCommands * 1000000, freq, start); + const UInt64 mipsVal = numMilCommands * freq / start; + if (printCallback) + { + if (realDelta == 0) + { + printCallback->Print(" -"); + } + else + { + // PrintNumber(*printCallback, start, 0); + PrintNumber(*printCallback, mipsVal, 5); + } + } + if (freqCallback) + { + RINOK(freqCallback->AddCpuFreq(1, hzVal, kBenchmarkUsageMult)) + } + + if (jj >= 1) + { + bool needStop = (numMilCommands >= (1 << + #ifdef _DEBUG + 7 + #else + 11 + #endif + )); + if (start >= freq * 16) + { + printCallback->Print(" (Cmplx)"); + if (!freqCallback) // we don't want complexity change for old gui lzma benchmark + { + needSetComplexity = true; + } + needStop = true; + } + if (needSetComplexity) + SetComplexCommandsMs(testTimeMs, false, mipsVal * 1000000, complexInCommands); + if (needStop) + break; + numMilCommands <<= 1; + } + } + if (freqCallback) + { + RINOK(freqCallback->FreqsFinished(1)) + } + } + + if (printCallback || freqCallback) + for (unsigned test = 0; test < 3; test++) + { + if (numThreadsSpecified < 2) + { + // if (test == 1) + break; + } + if (test == 2 && numThreadsSpecified <= numCPUs) + break; + if (printCallback) + printCallback->NewLine(); + + /* it can show incorrect frequency for HT threads. */ + + UInt32 numThreads = numThreadsSpecified; + if (test < 2) + { + if (numThreads >= numCPUs) + numThreads = numCPUs; + if (test == 0) + numThreads /= 2; + } + if (numThreads < 1) + numThreads = 1; + + if (printCallback) + { + char s[128]; + ConvertUInt64ToString(numThreads, s); + printCallback->Print(s); + printCallback->Print("T CPU Freq (MHz):"); + } + UInt64 numMilCommands = 1 << + #ifdef _DEBUG + 7; + #else + 10; + #endif + + if (specifiedFreq != 0) + { + while (numMilCommands > 1 && specifiedFreq < (numMilCommands * 1000000)) + numMilCommands >>= 1; + } + + // for (int jj = 0;; jj++) + for (;;) + { + if (printCallback) + RINOK(printCallback->CheckBreak()) + + { + // PrintLeft(f, "CPU", kFieldSize_Name); + + // UInt32 resVal; + + CFreqBench fb; + fb.complexInCommands = numMilCommands * 1000000; + fb.numThreads = numThreads; + // showFreq; + // fb.showFreq = (freqTest == kNumCpuTests - 1 || specifiedFreq != 0); + fb.showFreq = true; + fb.specifiedFreq = 1; + + const HRESULT res = fb.FreqBench(NULL /* printCallback */ + #ifndef Z7_ST + , &affinityMode + #endif + ); + RINOK(res) + + if (freqCallback) + { + RINOK(freqCallback->AddCpuFreq(numThreads, fb.CpuFreqRes, fb.UsageRes)) + } + + if (printCallback) + { + /* + if (realDelta == 0) + { + printCallback->Print(" -"); + } + else + */ + { + // PrintNumber(*printCallback, start, 0); + PrintUsage(*printCallback, fb.UsageRes, 3); + printCallback->Print("%"); + PrintNumber(*printCallback, fb.CpuFreqRes / 1000000, 0); + printCallback->Print(" "); + + // PrintNumber(*printCallback, fb.UsageRes, 5); + } + } + } + // if (jj >= 1) + { + const bool needStop = (numMilCommands >= (1 << + #ifdef _DEBUG + 7 + #else + 11 + #endif + )); + if (needStop) + break; + numMilCommands <<= 1; + } + } + if (freqCallback) + { + RINOK(freqCallback->FreqsFinished(numThreads)) + } + } + + + if (printCallback) + { + printCallback->NewLine(); + printCallback->NewLine(); + PrintRequirements(*printCallback, "size: ", ramSize_Defined, ramSize, "CPU hardware threads:", numCPUs); + printCallback->Print(GetProcessThreadsInfo(threadsInfo)); + printCallback->NewLine(); + } + + if (numThreadsSpecified < 1 || numThreadsSpecified > kNumThreadsMax) + return E_INVALIDARG; + + UInt64 dict = (UInt64)1 << startDicLog; + const bool dictIsDefined = (isFixedDict || method.Get_DicSize(dict)); + + const unsigned level = method.GetLevel(); + + AString &methodName = method.MethodName; + const AString original_MethodName = methodName; + if (methodName.IsEmpty()) + methodName = "LZMA"; + + if (benchCallback) + { + CBenchProps benchProps; + benchProps.SetLzmaCompexity(); + const UInt64 dictSize = method.Get_Lzma_DicSize(); + + size_t uncompressedDataSize; + if (use_fileData) + { + uncompressedDataSize = fileDataBuffer.Size(); + } + else + { + uncompressedDataSize = kAdditionalSize + (size_t)dictSize; + if (uncompressedDataSize < dictSize) + return E_INVALIDARG; + } + + return MethodBench( + EXTERNAL_CODECS_LOC_VARS + complexInCommands, + #ifndef Z7_ST + true, numThreadsSpecified, + &affinityMode, + #endif + method, + uncompressedDataSize, (const Byte *)fileDataBuffer, + kOldLzmaDictBits, printCallback, benchCallback, &benchProps); + } + + if (methodName.IsEqualTo_Ascii_NoCase("CRC")) + methodName = "crc32"; + + CMethodId hashID; + const bool isHashMethod = FindHashMethod(EXTERNAL_CODECS_LOC_VARS methodName, hashID); + int codecIndex = -1; + bool isFilter = false; + if (!isHashMethod) + { + UInt32 numStreams; + codecIndex = FindMethod_Index(EXTERNAL_CODECS_LOC_VARS original_MethodName, + true, // encode + hashID, numStreams, isFilter); + // we can allow non filter for BW tests + if (!isFilter) codecIndex = -1; + } + + CBenchCallbackToPrint callback; + callback.Init(); + callback._file = printCallback; + + if (isHashMethod || codecIndex != -1) + { + if (!printCallback) + return S_FALSE; + IBenchPrintCallback &f = *printCallback; + + UInt64 dict64 = dict; + if (!dictIsDefined) + dict64 = (1 << 27); + if (use_fileData) + { + if (!dictIsDefined) + dict64 = fileDataBuffer.Size(); + else if (dict64 > fileDataBuffer.Size()) + dict64 = fileDataBuffer.Size(); + } + + for (;;) + { + const int index = method.FindProp(NCoderPropID::kDictionarySize); + if (index < 0) + break; + method.Props.Delete((unsigned)index); + } + + // methodName.RemoveChar(L'-'); + Int32 complexity = 16 * k_Hash_Complex_Mult; // for unknown hash method + const UInt32 *checkSum = NULL; + int benchIndex = -1; + + if (isHashMethod) + { + for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_Hash); i++) + { + const CBenchHash &h = g_Hash[i]; + AString benchMethod (h.Name); + AString benchProps; + const int propPos = benchMethod.Find(':'); + if (propPos >= 0) + { + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); + } + + if (AreSameMethodNames(benchMethod, methodName)) + { + const bool sameProps = method.PropsString.IsEqualTo_Ascii_NoCase(benchProps); + /* + bool isMainMethod = method.PropsString.IsEmpty(); + if (isMainMethod) + isMainMethod = !checkSum + || (benchMethod.IsEqualTo_Ascii_NoCase("crc32") && benchProps.IsEqualTo_Ascii_NoCase("8")); + if (sameProps || isMainMethod) + */ + { + complexity = (Int32)h.Complex; + checkSum = &h.CheckSum; + if (sameProps) + break; + /* + if property. is not specified, we use the complexity + for latest fastest method (crc32:64) + */ + } + } + } + // if (!checkSum) return E_NOTIMPL; + } + else + { + for (unsigned i = 0; i < Z7_ARRAY_SIZE(g_Bench); i++) + { + const CBenchMethod &bench = g_Bench[i]; + AString benchMethod (bench.Name); + AString benchProps; + const int propPos = benchMethod.Find(':'); + if (propPos >= 0) + { + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); + } + + if (AreSameMethodNames(benchMethod, methodName)) + { + const bool sameProps = method.PropsString.IsEqualTo_Ascii_NoCase(benchProps); + // bool isMainMethod = method.PropsString.IsEmpty(); + // if (sameProps || isMainMethod) + { + benchIndex = (int)i; + if (sameProps) + break; + } + } + } + // if (benchIndex < 0) return E_NOTIMPL; + } + + { + /* we count usage only for crc and filter. non-filters are not supported */ + UInt64 usage = (1 << 20); + UInt64 bufSize = dict64; + UInt32 numBlocks = isHashMethod ? 1 : 3; + if (use_fileData) + { + usage += fileDataBuffer.Size(); + if (bufSize > fileDataBuffer.Size()) + bufSize = fileDataBuffer.Size(); + if (isHashMethod) + { + numBlocks = 0; + #ifndef Z7_ST + if (numThreadsSpecified != 1) + numBlocks = (k_Crc_CreateLocalBuf_For_File ? 1 : 0); + #endif + } + } + usage += numThreadsSpecified * bufSize * numBlocks; + Print_Usage_and_Threads(f, usage, numThreadsSpecified); + } + + CUIntVector numThreadsVector; + { + unsigned nt = numThreads_Start; + for (;;) + { + if (nt > numThreadsSpecified) + break; + numThreadsVector.Add(nt); + const unsigned next = nt * 2; + const UInt32 ntHalf= numThreadsSpecified / 2; + if (ntHalf > nt && ntHalf < next) + numThreadsVector.Add(ntHalf); + if (numThreadsSpecified > nt && numThreadsSpecified < next) + numThreadsVector.Add(numThreadsSpecified); + nt = next; + } + } + + unsigned numColumns = isHashMethod ? 1 : 2; + CTempValues speedTotals; + CTempValues usageTotals; + { + const unsigned numItems = numThreadsVector.Size() * numColumns; + speedTotals.Alloc(numItems); + usageTotals.Alloc(numItems); + for (unsigned i = 0; i < numItems; i++) + { + speedTotals.Values[i] = 0; + usageTotals.Values[i] = 0; + } + } + + f.NewLine(); + for (unsigned line = 0; line < 3; line++) + { + f.NewLine(); + f.Print(line == 0 ? "THRD" : line == 1 ? " " : "Size"); + FOR_VECTOR (ti, numThreadsVector) + { + if (ti != 0) + Print_Delimiter(f); + if (line == 0) + { + PrintSpaces(f, (kFieldSize_CrcSpeed + kFieldSize_Usage + 2) * (numColumns - 1)); + PrintNumber(f, numThreadsVector[ti], 1 + kFieldSize_Usage + kFieldSize_CrcSpeed); + } + else + { + for (unsigned c = 0; c < numColumns; c++) + { + PrintRight(f, line == 1 ? "Usage" : "%", kFieldSize_Usage + 1); + PrintRight(f, line == 1 ? "BW" : "MB/s", kFieldSize_CrcSpeed + 1); + } + } + } + } + f.NewLine(); + + UInt64 numSteps = 0; + + // for (UInt32 iter = 0; iter < numIterations; iter++) + // { + unsigned pow = 10; // kNumHashDictBits + if (startDicLog_Defined) + pow = startDicLog; + + // #define NUM_SUB_BITS 2 + // pow <<= NUM_SUB_BITS; + for (;; pow++) + { + const UInt64 bufSize = (UInt64)1 << pow; + // UInt64 bufSize = (UInt64)1 << (pow >> NUM_SUB_BITS); + // bufSize += ((UInt64)pow & ((1 << NUM_SUB_BITS) - 1)) << ((pow >> NUM_SUB_BITS) - NUM_SUB_BITS); + + size_t dataSize = fileDataBuffer.Size(); + if (dataSize > bufSize || !use_fileData) + dataSize = (size_t)bufSize; + + for (UInt32 iter = 0; iter < numIterations; iter++) + { + Print_Pow(f, pow); + // PrintNumber(f, bufSize >> 10, 4); + + FOR_VECTOR (ti, numThreadsVector) + { + RINOK(f.CheckBreak()) + const UInt32 numThreads = numThreadsVector[ti]; + if (isHashMethod) + { + UInt64 speed = 0; + UInt64 usage = 0; + const HRESULT res = CrcBench(EXTERNAL_CODECS_LOC_VARS complexInCommands, + numThreads, + dataSize, (const Byte *)fileDataBuffer, + speed, usage, + (UInt32)complexity, + 1, // benchWeight, + (pow == kNumHashDictBits && !use_fileData) ? checkSum : NULL, + method, + &f, + #ifndef Z7_ST + &affinityMode, + #endif + false, // showRating + NULL, false, 0); + RINOK(res) + + if (ti != 0) + Print_Delimiter(f); + + Bench_BW_Print_Usage_Speed(f, usage, speed); + speedTotals.Values[ti] += speed; + usageTotals.Values[ti] += usage; + } + else + { + { + unsigned keySize = 32; + if (IsString1PrefixedByString2(methodName, "AES128")) keySize = 16; + else if (IsString1PrefixedByString2(methodName, "AES192")) keySize = 24; + callback.BenchProps.KeySize = keySize; + } + + COneMethodInfo method2 = method; + unsigned bench_DictBits; + + if (benchIndex >= 0) + { + const CBenchMethod &bench = g_Bench[benchIndex]; + callback.BenchProps.EncComplex = bench.EncComplex; + callback.BenchProps.DecComplexUnc = bench.DecComplexUnc; + callback.BenchProps.DecComplexCompr = bench.DecComplexCompr; + bench_DictBits = bench.DictBits; + // bench_DictBits = kOldLzmaDictBits; = 32 default : for debug + } + else + { + bench_DictBits = kOldLzmaDictBits; // = 32 default + if (isFilter) + { + const unsigned k_UnknownCoderComplexity = 4; + callback.BenchProps.EncComplex = k_UnknownCoderComplexity; + callback.BenchProps.DecComplexUnc = k_UnknownCoderComplexity; + } + else + { + callback.BenchProps.EncComplex = 1 << 10; + callback.BenchProps.DecComplexUnc = 1 << 6; + } + callback.BenchProps.DecComplexCompr = 0; + } + callback.NeedPrint = false; + + if (StringsAreEqualNoCase_Ascii(method2.MethodName, "LZMA")) + { + const NCOM::CPropVariant propVariant = (UInt32)pow; + RINOK(method2.ParseMethodFromPROPVARIANT((UString)"d", propVariant)) + } + + const HRESULT res = MethodBench( + EXTERNAL_CODECS_LOC_VARS + complexInCommands, + #ifndef Z7_ST + false, // oldLzmaBenchMode + numThreadsVector[ti], + &affinityMode, + #endif + method2, + dataSize, (const Byte *)fileDataBuffer, + bench_DictBits, + printCallback, + &callback, + &callback.BenchProps); + RINOK(res) + + if (ti != 0) + Print_Delimiter(f); + + for (unsigned i = 0; i < 2; i++) + { + const CBenchInfo &bi = callback.BenchInfo_Results[i]; + const UInt64 usage = bi.GetUsage(); + const UInt64 speed = bi.GetUnpackSizeSpeed(); + usageTotals.Values[ti * 2 + i] += usage; + speedTotals.Values[ti * 2 + i] += speed; + Bench_BW_Print_Usage_Speed(f, usage, speed); + } + } + } + + f.NewLine(); + numSteps++; + } + if (dataSize >= dict64) + break; + } + + if (numSteps != 0) + { + f.Print("Avg:"); + for (unsigned ti = 0; ti < numThreadsVector.Size(); ti++) + { + if (ti != 0) + Print_Delimiter(f); + for (unsigned i = 0; i < numColumns; i++) + Bench_BW_Print_Usage_Speed(f, + usageTotals.Values[ti * numColumns + i] / numSteps, + speedTotals.Values[ti * numColumns + i] / numSteps); + } + f.NewLine(); + } + + return S_OK; + } + + bool use2Columns = false; + + bool totalBenchMode = false; + bool onlyHashBench = false; + if (methodName.IsEqualTo_Ascii_NoCase("hash")) + { + onlyHashBench = true; + methodName = "*"; + totalBenchMode = true; + } + else if (methodName.Find('*') >= 0) + totalBenchMode = true; + + // ---------- Threads loop ---------- + for (unsigned threadsPassIndex = 0; threadsPassIndex < 3; threadsPassIndex++) + { + + UInt32 numThreads = numThreadsSpecified; + + if (!multiThreadTests) + { + if (threadsPassIndex != 0) + break; + } + else + { + numThreads = 1; + if (threadsPassIndex != 0) + { + if (numCPUs < 2) + break; + numThreads = numCPUs; + if (threadsPassIndex == 1) + { + if (numCPUs >= 4) + numThreads = numCPUs / 2; + } + else if (numCPUs < 4) + break; + } + } + + IBenchPrintCallback &f = *printCallback; + + if (threadsPassIndex > 0) + { + f.NewLine(); + f.NewLine(); + } + + if (!dictIsDefined && !onlyHashBench) + { + const unsigned dicSizeLog_Main = (totalBenchMode ? 24 : 25); + unsigned dicSizeLog = dicSizeLog_Main; + + #ifdef UNDER_CE + dicSizeLog = (UInt64)1 << 20; + #endif + + if (ramSize_Defined) + for (; dicSizeLog > kBenchMinDicLogSize; dicSizeLog--) + if (GetBenchMemoryUsage(numThreads, (int)level, ((UInt64)1 << dicSizeLog), totalBenchMode) + (8 << 20) <= ramSize) + break; + + dict = (UInt64)1 << dicSizeLog; + + if (totalBenchMode && dicSizeLog != dicSizeLog_Main) + { + f.Print("Dictionary reduced to: "); + PrintNumber(f, dicSizeLog, 1); + f.NewLine(); + } + } + + Print_Usage_and_Threads(f, + onlyHashBench ? + GetBenchMemoryUsage_Hash(numThreads, dict) : + GetBenchMemoryUsage(numThreads, (int)level, dict, totalBenchMode), + numThreads); + + f.NewLine(); + + f.NewLine(); + + if (totalBenchMode) + { + callback.NameFieldSize = kFieldSize_Name; + use2Columns = false; + } + else + { + callback.NameFieldSize = kFieldSize_SmallName; + use2Columns = true; + } + callback.Use2Columns = use2Columns; + + bool showFreq = false; + UInt64 cpuFreq = 0; + + if (totalBenchMode) + { + showFreq = true; + } + + unsigned fileldSize = kFieldSize_TotalSize; + if (showFreq) + fileldSize += kFieldSize_EUAndEffec; + + if (use2Columns) + { + PrintSpaces(f, callback.NameFieldSize); + PrintRight(f, "Compressing", fileldSize); + f.Print(kSep); + PrintRight(f, "Decompressing", fileldSize); + } + f.NewLine(); + PrintLeft(f, totalBenchMode ? "Method" : "Dict", callback.NameFieldSize); + + int j; + + for (j = 0; j < 2; j++) + { + PrintRight(f, "Speed", kFieldSize_Speed + 1); + PrintRight(f, "Usage", kFieldSize_Usage + 1); + PrintRight(f, "R/U", kFieldSize_RU + 1); + PrintRight(f, "Rating", kFieldSize_Rating + 1); + if (showFreq) + { + PrintRight(f, "E/U", kFieldSize_EU + 1); + PrintRight(f, "Effec", kFieldSize_Effec + 1); + } + if (!use2Columns) + break; + if (j == 0) + f.Print(kSep); + } + + f.NewLine(); + PrintSpaces(f, callback.NameFieldSize); + + for (j = 0; j < 2; j++) + { + PrintRight(f, "KiB/s", kFieldSize_Speed + 1); + PrintRight(f, "%", kFieldSize_Usage + 1); + PrintRight(f, "MIPS", kFieldSize_RU + 1); + PrintRight(f, "MIPS", kFieldSize_Rating + 1); + if (showFreq) + { + PrintRight(f, "%", kFieldSize_EU + 1); + PrintRight(f, "%", kFieldSize_Effec + 1); + } + if (!use2Columns) + break; + if (j == 0) + f.Print(kSep); + } + + f.NewLine(); + f.NewLine(); + + if (specifiedFreq != 0) + cpuFreq = specifiedFreq; + + // bool showTotalSpeed = false; + + if (totalBenchMode) + { + for (UInt32 i = 0; i < numIterations; i++) + { + if (i != 0) + printCallback->NewLine(); + + const unsigned kNumCpuTests = 3; + for (unsigned freqTest = 0; freqTest < kNumCpuTests; freqTest++) + { + PrintLeft(f, "CPU", kFieldSize_Name); + + // UInt32 resVal; + + CFreqBench fb; + fb.complexInCommands = complexInCommands; + fb.numThreads = numThreads; + // showFreq; + fb.showFreq = (freqTest == kNumCpuTests - 1 || specifiedFreq != 0); + fb.specifiedFreq = specifiedFreq; + + const HRESULT res = fb.FreqBench(printCallback + #ifndef Z7_ST + , &affinityMode + #endif + ); + RINOK(res) + + cpuFreq = fb.CpuFreqRes; + callback.NewLine(); + + if (specifiedFreq != 0) + cpuFreq = specifiedFreq; + + if (testTimeMs >= 1000) + if (freqTest == kNumCpuTests - 1) + { + // SetComplexCommandsMs(testTimeMs, specifiedFreq != 0, cpuFreq, complexInCommands); + } + } + callback.NewLine(); + + // return S_OK; // change it + + callback.SetFreq(true, cpuFreq); + + if (!onlyHashBench) + { + size_t dataSize = (size_t)dict; + if (use_fileData) + { + dataSize = fileDataBuffer.Size(); + if (dictIsDefined && dataSize > dict) + dataSize = (size_t)dict; + } + + const HRESULT res = TotalBench(EXTERNAL_CODECS_LOC_VARS + method, complexInCommands, + #ifndef Z7_ST + numThreads, + &affinityMode, + #endif + dictIsDefined || use_fileData, // forceUnpackSize + dataSize, + (const Byte *)fileDataBuffer, + printCallback, &callback); + RINOK(res) + } + + { + size_t dataSize = (size_t)1 << kNumHashDictBits; + if (dictIsDefined) + { + dataSize = (size_t)dict; + if (dataSize != dict) + return E_OUTOFMEMORY; + } + if (use_fileData) + { + dataSize = fileDataBuffer.Size(); + if (dictIsDefined && dataSize > dict) + dataSize = (size_t)dict; + } + + const HRESULT res = TotalBench_Hash(EXTERNAL_CODECS_LOC_VARS + method, complexInCommands, + numThreads, + dataSize, (const Byte *)fileDataBuffer, + printCallback, &callback, + #ifndef Z7_ST + &affinityMode, + #endif + &callback.EncodeRes, true, cpuFreq); + RINOK(res) + } + + callback.NewLine(); + { + PrintLeft(f, "CPU", kFieldSize_Name); + + CFreqBench fb; + fb.complexInCommands = complexInCommands; + fb.numThreads = numThreads; + // showFreq; + fb.showFreq = (specifiedFreq != 0); + fb.specifiedFreq = specifiedFreq; + + const HRESULT res = fb.FreqBench(printCallback + #ifndef Z7_ST + , &affinityMode + #endif + ); + RINOK(res) + callback.NewLine(); + } + } + } + else + { + needSetComplexity = true; + if (!methodName.IsEqualTo_Ascii_NoCase("LZMA")) + { + unsigned i; + for (i = 0; i < Z7_ARRAY_SIZE(g_Bench); i++) + { + const CBenchMethod &h = g_Bench[i]; + AString benchMethod (h.Name); + AString benchProps; + const int propPos = benchMethod.Find(':'); + if (propPos >= 0) + { + benchProps = benchMethod.Ptr((unsigned)(propPos + 1)); + benchMethod.DeleteFrom((unsigned)propPos); + } + + if (AreSameMethodNames(benchMethod, methodName)) + { + if (benchProps.IsEmpty() + || (benchProps == "x5" && method.PropsString.IsEmpty()) + || method.PropsString.IsPrefixedBy_Ascii_NoCase(benchProps)) + { + callback.BenchProps.EncComplex = h.EncComplex; + callback.BenchProps.DecComplexCompr = h.DecComplexCompr; + callback.BenchProps.DecComplexUnc = h.DecComplexUnc; + needSetComplexity = false; + break; + } + } + } + /* + if (i == Z7_ARRAY_SIZE(g_Bench)) + return E_NOTIMPL; + */ + } + if (needSetComplexity) + callback.BenchProps.SetLzmaCompexity(); + + if (startDicLog < kBenchMinDicLogSize) + startDicLog = kBenchMinDicLogSize; + + for (unsigned i = 0; i < numIterations; i++) + { + unsigned pow = (dict < GetDictSizeFromLog(startDicLog)) ? kBenchMinDicLogSize : (unsigned)startDicLog; + if (!multiDict) + pow = 32; + while (GetDictSizeFromLog(pow) > dict && pow > 0) + pow--; + for (; GetDictSizeFromLog(pow) <= dict; pow++) + { + Print_Pow(f, pow); + callback.DictSize = (UInt64)1 << pow; + + COneMethodInfo method2 = method; + + if (StringsAreEqualNoCase_Ascii(method2.MethodName, "LZMA")) + { + // We add dictionary size property. + // method2 can have two different dictionary size properties. + // And last property is main. + NCOM::CPropVariant propVariant = (UInt32)pow; + RINOK(method2.ParseMethodFromPROPVARIANT((UString)"d", propVariant)) + } + + size_t uncompressedDataSize; + if (use_fileData) + { + uncompressedDataSize = fileDataBuffer.Size(); + } + else + { + uncompressedDataSize = (size_t)callback.DictSize; + if (uncompressedDataSize != callback.DictSize) + return E_OUTOFMEMORY; + if (uncompressedDataSize >= (1 << 18)) + uncompressedDataSize += kAdditionalSize; + } + + const HRESULT res = MethodBench( + EXTERNAL_CODECS_LOC_VARS + complexInCommands, + #ifndef Z7_ST + true, numThreads, + &affinityMode, + #endif + method2, + uncompressedDataSize, (const Byte *)fileDataBuffer, + kOldLzmaDictBits, printCallback, &callback, &callback.BenchProps); + f.NewLine(); + RINOK(res) + if (!multiDict) + break; + } + } + } + + PrintChars(f, '-', callback.NameFieldSize + fileldSize); + + if (use2Columns) + { + f.Print(kSep); + PrintChars(f, '-', fileldSize); + } + + f.NewLine(); + + if (use2Columns) + { + PrintLeft(f, "Avr:", callback.NameFieldSize); + PrintTotals(f, showFreq, cpuFreq, !totalBenchMode, callback.EncodeRes); + f.Print(kSep); + PrintTotals(f, showFreq, cpuFreq, !totalBenchMode, callback.DecodeRes); + f.NewLine(); + } + + PrintLeft(f, "Tot:", callback.NameFieldSize); + CTotalBenchRes midRes; + midRes = callback.EncodeRes; + midRes.Update_With_Res(callback.DecodeRes); + + // midRes.SetSum(callback.EncodeRes, callback.DecodeRes); + PrintTotals(f, showFreq, cpuFreq, false, midRes); + f.NewLine(); + + } + return S_OK; +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.h b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.h new file mode 100644 index 000000000..5200090e7 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Bench.h @@ -0,0 +1,121 @@ +// Bench.h + +#ifndef ZIP7_INC_7ZIP_BENCH_H +#define ZIP7_INC_7ZIP_BENCH_H + +#include "../../../Windows/System.h" + +#include "../../Common/CreateCoder.h" +#include "../../UI/Common/Property.h" + +UInt64 Benchmark_GetUsage_Percents(UInt64 usage); + +struct CBenchInfo +{ + UInt64 GlobalTime; + UInt64 GlobalFreq; + UInt64 UserTime; + UInt64 UserFreq; + UInt64 UnpackSize; + UInt64 PackSize; + UInt64 NumIterations; + + /* + during Code(): we track benchInfo only from one thread (theads with index[0]) + NumIterations means number of threads + UnpackSize and PackSize are total sizes of all iterations of current thread + after Code(): + NumIterations means the number of Iterations + UnpackSize and PackSize are total sizes of all threads + */ + + CBenchInfo(): NumIterations(0) {} + + UInt64 GetUsage() const; + UInt64 GetRatingPerUsage(UInt64 rating) const; + UInt64 GetSpeed(UInt64 numUnits) const; + UInt64 GetUnpackSizeSpeed() const { return GetSpeed(UnpackSize * NumIterations); } + + UInt64 Get_UnpackSize_Full() const { return UnpackSize * NumIterations; } + + UInt64 GetRating_LzmaEnc(UInt64 dictSize) const; + UInt64 GetRating_LzmaDec() const; +}; + + +struct CTotalBenchRes +{ + // UInt64 NumIterations1; // for Usage + UInt64 NumIterations2; // for Rating / RPU + + UInt64 Rating; + UInt64 Usage; + UInt64 RPU; + UInt64 Speed; + + void Init() { /* NumIterations1 = 0; */ NumIterations2 = 0; Rating = 0; Usage = 0; RPU = 0; Speed = 0; } + + void SetSum(const CTotalBenchRes &r1, const CTotalBenchRes &r2) + { + Rating = (r1.Rating + r2.Rating); + Usage = (r1.Usage + r2.Usage); + RPU = (r1.RPU + r2.RPU); + Speed = (r1.Speed + r2.Speed); + // NumIterations1 = (r1.NumIterations1 + r2.NumIterations1); + NumIterations2 = (r1.NumIterations2 + r2.NumIterations2); + } + + void Generate_From_BenchInfo(const CBenchInfo &info); + void Mult_For_Weight(unsigned weight); + void Update_With_Res(const CTotalBenchRes &r); +}; + + +const unsigned kBenchMinDicLogSize = 18; + +UInt64 GetBenchMemoryUsage(UInt32 numThreads, int level, UInt64 dictionary, bool totalBench); + +Z7_PURE_INTERFACES_BEGIN +DECLARE_INTERFACE(IBenchCallback) +{ + // virtual HRESULT SetFreq(bool showFreq, UInt64 cpuFreq) = 0; + virtual HRESULT SetEncodeResult(const CBenchInfo &info, bool final) = 0; + virtual HRESULT SetDecodeResult(const CBenchInfo &info, bool final) = 0; +}; + +DECLARE_INTERFACE(IBenchPrintCallback) +{ + virtual void Print(const char *s) = 0; + virtual void NewLine() = 0; + virtual HRESULT CheckBreak() = 0; +}; + +DECLARE_INTERFACE(IBenchFreqCallback) +{ + virtual HRESULT AddCpuFreq(unsigned numThreads, UInt64 freq, UInt64 usage) = 0; + virtual HRESULT FreqsFinished(unsigned numThreads) = 0; +}; +Z7_PURE_INTERFACES_END + +HRESULT Bench( + DECL_EXTERNAL_CODECS_LOC_VARS + IBenchPrintCallback *printCallback, + IBenchCallback *benchCallback, + const CObjectVector &props, + UInt32 numIterations, + bool multiDict, + IBenchFreqCallback *freqCallback = NULL); + +AString GetProcessThreadsInfo(const NWindows::NSystem::CProcessAffinity &ti); + +void GetSysInfo(AString &s1, AString &s2); +void GetCpuName(AString &s); +void AddCpuFeatures(AString &s); + +#ifdef Z7_LARGE_PAGES +void Add_LargePages_String(AString &s); +#else +// #define Add_LargePages_String +#endif + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/EnumDirItems.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/EnumDirItems.cpp new file mode 100644 index 000000000..941be1de1 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/EnumDirItems.cpp @@ -0,0 +1,1654 @@ +// EnumDirItems.cpp + +#include "StdAfx.h" + +#include +// #include + +#ifndef _WIN32 +#include +#include +#include "../../../Common/UTFConvert.h" +#endif + +#include "../../../Common/Wildcard.h" + +#include "../../../Windows/FileDir.h" +#include "../../../Windows/FileIO.h" +#include "../../../Windows/FileName.h" + +#if defined(_WIN32) && !defined(UNDER_CE) +#define Z7_USE_SECURITY_CODE +#include "../../../Windows/SecurityUtils.h" +#endif + +#include "EnumDirItems.h" +#include "SortUtils.h" + +using namespace NWindows; +using namespace NFile; +using namespace NName; + + +static bool FindFile_KeepDots(NFile::NFind::CFileInfo &fi, const FString &path, bool followLink) +{ + const bool res = fi.Find(path, followLink); + if (!res) + return res; + if (path.IsEmpty()) + return res; + // we keep name "." and "..", if it's without tail slash + const FChar *p = path.RightPtr(1); + if (*p != '.') + return res; + if (p != path.Ptr()) + { + FChar c = p[-1]; + if (!IS_PATH_SEPAR(c)) + { + if (c != '.') + return res; + p--; + if (p != path.Ptr()) + { + c = p[-1]; + if (!IS_PATH_SEPAR(c)) + return res; + } + } + } + fi.Name = p; + return res; +} + + +void CDirItems::AddDirFileInfo(int phyParent, int logParent, int secureIndex, + const NFind::CFileInfo &fi) +{ + /* + CDirItem di(fi); + di.PhyParent = phyParent; + di.LogParent = logParent; + di.SecureIndex = secureIndex; + Items.Add(di); + */ + VECTOR_ADD_NEW_OBJECT (Items, CDirItem(fi, phyParent, logParent, secureIndex)) + + if (fi.IsDir()) + Stat.NumDirs++; + #ifdef _WIN32 + else if (fi.IsAltStream) + { + Stat.NumAltStreams++; + Stat.AltStreamsSize += fi.Size; + } + #endif + else + { + Stat.NumFiles++; + Stat.FilesSize += fi.Size; + } +} + +// (DWORD)E_FAIL +#define DI_DEFAULT_ERROR ERROR_INVALID_FUNCTION + +HRESULT CDirItems::AddError(const FString &path, DWORD errorCode) +{ + if (errorCode == 0) + errorCode = DI_DEFAULT_ERROR; + Stat.NumErrors++; + if (Callback) + return Callback->ScanError(path, errorCode); + return S_OK; +} + +HRESULT CDirItems::AddError(const FString &path) +{ + return AddError(path, ::GetLastError()); +} + +static const unsigned kScanProgressStepMask = (1 << 12) - 1; + +HRESULT CDirItems::ScanProgress(const FString &dirPath) +{ + if (Callback) + return Callback->ScanProgress(Stat, dirPath, true); + return S_OK; +} + +UString CDirItems::GetPrefixesPath(const CIntVector &parents, int index, const UString &name) const +{ + UString path; + unsigned len = name.Len(); + + int i; + for (i = index; i >= 0; i = parents[(unsigned)i]) + len += Prefixes[(unsigned)i].Len(); + + wchar_t *p = path.GetBuf_SetEnd(len) + len; + + p -= name.Len(); + wmemcpy(p, (const wchar_t *)name, name.Len()); + + for (i = index; i >= 0; i = parents[(unsigned)i]) + { + const UString &s = Prefixes[(unsigned)i]; + p -= s.Len(); + wmemcpy(p, (const wchar_t *)s, s.Len()); + } + + return path; +} + +FString CDirItems::GetPhyPath(unsigned index) const +{ + const CDirItem &di = Items[index]; + return us2fs(GetPrefixesPath(PhyParents, di.PhyParent, di.Name)); +} + +UString CDirItems::GetLogPath(unsigned index) const +{ + const CDirItem &di = Items[index]; + return GetPrefixesPath(LogParents, di.LogParent, di.Name); +} + +void CDirItems::ReserveDown() +{ + Prefixes.ReserveDown(); + PhyParents.ReserveDown(); + LogParents.ReserveDown(); + Items.ReserveDown(); +} + +unsigned CDirItems::AddPrefix(int phyParent, int logParent, const UString &prefix) +{ + PhyParents.Add(phyParent); + LogParents.Add(logParent); + return Prefixes.Add(prefix); +} + +void CDirItems::DeleteLastPrefix() +{ + PhyParents.DeleteBack(); + LogParents.DeleteBack(); + Prefixes.DeleteBack(); +} + +bool InitLocalPrivileges(); + +CDirItems::CDirItems(): + SymLinks(false), + ScanAltStreams(false) + , ExcludeDirItems(false) + , ExcludeFileItems(false) + , ShareForWrite(false) + #ifdef Z7_USE_SECURITY_CODE + , ReadSecure(false) + #endif + #ifndef _WIN32 + , StoreOwnerName(false) + #endif + , Callback(NULL) +{ + #ifdef Z7_USE_SECURITY_CODE + _saclEnabled = InitLocalPrivileges(); + #endif +} + + +#ifdef Z7_USE_SECURITY_CODE + +HRESULT CDirItems::AddSecurityItem(const FString &path, int &secureIndex) +{ + secureIndex = -1; + + SECURITY_INFORMATION securInfo = + DACL_SECURITY_INFORMATION | + GROUP_SECURITY_INFORMATION | + OWNER_SECURITY_INFORMATION; + if (_saclEnabled) + securInfo |= SACL_SECURITY_INFORMATION; + + DWORD errorCode = 0; + DWORD secureSize; + + BOOL res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(void *)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); + + if (res) + { + if (secureSize == 0) + return S_OK; + if (secureSize > TempSecureBuf.Size()) + errorCode = ERROR_INVALID_FUNCTION; + } + else + { + errorCode = GetLastError(); + if (errorCode == ERROR_INSUFFICIENT_BUFFER) + { + if (secureSize <= TempSecureBuf.Size()) + errorCode = ERROR_INVALID_FUNCTION; + else + { + TempSecureBuf.Alloc(secureSize); + res = ::GetFileSecurityW(fs2us(path), securInfo, (PSECURITY_DESCRIPTOR)(void *)(Byte *)TempSecureBuf, (DWORD)TempSecureBuf.Size(), &secureSize); + if (res) + { + if (secureSize != TempSecureBuf.Size()) + errorCode = ERROR_INVALID_FUNCTION; + } + else + errorCode = GetLastError(); + } + } + } + + if (res) + { + secureIndex = (int)SecureBlocks.AddUniq(TempSecureBuf, secureSize); + return S_OK; + } + + return AddError(path, errorCode); +} + +#endif // Z7_USE_SECURITY_CODE + + +HRESULT CDirItems::EnumerateOneDir(const FString &phyPrefix, CObjectVector &files) +{ + NFind::CEnumerator enumerator; + // printf("\n enumerator.SetDirPrefix(phyPrefix) \n"); + + enumerator.SetDirPrefix(phyPrefix); + + #ifdef _WIN32 + + NFind::CFileInfo fi; + + for (unsigned ttt = 0; ; ttt++) + { + bool found; + if (!enumerator.Next(fi, found)) + return AddError(phyPrefix); + if (!found) + return S_OK; + files.Add(fi); + if (Callback && (ttt & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)) + } + } + + #else // _WIN32 + + // enumerator.SolveLinks = !SymLinks; + + CObjectVector entries; + + for (;;) + { + bool found; + NFind::CDirEntry de; + if (!enumerator.Next(de, found)) + return AddError(phyPrefix); + if (!found) + break; + entries.Add(de); + } + + FOR_VECTOR(i, entries) + { + const NFind::CDirEntry &de = entries[i]; + NFind::CFileInfo fi; + if (!enumerator.Fill_FileInfo(de, fi, !SymLinks)) + // if (!fi.Find_AfterEnumerator(path)) + { + const FString path = phyPrefix + de.Name; + { + RINOK(AddError(path)) + continue; + } + } + + files.Add(fi); + + if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)) + } + } + + return S_OK; + + #endif // _WIN32 +} + + + + +HRESULT CDirItems::EnumerateDir(int phyParent, int logParent, const FString &phyPrefix) +{ + RINOK(ScanProgress(phyPrefix)) + + CObjectVector files; + RINOK(EnumerateOneDir(phyPrefix, files)) + + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + const NFind::CFileInfo &fi = files[i]; + #else + const NFind::CFileInfo &fi = files[i]; + /* + NFind::CFileInfo fi; + { + const NFind::CDirEntry &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(AddError(path)); + continue; + } + fi.Name = di.Name; + } + */ + #endif + + if (CanIncludeItem(fi.IsDir())) + { + int secureIndex = -1; + #ifdef Z7_USE_SECURITY_CODE + if (ReadSecure) + { + RINOK(AddSecurityItem(phyPrefix + fi.Name, secureIndex)) + } + #endif + AddDirFileInfo(phyParent, logParent, secureIndex, fi); + } + + if (Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(ScanProgress(phyPrefix)) + } + + if (fi.IsDir()) + { + const FString name2 = fi.Name + FCHAR_PATH_SEPARATOR; + unsigned parent = AddPrefix(phyParent, logParent, fs2us(name2)); + RINOK(EnumerateDir((int)parent, (int)parent, phyPrefix + name2)) + } + } + return S_OK; +} + + +/* +EnumerateItems2() + const FStringVector &filePaths - are path without tail slashes. + All dir prefixes of filePaths will be not stores in logical paths +fix it: we can scan AltStream also. +*/ + +#ifdef _WIN32 +// #define FOLLOW_LINK_PARAM +// #define FOLLOW_LINK_PARAM2 +#define FOLLOW_LINK_PARAM , (!SymLinks) +#define FOLLOW_LINK_PARAM2 , (!dirItems.SymLinks) +#else +#define FOLLOW_LINK_PARAM , (!SymLinks) +#define FOLLOW_LINK_PARAM2 , (!dirItems.SymLinks) +#endif + +HRESULT CDirItems::EnumerateItems2( + const FString &phyPrefix, + const UString &logPrefix, + const FStringVector &filePaths, + FStringVector *requestedPaths) +{ + const int phyParent = phyPrefix.IsEmpty() ? -1 : (int)AddPrefix(-1, -1, fs2us(phyPrefix)); + const int logParent = logPrefix.IsEmpty() ? -1 : (int)AddPrefix(-1, -1, logPrefix); + + #ifdef _WIN32 + const bool phyPrefix_isAltStreamPrefix = + NFile::NName::IsAltStreamPrefixWithColon(fs2us(phyPrefix)); + #endif + + FOR_VECTOR (i, filePaths) + { + const FString &filePath = filePaths[i]; + NFind::CFileInfo fi; + const FString phyPath = phyPrefix + filePath; + if (!FindFile_KeepDots(fi, phyPath FOLLOW_LINK_PARAM)) + { + RINOK(AddError(phyPath)) + continue; + } + if (requestedPaths) + requestedPaths->Add(phyPath); + + const int delimiter = filePath.ReverseFind_PathSepar(); + FString phyPrefixCur; + int phyParentCur = phyParent; + if (delimiter >= 0) + { + phyPrefixCur.SetFrom(filePath, (unsigned)(delimiter + 1)); + phyParentCur = (int)AddPrefix(phyParent, logParent, fs2us(phyPrefixCur)); + } + + if (CanIncludeItem(fi.IsDir())) + { + int secureIndex = -1; + #ifdef Z7_USE_SECURITY_CODE + if (ReadSecure) + { + RINOK(AddSecurityItem(phyPath, secureIndex)) + } + #endif + #ifdef _WIN32 + if (phyPrefix_isAltStreamPrefix && fi.IsAltStream) + { + const int pos = fi.Name.Find(FChar(':')); + if (pos >= 0) + fi.Name.DeleteFrontal((unsigned)pos + 1); + } + #endif + AddDirFileInfo(phyParentCur, logParent, secureIndex, fi); + } + + if (fi.IsDir()) + { + const FString name2 = fi.Name + FCHAR_PATH_SEPARATOR; + const unsigned parent = AddPrefix(phyParentCur, logParent, fs2us(name2)); + RINOK(EnumerateDir((int)parent, (int)parent, phyPrefix + phyPrefixCur + name2)) + } + } + + ReserveDown(); + return S_OK; +} + + + + +static HRESULT EnumerateDirItems( + const NWildcard::CCensorNode &curNode, + const int phyParent, const int logParent, + const FString &phyPrefix, + const UStringVector &addParts, // additional parts from curNode + CDirItems &dirItems, + bool enterToSubFolders); + + +/* EnumerateDirItems_Spec() + adds new Dir item prefix, and enumerates dir items, + then it can remove that Dir item prefix, if there are no items in that dir. +*/ + + +/* + EnumerateDirItems_Spec() + it's similar to EnumerateDirItems, but phyPrefix doesn't include (curFolderName) +*/ + +static HRESULT EnumerateDirItems_Spec( + const NWildcard::CCensorNode &curNode, + const int phyParent, const int logParent, const FString &curFolderName, + const FString &phyPrefix, // without (curFolderName) + const UStringVector &addParts, // (curNode + addParts) includes (curFolderName) + CDirItems &dirItems, + bool enterToSubFolders) +{ + const FString name2 = curFolderName + FCHAR_PATH_SEPARATOR; + const unsigned parent = dirItems.AddPrefix(phyParent, logParent, fs2us(name2)); + const unsigned numItems = dirItems.Items.Size(); + HRESULT res = EnumerateDirItems( + curNode, (int)parent, (int)parent, phyPrefix + name2, + addParts, dirItems, enterToSubFolders); + if (numItems == dirItems.Items.Size()) + dirItems.DeleteLastPrefix(); + return res; +} + + +#ifndef UNDER_CE + +#ifdef _WIN32 + +static HRESULT EnumerateAltStreams( + const NFind::CFileInfo &fi, + const NWildcard::CCensorNode &curNode, + const int phyParent, const int logParent, + const FString &phyPath, // with (fi.Name), without tail slash for folders + const UStringVector &addParts, // with (fi.Name), prefix parts from curNode + bool addAllSubStreams, + CDirItems &dirItems) +{ + // we don't use (ExcludeFileItems) rules for AltStreams + // if (dirItems.ExcludeFileItems) return S_OK; + + NFind::CStreamEnumerator enumerator(phyPath); + for (;;) + { + NFind::CStreamInfo si; + bool found; + if (!enumerator.Next(si, found)) + { + return dirItems.AddError(phyPath + FTEXT(":*")); // , (DWORD)E_FAIL + } + if (!found) + return S_OK; + if (si.IsMainStream()) + continue; + UStringVector parts = addParts; + const UString reducedName = si.GetReducedName(); + parts.Back() += reducedName; + if (curNode.CheckPathToRoot(false, parts, true)) + continue; + if (!addAllSubStreams) + if (!curNode.CheckPathToRoot(true, parts, true)) + continue; + + NFind::CFileInfo fi2 = fi; + fi2.Name += us2fs(reducedName); + fi2.Size = si.Size; + fi2.Attrib &= ~(DWORD)(FILE_ATTRIBUTE_DIRECTORY | FILE_ATTRIBUTE_REPARSE_POINT); + fi2.IsAltStream = true; + dirItems.AddDirFileInfo(phyParent, logParent, -1, fi2); + } +} + +#endif // _WIN32 + + +/* We get Reparse data and parse it. + If there is Reparse error, we free dirItem.Reparse data. + Do we need to work with empty reparse data? +*/ + +HRESULT CDirItems::SetLinkInfo(CDirItem &dirItem, const NFind::CFileInfo &fi, + const FString &phyPrefix) +{ + if (!SymLinks) + return S_OK; + + #ifdef _WIN32 + if (!fi.HasReparsePoint() || fi.IsAltStream) + #else // _WIN32 + if (!fi.IsPosixLink()) + #endif // _WIN32 + return S_OK; + + const FString path = phyPrefix + fi.Name; + CByteBuffer &buf = dirItem.ReparseData; + if (NIO::GetReparseData(path, buf)) + { + // if (dirItem.ReparseData.Size() != 0) + Stat.FilesSize -= fi.Size; + return S_OK; + } + + DWORD res = ::GetLastError(); + buf.Free(); + return AddError(path, res); +} + +#endif // UNDER_CE + + + +static HRESULT EnumerateForItem( + const NFind::CFileInfo &fi, + const NWildcard::CCensorNode &curNode, + const int phyParent, const int logParent, const FString &phyPrefix, + const UStringVector &addParts, // additional parts from curNode, without (fi.Name) + CDirItems &dirItems, + bool enterToSubFolders) +{ + const UString name = fs2us(fi.Name); + UStringVector newParts = addParts; + newParts.Add(name); + + // check the path in exclude rules + if (curNode.CheckPathToRoot(false, newParts, !fi.IsDir())) + return S_OK; + + #if !defined(UNDER_CE) + int dirItemIndex = -1; + #if defined(_WIN32) + bool addAllSubStreams = false; + bool needAltStreams = true; + #endif // _WIN32 + #endif // !defined(UNDER_CE) + + // check the path in inlcude rules + if (curNode.CheckPathToRoot(true, newParts, !fi.IsDir())) + { + #if !defined(UNDER_CE) + // dirItemIndex = (int)dirItems.Items.Size(); + #if defined(_WIN32) + // we will not check include rules for substreams. + addAllSubStreams = true; + #endif // _WIN32 + #endif // !defined(UNDER_CE) + + if (dirItems.CanIncludeItem(fi.IsDir())) + { + int secureIndex = -1; + #ifdef Z7_USE_SECURITY_CODE + if (dirItems.ReadSecure) + { + RINOK(dirItems.AddSecurityItem(phyPrefix + fi.Name, secureIndex)) + } + #endif + #if !defined(UNDER_CE) + dirItemIndex = (int)dirItems.Items.Size(); + #endif // !defined(UNDER_CE) + dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); + } + else + { + #if defined(_WIN32) && !defined(UNDER_CE) + needAltStreams = false; + #endif + } + + if (fi.IsDir()) + enterToSubFolders = true; + } + + #if !defined(UNDER_CE) + + // we don't scan AltStreams for link files + + if (dirItemIndex >= 0) + { + CDirItem &dirItem = dirItems.Items[(unsigned)dirItemIndex]; + RINOK(dirItems.SetLinkInfo(dirItem, fi, phyPrefix)) + if (dirItem.ReparseData.Size() != 0) + return S_OK; + } + + #if defined(_WIN32) + if (needAltStreams && dirItems.ScanAltStreams) + { + RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, + phyPrefix + fi.Name, // with (fi.Name) + newParts, // with (fi.Name) + addAllSubStreams, + dirItems)) + } + #endif + + #endif // !defined(UNDER_CE) + + + #ifndef _WIN32 + if (!fi.IsPosixLink()) // posix link can follow to dir + #endif + if (!fi.IsDir()) + return S_OK; + + const NWildcard::CCensorNode *nextNode = NULL; + + if (addParts.IsEmpty()) + { + int index = curNode.FindSubNode(name); + if (index >= 0) + { + nextNode = &curNode.SubNodes[(unsigned)index]; + newParts.Clear(); + } + } + + if (!nextNode) + { + if (!enterToSubFolders) + return S_OK; + + #ifndef _WIN32 + if (fi.IsPosixLink()) + { + // here we can try to resolve posix link + // if the link to dir, then can we follow it + return S_OK; // we don't follow posix link + } + #else + if (dirItems.SymLinks && fi.HasReparsePoint()) + { + /* 20.03: in SymLinks mode: we don't enter to directory that + has reparse point and has no CCensorNode + NOTE: (curNode and parent nodes) still can have wildcard rules + to include some items of target directory (of reparse point), + but we ignore these rules here. + */ + return S_OK; + } + #endif + nextNode = &curNode; + } + + return EnumerateDirItems_Spec( + *nextNode, phyParent, logParent, fi.Name, + phyPrefix, // without (fi.Name) + newParts, // relative to (*nextNode). (*nextNode + newParts) includes (fi.Name) + dirItems, + enterToSubFolders); +} + + +static bool CanUseFsDirect(const NWildcard::CCensorNode &curNode) +{ + FOR_VECTOR (i, curNode.IncludeItems) + { + const NWildcard::CItem &item = curNode.IncludeItems[i]; + if (item.Recursive || item.PathParts.Size() != 1) + return false; + const UString &name = item.PathParts.Front(); + /* + if (name.IsEmpty()) + return false; + */ + + /* Windows doesn't support file name with wildcard + But if another system supports file name with wildcard, + and wildcard mode is disabled, we can ignore wildcard in name + */ + /* + #ifndef _WIN32 + if (!item.WildcardParsing) + continue; + #endif + */ + if (DoesNameContainWildcard(name)) + return false; + } + return true; +} + + +#if defined(_WIN32) && !defined(UNDER_CE) + +static bool IsVirtualFsFolder(const FString &prefix, const UString &name) +{ + UString s = fs2us(prefix); + s += name; + s.Add_PathSepar(); + // it returns (true) for non real FS folder path like - "\\SERVER\" + return IsPathSepar(s[0]) && GetRootPrefixSize(s) == 0; +} + +#endif + + + +static HRESULT EnumerateDirItems( + const NWildcard::CCensorNode &curNode, + const int phyParent, const int logParent, const FString &phyPrefix, + const UStringVector &addParts, // prefix from curNode including + CDirItems &dirItems, + bool enterToSubFolders) +{ + if (!enterToSubFolders) + { + /* if there are IncludeItems censor rules that affect items in subdirs, + then we will enter to all subfolders */ + if (curNode.NeedCheckSubDirs()) + enterToSubFolders = true; + } + + RINOK(dirItems.ScanProgress(phyPrefix)) + + // try direct_names case at first + if (addParts.IsEmpty() && !enterToSubFolders) + { + if (CanUseFsDirect(curNode)) + { + // all names are direct (no wildcards) + // so we don't need file_system's dir enumerator + CRecordVector needEnterVector; + unsigned i; + + for (i = 0; i < curNode.IncludeItems.Size(); i++) + { + const NWildcard::CItem &item = curNode.IncludeItems[i]; + const UString &name = item.PathParts.Front(); + FString fullPath = phyPrefix + us2fs(name); + + /* + // not possible now + if (!item.ForDir && !item.ForFile) + { + RINOK(dirItems.AddError(fullPath, ERROR_INVALID_PARAMETER)); + continue; + } + */ + + #if defined(_WIN32) && !defined(UNDER_CE) + bool needAltStreams = true; + #endif + + #ifdef Z7_USE_SECURITY_CODE + bool needSecurity = true; + #endif + + if (phyPrefix.IsEmpty()) + { + if (!item.ForFile) + { + /* we don't like some names for alt streams inside archive: + ":sname" for "\" + "c:::sname" for "C:\" + So we ignore alt streams for these cases */ + if (name.IsEmpty()) + { + #if defined(_WIN32) && !defined(UNDER_CE) + needAltStreams = false; + #endif + + /* + // do we need to ignore security info for "\\" folder ? + #ifdef Z7_USE_SECURITY_CODE + needSecurity = false; + #endif + */ + + fullPath = CHAR_PATH_SEPARATOR; + } + #if defined(_WIN32) && !defined(UNDER_CE) + else if (item.IsDriveItem()) + { + needAltStreams = false; + fullPath.Add_PathSepar(); + } + #endif + } + } + + NFind::CFileInfo fi; + #if defined(_WIN32) && !defined(UNDER_CE) + if (IsVirtualFsFolder(phyPrefix, name)) + { + fi.SetAsDir(); + fi.Name = us2fs(name); + } + else + #endif + if (!FindFile_KeepDots(fi, fullPath FOLLOW_LINK_PARAM2)) + { + RINOK(dirItems.AddError(fullPath)) + continue; + } + + /* + #ifdef _WIN32 + #define MY_ERROR_IS_DIR ERROR_FILE_NOT_FOUND + #define MY_ERROR_NOT_DIR DI_DEFAULT_ERROR + #else + #define MY_ERROR_IS_DIR EISDIR + #define MY_ERROR_NOT_DIR ENOTDIR + #endif + */ + + const bool isDir = fi.IsDir(); + if (isDir ? !item.ForDir : !item.ForFile) + { + // RINOK(dirItems.AddError(fullPath, isDir ? MY_ERROR_IS_DIR: MY_ERROR_NOT_DIR)); + RINOK(dirItems.AddError(fullPath, DI_DEFAULT_ERROR)) + continue; + } + { + UStringVector pathParts; + pathParts.Add(fs2us(fi.Name)); + if (curNode.CheckPathToRoot(false, pathParts, !isDir)) + continue; + } + + + if (dirItems.CanIncludeItem(fi.IsDir())) + { + int secureIndex = -1; + #ifdef Z7_USE_SECURITY_CODE + if (needSecurity && dirItems.ReadSecure) + { + RINOK(dirItems.AddSecurityItem(fullPath, secureIndex)) + } + #endif + + dirItems.AddDirFileInfo(phyParent, logParent, secureIndex, fi); + + // we don't scan AltStreams for link files + + #if !defined(UNDER_CE) + { + CDirItem &dirItem = dirItems.Items.Back(); + RINOK(dirItems.SetLinkInfo(dirItem, fi, phyPrefix)) + if (dirItem.ReparseData.Size() != 0) + continue; + } + + #if defined(_WIN32) + if (needAltStreams && dirItems.ScanAltStreams) + { + UStringVector pathParts; + pathParts.Add(fs2us(fi.Name)); + RINOK(EnumerateAltStreams(fi, curNode, phyParent, logParent, + fullPath, // including (name) + pathParts, // including (fi.Name) + true, /* addAllSubStreams */ + dirItems)) + } + #endif // defined(_WIN32) + + #endif // !defined(UNDER_CE) + } + + + #ifndef _WIN32 + if (!fi.IsPosixLink()) // posix link can follow to dir + #endif + if (!isDir) + continue; + + UStringVector newParts; + const NWildcard::CCensorNode *nextNode = NULL; + int index = curNode.FindSubNode(name); + if (index >= 0) + { + for (int t = (int)needEnterVector.Size(); t <= index; t++) + needEnterVector.Add(true); + needEnterVector[(unsigned)index] = false; + nextNode = &curNode.SubNodes[(unsigned)index]; + } + else + { + #ifndef _WIN32 + if (fi.IsPosixLink()) + { + // here we can try to resolve posix link + // if the link to dir, then can we follow it + continue; // we don't follow posix link + } + #else + if (dirItems.SymLinks) + { + if (fi.HasReparsePoint()) + { + /* 20.03: in SymLinks mode: we don't enter to directory that + has reparse point and has no CCensorNode */ + continue; + } + } + #endif + nextNode = &curNode; + newParts.Add(name); // don't change it to fi.Name. It's for shortnames support + } + + RINOK(EnumerateDirItems_Spec(*nextNode, phyParent, logParent, fi.Name, phyPrefix, + newParts, dirItems, true)) + } + + for (i = 0; i < curNode.SubNodes.Size(); i++) + { + if (i < needEnterVector.Size()) + if (!needEnterVector[i]) + continue; + const NWildcard::CCensorNode &nextNode = curNode.SubNodes[i]; + FString fullPath = phyPrefix + us2fs(nextNode.Name); + NFind::CFileInfo fi; + + if (nextNode.Name.IsEmpty()) + { + if (phyPrefix.IsEmpty()) + fullPath = CHAR_PATH_SEPARATOR; + } + #ifdef _WIN32 + else if(phyPrefix.IsEmpty() + || (phyPrefix.Len() == NName::kSuperPathPrefixSize + && IsSuperPath(phyPrefix))) + { + if (NWildcard::IsDriveColonName(nextNode.Name)) + fullPath.Add_PathSepar(); + } + #endif + + // we don't want to call fi.Find() for root folder or virtual folder + if ((phyPrefix.IsEmpty() && nextNode.Name.IsEmpty()) + #if defined(_WIN32) && !defined(UNDER_CE) + || IsVirtualFsFolder(phyPrefix, nextNode.Name) + #endif + ) + { + fi.SetAsDir(); + fi.Name = us2fs(nextNode.Name); + } + else + { + if (!FindFile_KeepDots(fi, fullPath FOLLOW_LINK_PARAM2)) + { + if (!nextNode.AreThereIncludeItems()) + continue; + RINOK(dirItems.AddError(fullPath)) + continue; + } + + if (!fi.IsDir()) + { + RINOK(dirItems.AddError(fullPath, DI_DEFAULT_ERROR)) + continue; + } + } + + RINOK(EnumerateDirItems_Spec(nextNode, phyParent, logParent, fi.Name, phyPrefix, + UStringVector(), dirItems, false)) + } + + return S_OK; + } + } + + #ifdef _WIN32 + #ifndef UNDER_CE + + // scan drives, if wildcard is "*:\" + + if (phyPrefix.IsEmpty() && curNode.IncludeItems.Size() > 0) + { + unsigned i; + for (i = 0; i < curNode.IncludeItems.Size(); i++) + { + const NWildcard::CItem &item = curNode.IncludeItems[i]; + if (item.PathParts.Size() < 1) + break; + const UString &name = item.PathParts.Front(); + if (name.Len() != 2 || name[1] != ':') + break; + if (item.PathParts.Size() == 1) + if (item.ForFile || !item.ForDir) + break; + if (NWildcard::IsDriveColonName(name)) + continue; + if (name[0] != '*' && name[0] != '?') + break; + } + if (i == curNode.IncludeItems.Size()) + { + FStringVector driveStrings; + NFind::MyGetLogicalDriveStrings(driveStrings); + for (i = 0; i < driveStrings.Size(); i++) + { + FString driveName = driveStrings[i]; + if (driveName.Len() < 3 || driveName.Back() != '\\') + return E_FAIL; + driveName.DeleteBack(); + NFind::CFileInfo fi; + fi.SetAsDir(); + fi.Name = driveName; + + RINOK(EnumerateForItem(fi, curNode, phyParent, logParent, phyPrefix, + addParts, dirItems, enterToSubFolders)) + } + return S_OK; + } + } + + #endif + #endif + + + CObjectVector files; + + // for (int y = 0; y < 1; y++) + { + // files.Clear(); + RINOK(dirItems.EnumerateOneDir(phyPrefix, files)) + /* + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + // const NFind::CFileInfo &fi = files[i]; + #else + NFind::CFileInfo &fi = files[i]; + { + const NFind::CFileInfo &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(dirItems.AddError(path)); + continue; + } + fi.Name = di.Name; + } + #endif + + } + */ + } + + FOR_VECTOR (i, files) + { + #ifdef _WIN32 + const NFind::CFileInfo &fi = files[i]; + #else + const NFind::CFileInfo &fi = files[i]; + /* + NFind::CFileInfo fi; + { + const NFind::CDirEntry &di = files[i]; + const FString path = phyPrefix + di.Name; + if (!fi.Find_AfterEnumerator(path)) + { + RINOK(dirItems.AddError(path)); + continue; + } + fi.Name = di.Name; + } + */ + #endif + + RINOK(EnumerateForItem(fi, curNode, phyParent, logParent, phyPrefix, + addParts, dirItems, enterToSubFolders)) + if (dirItems.Callback && (i & kScanProgressStepMask) == kScanProgressStepMask) + { + RINOK(dirItems.ScanProgress(phyPrefix)) + } + } + + return S_OK; +} + + + + +HRESULT EnumerateItems( + const NWildcard::CCensor &censor, + const NWildcard::ECensorPathMode pathMode, + const UString &addPathPrefix, // prefix that will be added to Logical Path + CDirItems &dirItems) +{ + FOR_VECTOR (i, censor.Pairs) + { + const NWildcard::CPair &pair = censor.Pairs[i]; + const int phyParent = pair.Prefix.IsEmpty() ? -1 : (int)dirItems.AddPrefix(-1, -1, pair.Prefix); + int logParent = -1; + + if (pathMode == NWildcard::k_AbsPath) + logParent = phyParent; + else + { + if (!addPathPrefix.IsEmpty()) + logParent = (int)dirItems.AddPrefix(-1, -1, addPathPrefix); + } + + RINOK(EnumerateDirItems(pair.Head, phyParent, logParent, us2fs(pair.Prefix), UStringVector(), + dirItems, + false // enterToSubFolders + )) + } + dirItems.ReserveDown(); + + #if defined(_WIN32) && !defined(UNDER_CE) + RINOK(dirItems.FillFixedReparse()) + #endif + + #ifndef _WIN32 + RINOK(dirItems.FillDeviceSizes()) + #endif + + return S_OK; +} + + +#if defined(_WIN32) && !defined(UNDER_CE) + +HRESULT CDirItems::FillFixedReparse() +{ + FOR_VECTOR(i, Items) + { + CDirItem &item = Items[i]; + + if (!SymLinks) + { + // continue; // for debug + if (!item.Has_Attrib_ReparsePoint()) + continue; + + // if (item.IsDir()) continue; + + const FString phyPath = GetPhyPath(i); + + NFind::CFileInfo fi; + if (fi.Fill_From_ByHandleFileInfo(phyPath)) // item.IsDir() + { + item.Size = fi.Size; + item.CTime = fi.CTime; + item.ATime = fi.ATime; + item.MTime = fi.MTime; + item.Attrib = fi.Attrib; + continue; + } + + /* + // we request properties of target file instead of properies of symbolic link + // here we also can manually parse unsupported links (like WSL links) + NIO::CInFile inFile; + if (inFile.Open(phyPath)) + { + BY_HANDLE_FILE_INFORMATION info; + if (inFile.GetFileInformation(&info)) + { + // Stat.FilesSize doesn't contain item.Size already + // Stat.FilesSize -= item.Size; + item.Size = (((UInt64)info.nFileSizeHigh) << 32) + info.nFileSizeLow; + Stat.FilesSize += item.Size; + item.CTime = info.ftCreationTime; + item.ATime = info.ftLastAccessTime; + item.MTime = info.ftLastWriteTime; + item.Attrib = info.dwFileAttributes; + continue; + } + } + */ + + RINOK(AddError(phyPath)) + continue; + } + + // (SymLinks == true) here + + if (item.ReparseData.Size() == 0) + continue; + + // if (item.Size == 0) + { + // 20.03: we use Reparse Data instead of real data + item.Size = item.ReparseData.Size(); + } + + CReparseAttr attr; + if (!attr.Parse(item.ReparseData, item.ReparseData.Size())) + { + const FString phyPath = GetPhyPath(i); + AddError(phyPath, attr.ErrorCode); + continue; + } + + /* imagex/WIM reduces absolute paths in links (raparse data), + if we archive non root folder. We do same thing here */ + + bool isWSL = false; + if (attr.IsSymLink_WSL()) + { + // isWSL = true; + // we don't change WSL symlinks + continue; + } + else + { + if (attr.IsRelative_Win()) + continue; + } + + const UString &link = attr.GetPath(); + if (!IsDrivePath(link)) + continue; + // maybe we need to support networks paths also ? + + FString fullPathF; + if (!NDir::MyGetFullPathName(GetPhyPath(i), fullPathF)) + continue; + const UString fullPath = fs2us(fullPathF); + const UString logPath = GetLogPath(i); + if (logPath.Len() >= fullPath.Len()) + continue; + if (CompareFileNames(logPath, fullPath.RightPtr(logPath.Len())) != 0) + continue; + + const UString prefix = fullPath.Left(fullPath.Len() - logPath.Len()); + if (!IsPathSepar(prefix.Back())) + continue; + + const unsigned rootPrefixSize = GetRootPrefixSize(prefix); + if (rootPrefixSize == 0) + continue; + if (rootPrefixSize == prefix.Len()) + continue; // simple case: paths are from root + + if (link.Len() <= prefix.Len()) + continue; + + if (CompareFileNames(link.Left(prefix.Len()), prefix) != 0) + continue; + + UString newLink = prefix.Left(rootPrefixSize); + newLink += link.Ptr(prefix.Len()); + + CByteBuffer data; + bool isSymLink = !attr.IsMountPoint(); + if (!FillLinkData(data, newLink, isSymLink, isWSL)) + continue; + item.ReparseData2 = data; + } + return S_OK; +} + +#endif + + +#ifndef _WIN32 + +HRESULT CDirItems::FillDeviceSizes() +{ + { + FOR_VECTOR (i, Items) + { + CDirItem &item = Items[i]; + + if (S_ISBLK(item.mode) && item.Size == 0) + { + const FString phyPath = GetPhyPath(i); + NIO::CInFile inFile; + inFile.PreserveATime = true; + if (inFile.OpenShared(phyPath, ShareForWrite)) // fixme: OpenShared ?? + { + UInt64 size = 0; + if (inFile.GetLength(size)) + item.Size = size; + } + } + if (StoreOwnerName) + { + OwnerNameMap.Add_UInt32(item.uid); + OwnerGroupMap.Add_UInt32(item.gid); + } + } + } + + if (StoreOwnerName) + { + UString u; + AString a; + { + FOR_VECTOR (i, OwnerNameMap.Numbers) + { + // 200K/sec speed + u.Empty(); + const passwd *pw = getpwuid(OwnerNameMap.Numbers[i]); + // printf("\ngetpwuid=%s\n", pw->pw_name); + if (pw) + { + a = pw->pw_name; + ConvertUTF8ToUnicode(a, u); + } + OwnerNameMap.Strings.Add(u); + } + } + { + FOR_VECTOR (i, OwnerGroupMap.Numbers) + { + u.Empty(); + const group *gr = getgrgid(OwnerGroupMap.Numbers[i]); + if (gr) + { + // printf("\ngetgrgid %d %s\n", OwnerGroupMap.Numbers[i], gr->gr_name); + a = gr->gr_name; + ConvertUTF8ToUnicode(a, u); + } + OwnerGroupMap.Strings.Add(u); + } + } + + FOR_VECTOR (i, Items) + { + CDirItem &item = Items[i]; + { + const int index = OwnerNameMap.Find(item.uid); + if (index < 0) throw 1; + item.OwnerNameIndex = index; + } + { + const int index = OwnerGroupMap.Find(item.gid); + if (index < 0) throw 1; + item.OwnerGroupIndex = index; + } + } + } + + + // if (NeedOwnerNames) + { + /* + { + for (unsigned i = 0 ; i < 10000; i++) + { + const passwd *pw = getpwuid(i); + if (pw) + { + UString u; + ConvertUTF8ToUnicode(AString(pw->pw_name), u); + OwnerNameMap.Add(i, u); + OwnerNameMap.Add(i, u); + OwnerNameMap.Add(i, u); + } + const group *gr = getgrgid(i); + if (gr) + { + // we can use utf-8 here. + UString u; + ConvertUTF8ToUnicode(AString(gr->gr_name), u); + OwnerGroupMap.Add(i, u); + } + } + } + */ + /* + { + FOR_VECTOR (i, OwnerNameMap.Strings) + { + AString s; + ConvertUnicodeToUTF8(OwnerNameMap.Strings[i], s); + printf("\n%5d %s", (unsigned)OwnerNameMap.Numbers[i], s.Ptr()); + } + } + { + printf("\n\n=========Groups\n"); + FOR_VECTOR (i, OwnerGroupMap.Strings) + { + AString s; + ConvertUnicodeToUTF8(OwnerGroupMap.Strings[i], s); + printf("\n%5d %s", (unsigned)OwnerGroupMap.Numbers[i], s.Ptr()); + } + } + */ + } + /* + for (unsigned i = 0 ; i < 100000000; i++) + { + // const passwd *pw = getpwuid(1000); + // pw = pw; + int pos = OwnerNameMap.Find(1000); + if (pos < 0 - (int)i) + throw 1; + } + */ + + return S_OK; +} + +#endif + + + +static const char * const kCannotFindArchive = "Cannot find archive"; + +HRESULT EnumerateDirItemsAndSort( + NWildcard::CCensor &censor, + NWildcard::ECensorPathMode censorPathMode, + const UString &addPathPrefix, + UStringVector &sortedPaths, + UStringVector &sortedFullPaths, + CDirItemsStat &st, + IDirItemsCallback *callback) +{ + FStringVector paths; + + { + CDirItems dirItems; + dirItems.Callback = callback; + { + HRESULT res = EnumerateItems(censor, censorPathMode, addPathPrefix, dirItems); + st = dirItems.Stat; + RINOK(res) + } + + FOR_VECTOR (i, dirItems.Items) + { + const CDirItem &dirItem = dirItems.Items[i]; + if (!dirItem.IsDir()) + paths.Add(dirItems.GetPhyPath(i)); + } + } + + if (paths.Size() == 0) + { + // return S_OK; + throw CMessagePathException(kCannotFindArchive); + } + + UStringVector fullPaths; + + unsigned i; + + for (i = 0; i < paths.Size(); i++) + { + FString fullPath; + NFile::NDir::MyGetFullPathName(paths[i], fullPath); + fullPaths.Add(fs2us(fullPath)); + } + + CUIntVector indices; + SortFileNames(fullPaths, indices); + sortedPaths.ClearAndReserve(indices.Size()); + sortedFullPaths.ClearAndReserve(indices.Size()); + + for (i = 0; i < indices.Size(); i++) + { + unsigned index = indices[i]; + sortedPaths.AddInReserved(fs2us(paths[index])); + sortedFullPaths.AddInReserved(fullPaths[index]); + if (i > 0 && CompareFileNames(sortedFullPaths[i], sortedFullPaths[i - 1]) == 0) + throw CMessagePathException("Duplicate archive path:", sortedFullPaths[i]); + } + + return S_OK; +} + + + + +#ifdef _WIN32 + +static bool IsDotsName(const wchar_t *s) +{ + return s[0] == '.' && (s[1] == 0 || (s[1] == '.' && s[2] == 0)); +} + +// This code converts all short file names to long file names. + +static void ConvertToLongName(const UString &prefix, UString &name) +{ + if (name.IsEmpty() + || DoesNameContainWildcard(name) + || IsDotsName(name)) + return; + NFind::CFileInfo fi; + const FString path (us2fs(prefix + name)); + #ifndef UNDER_CE + if (NFile::NName::IsDevicePath(path)) + return; + #endif + if (fi.Find(path)) + name = fs2us(fi.Name); +} + +static void ConvertToLongNames(const UString &prefix, CObjectVector &items) +{ + FOR_VECTOR (i, items) + { + NWildcard::CItem &item = items[i]; + if (item.Recursive || item.PathParts.Size() != 1) + continue; + if (prefix.IsEmpty() && item.IsDriveItem()) + continue; + ConvertToLongName(prefix, item.PathParts.Front()); + } +} + +static void ConvertToLongNames(const UString &prefix, NWildcard::CCensorNode &node) +{ + ConvertToLongNames(prefix, node.IncludeItems); + ConvertToLongNames(prefix, node.ExcludeItems); + unsigned i; + for (i = 0; i < node.SubNodes.Size(); i++) + { + UString &name = node.SubNodes[i].Name; + if (prefix.IsEmpty() && NWildcard::IsDriveColonName(name)) + continue; + ConvertToLongName(prefix, name); + } + // mix folders with same name + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode1 = node.SubNodes[i]; + for (unsigned j = i + 1; j < node.SubNodes.Size();) + { + const NWildcard::CCensorNode &nextNode2 = node.SubNodes[j]; + if (nextNode1.Name.IsEqualTo_NoCase(nextNode2.Name)) + { + nextNode1.IncludeItems += nextNode2.IncludeItems; + nextNode1.ExcludeItems += nextNode2.ExcludeItems; + node.SubNodes.Delete(j); + } + else + j++; + } + } + for (i = 0; i < node.SubNodes.Size(); i++) + { + NWildcard::CCensorNode &nextNode = node.SubNodes[i]; + ConvertToLongNames(prefix + nextNode.Name + WCHAR_PATH_SEPARATOR, nextNode); + } +} + +void ConvertToLongNames(NWildcard::CCensor &censor) +{ + FOR_VECTOR (i, censor.Pairs) + { + NWildcard::CPair &pair = censor.Pairs[i]; + ConvertToLongNames(pair.Prefix, pair.Head); + } +} + +#endif + + +CMessagePathException::CMessagePathException(const char *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} + +CMessagePathException::CMessagePathException(const wchar_t *a, const wchar_t *u) +{ + (*this) += a; + if (u) + { + Add_LF(); + (*this) += u; + } +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/HashCalc.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/HashCalc.cpp new file mode 100644 index 000000000..cd43b1a18 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/HashCalc.cpp @@ -0,0 +1,2096 @@ +// HashCalc.cpp + +#include "StdAfx.h" + +#include "../../../../C/Alloc.h" +#include "../../../../C/CpuArch.h" + +#include "../../../Common/DynLimBuf.h" +#include "../../../Common/IntToString.h" +#include "../../../Common/StringToInt.h" + +#include "../../Common/FileStreams.h" +#include "../../Common/ProgressUtils.h" +#include "../../Common/StreamObjects.h" +#include "../../Common/StreamUtils.h" + +#include "../../Archive/Common/ItemNameUtils.h" +#include "../../Archive/IArchive.h" + +#include "EnumDirItems.h" +#include "HashCalc.h" + +using namespace NWindows; + +#ifdef Z7_EXTERNAL_CODECS +extern const CExternalCodecs *g_ExternalCodecs_Ptr; +#endif + +class CHashMidBuf +{ + void *_data; +public: + CHashMidBuf(): _data(NULL) {} + operator void *() { return _data; } + bool Alloc(size_t size) + { + if (_data) + return false; + _data = ::MidAlloc(size); + return _data != NULL; + } + ~CHashMidBuf() { ::MidFree(_data); } +}; + +static const char * const k_DefaultHashMethod = "CRC32"; + +HRESULT CHashBundle::SetMethods(DECL_EXTERNAL_CODECS_LOC_VARS const UStringVector &hashMethods) +{ + UStringVector names = hashMethods; + if (names.IsEmpty()) + names.Add(UString(k_DefaultHashMethod)); + + CRecordVector ids; + CObjectVector methods; + + unsigned i; + for (i = 0; i < names.Size(); i++) + { + COneMethodInfo m; + RINOK(m.ParseMethodFromString(names[i])) + + if (m.MethodName.IsEmpty()) + m.MethodName = k_DefaultHashMethod; + + if (m.MethodName == "*") + { + CRecordVector tempMethods; + GetHashMethods(EXTERNAL_CODECS_LOC_VARS tempMethods); + methods.Clear(); + ids.Clear(); + FOR_VECTOR (t, tempMethods) + { + unsigned index = ids.AddToUniqueSorted(tempMethods[t]); + if (ids.Size() != methods.Size()) + methods.Insert(index, m); + } + break; + } + else + { + // m.MethodName.RemoveChar(L'-'); + CMethodId id; + if (!FindHashMethod(EXTERNAL_CODECS_LOC_VARS m.MethodName, id)) + return E_NOTIMPL; + unsigned index = ids.AddToUniqueSorted(id); + if (ids.Size() != methods.Size()) + methods.Insert(index, m); + } + } + + for (i = 0; i < ids.Size(); i++) + { + CMyComPtr hasher; + AString name; + RINOK(CreateHasher(EXTERNAL_CODECS_LOC_VARS ids[i], name, hasher)) + if (!hasher) + throw "Can't create hasher"; + const COneMethodInfo &m = methods[i]; + { + CMyComPtr scp; + hasher.QueryInterface(IID_ICompressSetCoderProperties, &scp); + if (scp) + RINOK(m.SetCoderProps(scp, NULL)) + } + const UInt32 digestSize = hasher->GetDigestSize(); + if (digestSize > k_HashCalc_DigestSize_Max) + return E_NOTIMPL; + CHasherState &h = Hashers.AddNew(); + h.DigestSize = digestSize; + h.Hasher = hasher; + h.Name = name; + for (unsigned k = 0; k < k_HashCalc_NumGroups; k++) + h.InitDigestGroup(k); + } + + return S_OK; +} + +void CHashBundle::InitForNewFile() +{ + CurSize = 0; + FOR_VECTOR (i, Hashers) + { + CHasherState &h = Hashers[i]; + h.Hasher->Init(); + h.InitDigestGroup(k_HashCalc_Index_Current); + } +} + +void CHashBundle::Update(const void *data, UInt32 size) +{ + CurSize += size; + FOR_VECTOR (i, Hashers) + Hashers[i].Hasher->Update(data, size); +} + +void CHashBundle::SetSize(UInt64 size) +{ + CurSize = size; +} + +static void AddDigests(Byte *dest, const Byte *src, UInt32 size) +{ + unsigned next = 0; + /* + // we could use big-endian addition for sha-1 and sha-256 + // but another hashers are little-endian + if (size > 8) + { + for (unsigned i = size; i != 0;) + { + i--; + next += (unsigned)dest[i] + (unsigned)src[i]; + dest[i] = (Byte)next; + next >>= 8; + } + } + else + */ + { + for (unsigned i = 0; i < size; i++) + { + next += (unsigned)dest[i] + (unsigned)src[i]; + dest[i] = (Byte)next; + next >>= 8; + } + } + + // we use little-endian to store extra bytes + dest += k_HashCalc_DigestSize_Max; + for (unsigned i = 0; i < k_HashCalc_ExtraSize; i++) + { + next += (unsigned)dest[i]; + dest[i] = (Byte)next; + next >>= 8; + } +} + +void CHasherState::AddDigest(unsigned groupIndex, const Byte *data) +{ + NumSums[groupIndex]++; + AddDigests(Digests[groupIndex], data, DigestSize); +} + +void CHashBundle::Final(bool isDir, bool isAltStream, const UString &path) +{ + if (isDir) + NumDirs++; + else if (isAltStream) + { + NumAltStreams++; + AltStreamsSize += CurSize; + } + else + { + NumFiles++; + FilesSize += CurSize; + } + + Byte pre[16]; + memset(pre, 0, sizeof(pre)); + if (isDir) + pre[0] = 1; + + FOR_VECTOR (i, Hashers) + { + CHasherState &h = Hashers[i]; + if (!isDir) + { + h.Hasher->Final(h.Digests[0]); // k_HashCalc_Index_Current + if (!isAltStream) + h.AddDigest(k_HashCalc_Index_DataSum, h.Digests[0]); + } + + h.Hasher->Init(); + h.Hasher->Update(pre, sizeof(pre)); + h.Hasher->Update(h.Digests[0], h.DigestSize); + + for (unsigned k = 0; k < path.Len(); k++) + { + wchar_t c = path[k]; + + // 21.04: we want same hash for linux and windows paths + #if CHAR_PATH_SEPARATOR != '/' + if (c == CHAR_PATH_SEPARATOR) + c = '/'; + // if (c == (wchar_t)('\\' + 0xf000)) c = '\\'; // to debug WSL + // if (c > 0xf000 && c < 0xf080) c -= 0xf000; // to debug WSL + #endif + + Byte temp[2] = { (Byte)(c & 0xFF), (Byte)((c >> 8) & 0xFF) }; + h.Hasher->Update(temp, 2); + } + + Byte tempDigest[k_HashCalc_DigestSize_Max]; + h.Hasher->Final(tempDigest); + if (!isAltStream) + h.AddDigest(k_HashCalc_Index_NamesSum, tempDigest); + h.AddDigest(k_HashCalc_Index_StreamsSum, tempDigest); + } +} + + +static void CSum_Name_OriginalToEscape(const AString &src, AString &dest) +{ + dest.Empty(); + for (unsigned i = 0; i < src.Len();) + { + char c = src[i++]; + if (c == '\n') + { + dest.Add_Char('\\'); + c = 'n'; + } + else if (c == '\\') + dest.Add_Char('\\'); + dest.Add_Char(c); + } +} + + +static bool CSum_Name_EscapeToOriginal(const char *s, AString &dest) +{ + bool isOK = true; + dest.Empty(); + for (;;) + { + char c = *s++; + if (c == 0) + break; + if (c == '\\') + { + const char c1 = *s; + if (c1 == 'n') + { + c = '\n'; + s++; + } + else if (c1 == '\\') + { + c = c1; + s++; + } + else + { + // original md5sum returns NULL for such bad strings + isOK = false; + } + } + dest.Add_Char(c); + } + return isOK; +} + + + +static void SetSpacesAndNul(char *s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + +static const unsigned kHashColumnWidth_Min = 4 * 2; + +static unsigned GetColumnWidth(unsigned digestSize) +{ + const unsigned width = digestSize * 2; + return width < kHashColumnWidth_Min ? kHashColumnWidth_Min: width; +} + + +static void AddHashResultLine( + AString &_s, + // bool showHash, + // UInt64 fileSize, bool showSize, + const CObjectVector &hashers + // unsigned digestIndex, = k_HashCalc_Index_Current + ) +{ + FOR_VECTOR (i, hashers) + { + const CHasherState &h = hashers[i]; + char s[k_HashCalc_DigestSize_Max * 2 + 64]; + s[0] = 0; + // if (showHash) + HashHexToString(s, h.Digests[k_HashCalc_Index_Current], h.DigestSize); + const unsigned pos = (unsigned)strlen(s); + const int numSpaces = (int)GetColumnWidth(h.DigestSize) - (int)pos; + if (numSpaces > 0) + SetSpacesAndNul(s + pos, (unsigned)numSpaces); + if (i != 0) + _s.Add_Space(); + _s += s; + } + + /* + if (showSize) + { + _s.Add_Space(); + static const unsigned kSizeField_Len = 13; // same as in HashCon.cpp + char s[kSizeField_Len + 32]; + char *p = s; + SetSpacesAndNul(s, kSizeField_Len); + p = s + kSizeField_Len; + ConvertUInt64ToString(fileSize, p); + int numSpaces = (int)kSizeField_Len - (int)strlen(p); + if (numSpaces > 0) + p -= (unsigned)numSpaces; + _s += p; + } + */ +} + + +static void Add_LF(CDynLimBuf &hashFileString, const CHashOptionsLocal &options) +{ + hashFileString += (char)(options.HashMode_Zero.Val ? 0 : '\n'); +} + + + + +static void WriteLine(CDynLimBuf &hashFileString, + const CHashOptionsLocal &options, + const UString &path2, + bool isDir, + const AString &methodName, + const AString &hashesString) +{ + if (options.HashMode_OnlyHash.Val) + { + hashFileString += hashesString; + Add_LF(hashFileString, options); + return; + } + + UString path = path2; + + bool isBin = false; + const bool zeroMode = options.HashMode_Zero.Val; + const bool tagMode = options.HashMode_Tag.Val; + +#if CHAR_PATH_SEPARATOR != '/' + path.Replace(WCHAR_PATH_SEPARATOR, L'/'); + // path.Replace((wchar_t)('\\' + 0xf000), L'\\'); // to debug WSL +#endif + + AString utf8; + ConvertUnicodeToUTF8(path, utf8); + + AString esc; + CSum_Name_OriginalToEscape(utf8, esc); + + if (!zeroMode) + { + if (esc != utf8) + { + /* Original md5sum writes escape in that case. + We do same for compatibility with original md5sum. */ + hashFileString += '\\'; + } + } + + if (isDir && !esc.IsEmpty() && esc.Back() != '/') + esc.Add_Slash(); + + if (tagMode) + { + if (!methodName.IsEmpty()) + { + hashFileString += methodName; + hashFileString += ' '; + } + hashFileString += '('; + hashFileString += esc; + hashFileString += ')'; + hashFileString += " = "; + } + + hashFileString += hashesString; + + if (!tagMode) + { + hashFileString += ' '; + hashFileString += (char)(isBin ? '*' : ' '); + hashFileString += esc; + } + + Add_LF(hashFileString, options); +} + + + +static void WriteLine(CDynLimBuf &hashFileString, + const CHashOptionsLocal &options, + const UString &path, + bool isDir, + const CHashBundle &hb) +{ + AString methodName; + if (!hb.Hashers.IsEmpty()) + methodName = hb.Hashers[0].Name; + + AString hashesString; + AddHashResultLine(hashesString, hb.Hashers); + WriteLine(hashFileString, options, path, isDir, methodName, hashesString); +} + + +HRESULT HashCalc( + DECL_EXTERNAL_CODECS_LOC_VARS + const NWildcard::CCensor &censor, + const CHashOptions &options, + AString &errorInfo, + IHashCallbackUI *callback) +{ + CDirItems dirItems; + dirItems.Callback = callback; + + if (options.StdInMode) + { + CDirItem di; + if (!di.SetAs_StdInFile()) + return GetLastError_noZero_HRESULT(); + dirItems.Items.Add(di); + } + else + { + RINOK(callback->StartScanning()) + + dirItems.SymLinks = options.SymLinks.Val; + dirItems.ScanAltStreams = options.AltStreamsMode; + dirItems.ExcludeDirItems = censor.ExcludeDirItems; + dirItems.ExcludeFileItems = censor.ExcludeFileItems; + + dirItems.ShareForWrite = options.OpenShareForWrite; + + HRESULT res = EnumerateItems(censor, + options.PathMode, + UString(), + dirItems); + + if (res != S_OK) + { + if (res != E_ABORT) + errorInfo = "Scanning error"; + return res; + } + RINOK(callback->FinishScanning(dirItems.Stat)) + } + + unsigned i; + CHashBundle hb; + RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS options.Methods)) + // hb.Init(); + + hb.NumErrors = dirItems.Stat.NumErrors; + + UInt64 totalSize = 0; + if (options.StdInMode) + { + RINOK(callback->SetNumFiles(1)) + } + else + { + totalSize = dirItems.Stat.GetTotalBytes(); + RINOK(callback->SetTotal(totalSize)) + } + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + UInt64 completeValue = 0; + + RINOK(callback->BeforeFirstFile(hb)) + + /* + CDynLimBuf hashFileString((size_t)1 << 31); + const bool needGenerate = !options.HashFilePath.IsEmpty(); + */ + + for (i = 0; i < dirItems.Items.Size(); i++) + { + CMyComPtr inStream; + UString path; + bool isDir = false; + bool isAltStream = false; + + if (options.StdInMode) + { +#if 1 + inStream = new CStdInFileStream; +#else + if (!CreateStdInStream(inStream)) + { + const DWORD lastError = ::GetLastError(); + const HRESULT res = callback->OpenFileError(FString("stdin"), lastError); + hb.NumErrors++; + if (res != S_FALSE && res != S_OK) + return res; + continue; + } +#endif + } + else + { + path = dirItems.GetLogPath(i); + const CDirItem &di = dirItems.Items[i]; + #ifdef _WIN32 + isAltStream = di.IsAltStream; + #endif + + #ifndef UNDER_CE + // if (di.AreReparseData()) + if (di.ReparseData.Size() != 0) + { + CBufInStream *inStreamSpec = new CBufInStream(); + inStream = inStreamSpec; + inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); + } + else + #endif + { + CInFileStream *inStreamSpec = new CInFileStream; + inStreamSpec->Set_PreserveATime(options.PreserveATime); + inStream = inStreamSpec; + isDir = di.IsDir(); + if (!isDir) + { + const FString phyPath = dirItems.GetPhyPath(i); + if (!inStreamSpec->OpenShared(phyPath, options.OpenShareForWrite)) + { + const HRESULT res = callback->OpenFileError(phyPath, ::GetLastError()); + hb.NumErrors++; + if (res != S_FALSE) + return res; + continue; + } + if (!options.StdInMode) + { + UInt64 curSize = 0; + if (inStreamSpec->GetSize(&curSize) == S_OK) + { + if (curSize > di.Size) + { + totalSize += curSize - di.Size; + RINOK(callback->SetTotal(totalSize)) + // printf("\ntotal = %d MiB\n", (unsigned)(totalSize >> 20)); + } + } + } + // inStreamSpec->ReloadProps(); + } + } + } + + RINOK(callback->GetStream(path, isDir)) + UInt64 fileSize = 0; + + hb.InitForNewFile(); + + if (!isDir) + { + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + // printf("\ncompl = %d\n", (unsigned)(completeValue >> 20)); + RINOK(callback->SetCompleted(&completeValue)) + } + UInt32 size; + RINOK(inStream->Read(buf, kBufSize, &size)) + if (size == 0) + break; + hb.Update(buf, size); + fileSize += size; + completeValue += size; + } + } + + hb.Final(isDir, isAltStream, path); + + /* + if (needGenerate + && (options.HashMode_Dirs.Val || !isDir)) + { + WriteLine(hashFileString, + options, + path, // change it + isDir, + hb); + + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + */ + + RINOK(callback->SetOperationResult(fileSize, hb, !isDir)) + RINOK(callback->SetCompleted(&completeValue)) + } + + /* + if (needGenerate) + { + NFile::NIO::COutFile file; + if (!file.Create(us2fs(options.HashFilePath), true)) // createAlways + return GetLastError_noZero_HRESULT(); + if (!file.WriteFull(hashFileString, hashFileString.Len())) + return GetLastError_noZero_HRESULT(); + } + */ + + return callback->AfterLastFile(hb); +} + + +void HashHexToString(char *dest, const Byte *data, size_t size) +{ + if (!data) + { + for (size_t i = 0; i < size; i++) + { + dest[0] = ' '; + dest[1] = ' '; + dest += 2; + } + *dest = 0; + return; + } + + if (size > 8) + ConvertDataToHex_Lower(dest, data, size); + else if (size == 0) + { + *dest = 0; + return; + } + else + { + const char *dest_start = dest; + dest += size * 2; + *dest = 0; + do + { + const size_t b = *data++; + dest -= 2; + dest[0] = GET_HEX_CHAR_UPPER(b >> 4); + dest[1] = GET_HEX_CHAR_UPPER(b & 15); + } + while (dest != dest_start); + } +} + +void CHasherState::WriteToString(unsigned digestIndex, char *s) const +{ + HashHexToString(s, Digests[digestIndex], DigestSize); + + if (digestIndex != 0 && NumSums[digestIndex] != 1) + { + unsigned numExtraBytes = GetNumExtraBytes_for_Group(digestIndex); + if (numExtraBytes > 4) + numExtraBytes = 8; + else // if (numExtraBytes >= 0) + numExtraBytes = 4; + // if (numExtraBytes != 0) + { + s += strlen(s); + *s++ = '-'; + // *s = 0; + HashHexToString(s, GetExtraData_for_Group(digestIndex), numExtraBytes); + } + } +} + + + +// ---------- Hash Handler ---------- + +namespace NHash { + +#define IsWhite(c) ((c) == ' ' || (c) == '\t') + +bool CHashPair::IsDir() const +{ + if (Name.IsEmpty() || Name.Back() != '/') + return false; + // here we expect that Dir items contain only zeros or no Hash + for (size_t i = 0; i < Hash.Size(); i++) + if (Hash.ConstData()[i] != 0) + return false; + return true; +} + + +bool CHashPair::ParseCksum(const char *s) +{ + const char *end; + + const UInt32 crc = ConvertStringToUInt32(s, &end); + if (*end != ' ') + return false; + end++; + + const UInt64 size = ConvertStringToUInt64(end, &end); + if (*end != ' ') + return false; + end++; + + Name = end; + + Hash.Alloc(4); + SetBe32(Hash, crc) + + Size_from_Arc = size; + Size_from_Arc_Defined = true; + + return true; +} + + + +static const char *SkipWhite(const char *s) +{ + while (IsWhite(*s)) + s++; + return s; +} + +static const char * const k_CsumMethodNames[] = +{ + "sha256" + , "sha224" +// , "sha512/224" +// , "sha512/256" + , "sha512" + , "sha384" + , "sha1" + , "md5" + , "blake2b" + , "crc64" + , "crc32" + , "cksum" +}; + +static UString GetMethod_from_FileName(const UString &name) +{ + AString s; + ConvertUnicodeToUTF8(name, s); + const int dotPos = s.ReverseFind_Dot(); + const char *src = s.Ptr(); + bool isExtension = false; + if (dotPos >= 0) + { + isExtension = true; + src = s.Ptr(dotPos + 1); + } + const char *m = ""; + unsigned i; + for (i = 0; i < Z7_ARRAY_SIZE(k_CsumMethodNames); i++) + { + m = k_CsumMethodNames[i]; + if (isExtension) + { + if (StringsAreEqual_Ascii(src, m)) + break; + } + else if (IsString1PrefixedByString2_NoCase_Ascii(src, m)) + if (StringsAreEqual_Ascii(src + strlen(m), "sums")) + break; + } + UString res; + if (i != Z7_ARRAY_SIZE(k_CsumMethodNames)) + res = m; + return res; +} + + +bool CHashPair::Parse(const char *s) +{ + // here we keep compatibility with original md5sum / shasum + bool escape = false; + + s = SkipWhite(s); + + if (*s == '\\') + { + s++; + escape = true; + } + Escape = escape; + + // const char *kMethod = GetMethod_from_FileName(s); + // if (kMethod) + if ((size_t)(FindNonHexChar(s) - s) < 4) + { + // BSD-style checksum line + { + const char *s2 = s; + for (; *s2 != 0; s2++) + { + const char c = *s2; + if (c == 0) + return false; + if (c == ' ' || c == '(') + break; + } + Method.SetFrom(s, (unsigned)(s2 - s)); + s = s2; + } + IsBSD = true; + if (*s == ' ') + s++; + if (*s != '(') + return false; + s++; + { + const char *s2 = s; + for (; *s2 != 0; s2++) + {} + for (;;) + { + s2--; + if (s2 < s) + return false; + if (*s2 == ')') + break; + } + Name.SetFrom(s, (unsigned)(s2 - s)); + s = s2 + 1; + } + + s = SkipWhite(s); + if (*s != '=') + return false; + s++; + s = SkipWhite(s); + } + + { + const size_t numChars = (size_t)(FindNonHexChar(s) - s) & ~(size_t)1; + Hash.Alloc(numChars / 2); + if ((size_t)(ParseHexString(s, Hash) - Hash) != numChars / 2) + throw 101; + HashString.SetFrom(s, (unsigned)numChars); + s += numChars; + } + + if (IsBSD) + { + if (*s != 0) + return false; + if (escape) + { + const AString temp (Name); + return CSum_Name_EscapeToOriginal(temp, Name); + } + return true; + } + + if (*s == 0) + return true; + + if (*s != ' ') + return false; + s++; + const char c = *s; + if (c != ' ' + && c != '*' + && c != 'U' // shasum Universal + && c != '^' // shasum 0/1 + ) + return false; + Mode = c; + s++; + if (escape) + return CSum_Name_EscapeToOriginal(s, Name); + Name = s; + return true; +} + + +static bool GetLine(CByteBuffer &buf, bool zeroMode, bool cr_lf_Mode, size_t &posCur, AString &s) +{ + s.Empty(); + size_t pos = posCur; + const Byte *p = buf; + unsigned numDigits = 0; + for (; pos < buf.Size(); pos++) + { + const Byte b = p[pos]; + if (b == 0) + { + numDigits = 1; + break; + } + if (zeroMode) + continue; + if (b == 0x0a) + { + numDigits = 1; + break; + } + if (!cr_lf_Mode) + continue; + if (b == 0x0d) + { + if (pos + 1 >= buf.Size()) + { + numDigits = 1; + break; + // return false; + } + if (p[pos + 1] == 0x0a) + { + numDigits = 2; + break; + } + } + } + s.SetFrom((const char *)(p + posCur), (unsigned)(pos - posCur)); + posCur = pos + numDigits; + return true; +} + + +static bool Is_CR_LF_Data(const Byte *buf, size_t size) +{ + bool isCrLf = false; + for (size_t i = 0; i < size;) + { + const Byte b = buf[i]; + if (b == 0x0a) + return false; + if (b == 0x0d) + { + if (i == size - 1) + return false; + if (buf[i + 1] != 0x0a) + return false; + isCrLf = true; + i += 2; + } + else + i++; + } + return isCrLf; +} + + +static const Byte kArcProps[] = +{ + // kpidComment, + kpidCharacts +}; + +static const Byte kProps[] = +{ + kpidPath, + kpidSize, + kpidPackSize, + kpidMethod +}; + +static const Byte kRawProps[] = +{ + kpidChecksum +}; + + +Z7_COM7F_IMF(CHandler::GetParent(UInt32 /* index */ , UInt32 *parent, UInt32 *parentType)) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetNumRawProps(UInt32 *numProps)) +{ + *numProps = Z7_ARRAY_SIZE(kRawProps); + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetRawPropInfo(UInt32 index, BSTR *name, PROPID *propID)) +{ + *propID = kRawProps[index]; + *name = NULL; + return S_OK; +} + +Z7_COM7F_IMF(CHandler::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType)) +{ + *data = NULL; + *dataSize = 0; + *propType = 0; + + if (propID == kpidChecksum) + { + const CHashPair &hp = HashPairs[index]; + if (hp.Hash.Size() > 0) + { + *data = hp.Hash; + *dataSize = (UInt32)hp.Hash.Size(); + *propType = NPropDataType::kRaw; + } + return S_OK; + } + + return S_OK; +} + +IMP_IInArchive_Props +IMP_IInArchive_ArcProps + +Z7_COM7F_IMF(CHandler::GetNumberOfItems(UInt32 *numItems)) +{ + *numItems = HashPairs.Size(); + return S_OK; +} + +static void Add_OptSpace_String(UString &dest, const char *src) +{ + dest.Add_Space_if_NotEmpty(); + dest += src; +} + +Z7_COM7F_IMF(CHandler::GetArchiveProperty(PROPID propID, PROPVARIANT *value)) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidPhySize: if (_phySize != 0) prop = _phySize; break; + /* + case kpidErrorFlags: + { + UInt32 v = 0; + if (!_isArc) v |= kpv_ErrorFlags_IsNotArc; + // if (_sres == k_Base64_RES_NeedMoreInput) v |= kpv_ErrorFlags_UnexpectedEnd; + if (v != 0) + prop = v; + break; + } + */ + case kpidCharacts: + { + UString s; + if (_hashSize_Defined) + { + s.Add_Space_if_NotEmpty(); + s.Add_UInt32(_hashSize * 8); + s += "-bit"; + } + if (!_nameExtenstion.IsEmpty()) + { + s.Add_Space_if_NotEmpty(); + s += _nameExtenstion; + } + if (_is_PgpMethod) + { + Add_OptSpace_String(s, "PGP"); + if (!_pgpMethod.IsEmpty()) + { + s.Add_Colon(); + s += _pgpMethod; + } + } + if (_is_ZeroMode) + Add_OptSpace_String(s, "ZERO"); + if (_are_there_Tags) + Add_OptSpace_String(s, "TAG"); + if (_are_there_Dirs) + Add_OptSpace_String(s, "DIRS"); + prop = s; + break; + } + + case kpidReadOnly: + { + if (_isArc) + if (!CanUpdate()) + prop = true; + break; + } + default: break; + } + prop.Detach(value); + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) +{ + // COM_TRY_BEGIN + NCOM::CPropVariant prop; + const CHashPair &hp = HashPairs[index]; + switch (propID) + { + case kpidIsDir: + { + prop = hp.IsDir(); + break; + } + case kpidPath: + { + UString path; + hp.Get_UString_Path(path); + + bool useBackslashReplacement = true; + if (_supportWindowsBackslash && !hp.Escape && path.Find(L"\\\\") < 0) + { +#if WCHAR_PATH_SEPARATOR == L'/' + path.Replace(L'\\', L'/'); +#else + useBackslashReplacement = false; +#endif + } + NArchive::NItemName::ReplaceToOsSlashes_Remove_TailSlash( + path, useBackslashReplacement); + prop = path; + break; + } + case kpidSize: + { + // client needs processed size of last file + if (hp.Size_from_Disk_Defined) + prop = (UInt64)hp.Size_from_Disk; + else if (hp.Size_from_Arc_Defined) + prop = (UInt64)hp.Size_from_Arc; + break; + } + case kpidPackSize: + { + prop = (UInt64)hp.Hash.Size(); + break; + } + case kpidMethod: + { + if (!hp.Method.IsEmpty()) + prop = hp.Method; + break; + } + default: break; + } + prop.Detach(value); + return S_OK; + // COM_TRY_END +} + + +static HRESULT ReadStream_to_Buf(IInStream *stream, CByteBuffer &buf, IArchiveOpenCallback *openCallback) +{ + buf.Free(); + UInt64 len; + RINOK(InStream_AtBegin_GetSize(stream, len)) + if (len == 0 || len >= ((UInt64)1 << 31)) + return S_FALSE; + buf.Alloc((size_t)len); + UInt64 pos = 0; + // return ReadStream_FALSE(stream, buf, (size_t)len); + for (;;) + { + const UInt32 kBlockSize = ((UInt32)1 << 24); + const UInt32 curSize = (len < kBlockSize) ? (UInt32)len : kBlockSize; + UInt32 processedSizeLoc; + RINOK(stream->Read((Byte *)buf + pos, curSize, &processedSizeLoc)) + if (processedSizeLoc == 0) + return E_FAIL; + len -= processedSizeLoc; + pos += processedSizeLoc; + if (len == 0) + return S_OK; + if (openCallback) + { + const UInt64 files = 0; + RINOK(openCallback->SetCompleted(&files, &pos)) + } + } +} + + +Z7_COM7F_IMF(CHandler::Open(IInStream *stream, const UInt64 *, IArchiveOpenCallback *openCallback)) +{ + COM_TRY_BEGIN + { + Close(); + + CByteBuffer buf; + RINOK(ReadStream_to_Buf(stream, buf, openCallback)) + + CObjectVector &pairs = HashPairs; + + bool zeroMode = false; + bool cr_lf_Mode = false; + { + for (size_t i = 0; i < buf.Size(); i++) + if (buf.ConstData()[i] == 0) + { + zeroMode = true; + break; + } + } + _is_ZeroMode = zeroMode; + if (!zeroMode) + cr_lf_Mode = Is_CR_LF_Data(buf, buf.Size()); + + if (openCallback) + { + Z7_DECL_CMyComPtr_QI_FROM( + IArchiveOpenVolumeCallback, + openVolumeCallback, openCallback) + if (openVolumeCallback) + { + NCOM::CPropVariant prop; + RINOK(openVolumeCallback->GetProperty(kpidName, &prop)) + if (prop.vt == VT_BSTR) + _nameExtenstion = GetMethod_from_FileName(prop.bstrVal); + } + } + + bool cksumMode = false; + if (_nameExtenstion.IsEqualTo_Ascii_NoCase("cksum")) + cksumMode = true; + _is_CksumMode = cksumMode; + + size_t pos = 0; + AString s; + bool minusMode = false; + unsigned numLines = 0; + + while (pos < buf.Size()) + { + if (!GetLine(buf, zeroMode, cr_lf_Mode, pos, s)) + return S_FALSE; + numLines++; + if (s.IsEmpty()) + continue; + + if (s.IsPrefixedBy_Ascii_NoCase("; ")) + { + if (numLines != 1) + return S_FALSE; + // comment line of FileVerifier++ + continue; + } + + if (s.IsPrefixedBy_Ascii_NoCase("-----")) + { + if (minusMode) + break; // end of pgp mode + minusMode = true; + if (s.IsPrefixedBy_Ascii_NoCase("-----BEGIN PGP SIGNED MESSAGE")) + { + if (_is_PgpMethod) + return S_FALSE; + if (!GetLine(buf, zeroMode, cr_lf_Mode, pos, s)) + return S_FALSE; + const char *kStart = "Hash: "; + if (!s.IsPrefixedBy_Ascii_NoCase(kStart)) + return S_FALSE; + _pgpMethod = s.Ptr((unsigned)strlen(kStart)); + _is_PgpMethod = true; + } + continue; + } + + CHashPair pair; + pair.FullLine = s; + if (cksumMode) + { + if (!pair.ParseCksum(s)) + return S_FALSE; + } + else if (!pair.Parse(s)) + return S_FALSE; + pairs.Add(pair); + } + + { + unsigned hashSize = 0; + bool hashSize_Dismatch = false; + for (unsigned i = 0; i < HashPairs.Size(); i++) + { + const CHashPair &hp = HashPairs[i]; + if (i == 0) + hashSize = (unsigned)hp.Hash.Size(); + else + if (hashSize != hp.Hash.Size()) + hashSize_Dismatch = true; + + if (hp.IsBSD) + _are_there_Tags = true; + if (!_are_there_Dirs && hp.IsDir()) + _are_there_Dirs = true; + } + if (!hashSize_Dismatch && hashSize != 0) + { + _hashSize = hashSize; + _hashSize_Defined = true; + } + } + + _phySize = buf.Size(); + _isArc = true; + return S_OK; + } + COM_TRY_END +} + + +void CHandler::ClearVars() +{ + _phySize = 0; + _isArc = false; + _is_CksumMode = false; + _is_PgpMethod = false; + _is_ZeroMode = false; + _are_there_Tags = false; + _are_there_Dirs = false; + _hashSize_Defined = false; + _hashSize = 0; +} + + +Z7_COM7F_IMF(CHandler::Close()) +{ + ClearVars(); + _nameExtenstion.Empty(); + _pgpMethod.Empty(); + HashPairs.Clear(); + return S_OK; +} + + +static bool CheckDigests(const Byte *a, const Byte *b, size_t size) +{ + if (size <= 8) + { + /* we use reversed order for one digest, when text representation + uses big-order for crc-32 and crc-64 */ + for (size_t i = 0; i < size; i++) + if (a[i] != b[size - 1 - i]) + return false; + return true; + } + { + for (size_t i = 0; i < size; i++) + if (a[i] != b[i]) + return false; + return true; + } +} + + +static void AddDefaultMethod(UStringVector &methods, unsigned size) +{ + const char *m = NULL; + if (size == 32) m = "sha256"; + else if (size == 20) m = "sha1"; + else if (size == 16) m = "md5"; + else if (size == 8) m = "crc64"; + else if (size == 4) m = "crc32"; + else + return; + #ifdef Z7_EXTERNAL_CODECS + const CExternalCodecs *_externalCodecs = g_ExternalCodecs_Ptr; + #endif + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS + AString(m), id)) + methods.Add(UString(m)); +} + + +Z7_COM7F_IMF(CHandler::Extract(const UInt32 *indices, UInt32 numItems, + Int32 testMode, IArchiveExtractCallback *extractCallback)) +{ + COM_TRY_BEGIN + + /* + if (testMode == 0) + return E_NOTIMPL; + */ + + const bool allFilesMode = (numItems == (UInt32)(Int32)-1); + if (allFilesMode) + numItems = HashPairs.Size(); + if (numItems == 0) + return S_OK; + + #ifdef Z7_EXTERNAL_CODECS + const CExternalCodecs *_externalCodecs = g_ExternalCodecs_Ptr; + #endif + + CHashBundle hb_Glob; + // UStringVector methods = options.Methods; + UStringVector methods; + + if (methods.IsEmpty() && !_nameExtenstion.IsEmpty()) + { + AString utf; + ConvertUnicodeToUTF8(_nameExtenstion, utf); + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS utf, id)) + methods.Add(_nameExtenstion); + } + + if (methods.IsEmpty() && !_pgpMethod.IsEmpty()) + { + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS _pgpMethod, id)) + methods.Add(UString(_pgpMethod)); + } + + if (methods.IsEmpty() && _pgpMethod.IsEmpty() && _hashSize_Defined) + AddDefaultMethod(methods, _hashSize); + + RINOK(hb_Glob.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods)) + + Z7_DECL_CMyComPtr_QI_FROM( + IArchiveUpdateCallbackFile, + updateCallbackFile, extractCallback) + if (!updateCallbackFile) + return E_NOTIMPL; + { + Z7_DECL_CMyComPtr_QI_FROM( + IArchiveGetDiskProperty, + GetDiskProperty, extractCallback) + if (GetDiskProperty) + { + UInt64 totalSize = 0; + UInt32 i; + for (i = 0; i < numItems; i++) + { + const UInt32 index = allFilesMode ? i : indices[i]; + const CHashPair &hp = HashPairs[index]; + if (hp.IsDir()) + continue; + { + NCOM::CPropVariant prop; + RINOK(GetDiskProperty->GetDiskProperty(index, kpidSize, &prop)) + if (prop.vt != VT_UI8) + continue; + totalSize += prop.uhVal.QuadPart; + } + } + RINOK(extractCallback->SetTotal(totalSize)) + // RINOK(Hash_SetTotalUnpacked->Hash_SetTotalUnpacked(indices, numItems)); + } + } + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + CMyComPtr2_Create lps; + lps->Init(extractCallback, false); + + for (UInt32 i = 0;; i++) + { + RINOK(lps->SetCur()) + if (i >= numItems) + break; + const UInt32 index = allFilesMode ? i : indices[i]; + + CHashPair &hp = HashPairs[index]; + + UString path; + hp.Get_UString_Path(path); + + CMyComPtr inStream; + const bool isDir = hp.IsDir(); + if (!isDir) + { + RINOK(updateCallbackFile->GetStream2(index, &inStream, NUpdateNotifyOp::kHashRead)) + if (!inStream) + { + continue; // we have shown error in GetStream2() + } + // askMode = NArchive::NExtract::NAskMode::kSkip; + } + + Int32 askMode = testMode ? + NArchive::NExtract::NAskMode::kTest : + NArchive::NExtract::NAskMode::kExtract; + + CMyComPtr realOutStream; + RINOK(extractCallback->GetStream(index, &realOutStream, askMode)) + + /* PrepareOperation() can expect kExtract to set + Attrib and security of output file */ + askMode = NArchive::NExtract::NAskMode::kReadExternal; + + RINOK(extractCallback->PrepareOperation(askMode)) + + const bool isAltStream = false; + + UInt64 fileSize = 0; + + CHashBundle hb_Loc; + + CHashBundle *hb_Use = &hb_Glob; + + HRESULT res_SetMethods = S_OK; + + UStringVector methods_loc; + + if (!hp.Method.IsEmpty()) + { + hb_Use = &hb_Loc; + CMethodId id; + if (FindHashMethod(EXTERNAL_CODECS_LOC_VARS hp.Method, id)) + { + methods_loc.Add(UString(hp.Method)); + RINOK(hb_Loc.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods_loc)) + } + else + res_SetMethods = E_NOTIMPL; + } + else if (methods.IsEmpty()) + { + AddDefaultMethod(methods_loc, (unsigned)hp.Hash.Size()); + if (!methods_loc.IsEmpty()) + { + hb_Use = &hb_Loc; + RINOK(hb_Loc.SetMethods( + EXTERNAL_CODECS_LOC_VARS + methods_loc)) + } + } + + const bool isSupportedMode = hp.IsSupportedMode(); + hb_Use->InitForNewFile(); + + if (inStream) + { + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + RINOK(lps.Interface()->SetRatioInfo(NULL, &fileSize)) + } + UInt32 size; + RINOK(inStream->Read(buf, kBufSize, &size)) + if (size == 0) + break; + hb_Use->Update(buf, size); + if (realOutStream) + { + RINOK(WriteStream(realOutStream, buf, size)) + } + fileSize += size; + } + + hp.Size_from_Disk = fileSize; + hp.Size_from_Disk_Defined = true; + } + + realOutStream.Release(); + inStream.Release(); + + lps->InSize += hp.Hash.Size(); + lps->OutSize += fileSize; + + hb_Use->Final(isDir, isAltStream, path); + + Int32 opRes = NArchive::NExtract::NOperationResult::kUnsupportedMethod; + if (isSupportedMode + && res_SetMethods != E_NOTIMPL + && hb_Use->Hashers.Size() > 0 + ) + { + const CHasherState &hs = hb_Use->Hashers[0]; + if (hs.DigestSize == hp.Hash.Size()) + { + opRes = NArchive::NExtract::NOperationResult::kCRCError; + if (CheckDigests(hp.Hash, hs.Digests[0], hs.DigestSize)) + if (!hp.Size_from_Arc_Defined || hp.Size_from_Arc == fileSize) + opRes = NArchive::NExtract::NOperationResult::kOK; + } + } + + RINOK(extractCallback->SetOperationResult(opRes)) + } + + return S_OK; + COM_TRY_END +} + + +// ---------- UPDATE ---------- + +struct CUpdateItem +{ + int IndexInArc; + unsigned IndexInClient; + UInt64 Size; + bool NewData; + bool NewProps; + bool IsDir; + UString Path; + + CUpdateItem(): Size(0), IsDir(false) {} +}; + + +static HRESULT GetPropString(IArchiveUpdateCallback *callback, UInt32 index, PROPID propId, + UString &res, + bool convertSlash) +{ + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(index, propId, &prop)) + if (prop.vt == VT_BSTR) + { + res = prop.bstrVal; + if (convertSlash) + NArchive::NItemName::ReplaceSlashes_OsToUnix(res); + } + else if (prop.vt != VT_EMPTY) + return E_INVALIDARG; + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::GetFileTimeType(UInt32 *type)) +{ + *type = NFileTimeType::kUnix; + return S_OK; +} + + +Z7_COM7F_IMF(CHandler::UpdateItems(ISequentialOutStream *outStream, UInt32 numItems, + IArchiveUpdateCallback *callback)) +{ + COM_TRY_BEGIN + + if (_isArc && !CanUpdate()) + return E_NOTIMPL; + + /* + Z7_DECL_CMyComPtr_QI_FROM(IArchiveUpdateCallbackArcProp, + reportArcProp, callback) + */ + + CObjectVector updateItems; + + UInt64 complexity = 0; + + UInt32 i; + for (i = 0; i < numItems; i++) + { + CUpdateItem ui; + Int32 newData; + Int32 newProps; + UInt32 indexInArc; + + if (!callback) + return E_FAIL; + + RINOK(callback->GetUpdateItemInfo(i, &newData, &newProps, &indexInArc)) + + ui.NewProps = IntToBool(newProps); + ui.NewData = IntToBool(newData); + ui.IndexInArc = (int)indexInArc; + ui.IndexInClient = i; + if (IntToBool(newProps)) + { + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidIsDir, &prop)) + if (prop.vt == VT_EMPTY) + ui.IsDir = false; + else if (prop.vt != VT_BOOL) + return E_INVALIDARG; + else + ui.IsDir = (prop.boolVal != VARIANT_FALSE); + } + + RINOK(GetPropString(callback, i, kpidPath, ui.Path, + true)) // convertSlash + /* + if (ui.IsDir && !ui.Name.IsEmpty() && ui.Name.Back() != '/') + ui.Name += '/'; + */ + } + + if (IntToBool(newData)) + { + NCOM::CPropVariant prop; + RINOK(callback->GetProperty(i, kpidSize, &prop)) + if (prop.vt == VT_UI8) + { + ui.Size = prop.uhVal.QuadPart; + complexity += ui.Size; + } + else if (prop.vt == VT_EMPTY) + ui.Size = (UInt64)(Int64)-1; + else + return E_INVALIDARG; + } + + updateItems.Add(ui); + } + + if (complexity != 0) + { + RINOK(callback->SetTotal(complexity)) + } + + #ifdef Z7_EXTERNAL_CODECS + const CExternalCodecs *_externalCodecs = g_ExternalCodecs_Ptr; + #endif + + CHashBundle hb; + UStringVector methods; + if (!_methods.IsEmpty()) + { + FOR_VECTOR(k, _methods) + { + methods.Add(_methods[k]); + } + } + else if (_crcSize_WasSet) + { + AddDefaultMethod(methods, _crcSize); + } + else + { + Z7_DECL_CMyComPtr_QI_FROM( + IArchiveGetRootProps, + getRootProps, callback) + if (getRootProps) + { + NCOM::CPropVariant prop; + RINOK(getRootProps->GetRootProp(kpidArcFileName, &prop)) + if (prop.vt == VT_BSTR) + { + const UString method = GetMethod_from_FileName(prop.bstrVal); + if (!method.IsEmpty()) + methods.Add(method); + } + } + } + + RINOK(hb.SetMethods(EXTERNAL_CODECS_LOC_VARS methods)) + + CMyComPtr2_Create lps; + lps->Init(callback, true); + + const UInt32 kBufSize = 1 << 15; + CHashMidBuf buf; + if (!buf.Alloc(kBufSize)) + return E_OUTOFMEMORY; + + CDynLimBuf hashFileString((size_t)1 << 31); + + CHashOptionsLocal options = _options; + + if (_isArc) + { + if (!options.HashMode_Zero.Def && _is_ZeroMode) + options.HashMode_Zero.Val = true; + if (!options.HashMode_Tag.Def && _are_there_Tags) + options.HashMode_Tag.Val = true; + if (!options.HashMode_Dirs.Def && _are_there_Dirs) + options.HashMode_Dirs.Val = true; + } + if (options.HashMode_OnlyHash.Val && updateItems.Size() != 1) + options.HashMode_OnlyHash.Val = false; + + complexity = 0; + + for (i = 0; i < updateItems.Size(); i++) + { + lps->InSize = complexity; + RINOK(lps->SetCur()) + + const CUpdateItem &ui = updateItems[i]; + + /* + CHashPair item; + if (!ui.NewProps) + item = HashPairs[(unsigned)ui.IndexInArc]; + */ + + if (ui.NewData) + { + UInt64 currentComplexity = ui.Size; + UInt64 fileSize = 0; + + CMyComPtr fileInStream; + bool needWrite = true; + { + HRESULT res = callback->GetStream(ui.IndexInClient, &fileInStream); + + if (res == S_FALSE) + needWrite = false; + else + { + RINOK(res) + + if (fileInStream) + { + Z7_DECL_CMyComPtr_QI_FROM( + IStreamGetSize, + streamGetSize, fileInStream) + if (streamGetSize) + { + UInt64 size; + if (streamGetSize->GetSize(&size) == S_OK) + currentComplexity = size; + } + /* + Z7_DECL_CMyComPtr_QI_FROM( + IStreamGetProps, + getProps, fileInStream) + if (getProps) + { + FILETIME mTime; + UInt64 size2; + if (getProps->GetProps(&size2, NULL, NULL, &mTime, NULL) == S_OK) + { + currentComplexity = size2; + // item.MTime = NTime::FileTimeToUnixTime64(mTime);; + } + } + */ + } + else + { + currentComplexity = 0; + } + } + } + + hb.InitForNewFile(); + const bool isDir = ui.IsDir; + + if (needWrite && fileInStream && !isDir) + { + for (UInt32 step = 0;; step++) + { + if ((step & 0xFF) == 0) + { + RINOK(lps.Interface()->SetRatioInfo(&fileSize, NULL)) + // RINOK(callback->SetCompleted(&completeValue)); + } + UInt32 size; + RINOK(fileInStream->Read(buf, kBufSize, &size)) + if (size == 0) + break; + hb.Update(buf, size); + fileSize += size; + } + currentComplexity = fileSize; + } + + fileInStream.Release(); + const bool isAltStream = false; + hb.Final(isDir, isAltStream, ui.Path); + + if (options.HashMode_Dirs.Val || !isDir) + { + if (!hb.Hashers.IsEmpty()) + lps->OutSize += hb.Hashers[0].DigestSize; + WriteLine(hashFileString, + options, + ui.Path, + isDir, + hb); + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + + complexity += currentComplexity; + + /* + if (reportArcProp) + { + PROPVARIANT prop; + prop.vt = VT_EMPTY; + prop.wReserved1 = 0; + + NCOM::PropVarEm_Set_UInt64(&prop, fileSize); + RINOK(reportArcProp->ReportProp(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, kpidSize, &prop)); + + for (unsigned k = 0; k < hb.Hashers.Size(); k++) + { + const CHasherState &hs = hb.Hashers[k]; + + if (hs.DigestSize == 4 && hs.Name.IsEqualTo_Ascii_NoCase("crc32")) + { + NCOM::PropVarEm_Set_UInt32(&prop, GetUi32(hs.Digests[k_HashCalc_Index_Current])); + RINOK(reportArcProp->ReportProp(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, kpidCRC, &prop)); + } + else + { + RINOK(reportArcProp->ReportRawProp(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, + kpidChecksum, hs.Digests[k_HashCalc_Index_Current], + hs.DigestSize, NPropDataType::kRaw)); + } + RINOK(reportArcProp->ReportFinished(NArchive::NEventIndexType::kOutArcIndex, ui.IndexInClient, NArchive::NUpdate::NOperationResult::kOK)); + } + } + */ + RINOK(callback->SetOperationResult(NArchive::NUpdate::NOperationResult::kOK)) + } + else + { + // old data + const CHashPair &existItem = HashPairs[(unsigned)ui.IndexInArc]; + if (ui.NewProps) + { + WriteLine(hashFileString, + options, + ui.Path, + ui.IsDir, + existItem.Method, existItem.HashString + ); + } + else + { + hashFileString += existItem.FullLine; + Add_LF(hashFileString, options); + } + } + if (hashFileString.IsError()) + return E_OUTOFMEMORY; + } + + RINOK(WriteStream(outStream, hashFileString, hashFileString.Len())) + + return S_OK; + COM_TRY_END +} + + + +HRESULT CHandler::SetProperty(const wchar_t *nameSpec, const PROPVARIANT &value) +{ + UString name = nameSpec; + name.MakeLower_Ascii(); + if (name.IsEmpty()) + return E_INVALIDARG; + + if (name.IsEqualTo("m")) // "hm" hash method + { + // COneMethodInfo omi; + // RINOK(omi.ParseMethodFromPROPVARIANT(L"", value)); + // _methods.Add(omi.MethodName); // change it. use omi.PropsString + if (value.vt != VT_BSTR) + return E_INVALIDARG; + UString s (value.bstrVal); + _methods.Add(s); + return S_OK; + } + + if (name.IsEqualTo("flags")) + { + if (value.vt != VT_BSTR) + return E_INVALIDARG; + if (!_options.ParseString(value.bstrVal)) + return E_INVALIDARG; + return S_OK; + } + + if (name.IsEqualTo("backslash")) + return PROPVARIANT_to_bool(value, _supportWindowsBackslash); + + if (name.IsPrefixedBy_Ascii_NoCase("crc")) + { + name.Delete(0, 3); + _crcSize = 4; + _crcSize_WasSet = true; + return ParsePropToUInt32(name, value, _crcSize); + } + + // common properties + if (name.IsPrefixedBy_Ascii_NoCase("mt") + || name.IsPrefixedBy_Ascii_NoCase("memuse")) + return S_OK; + + return E_INVALIDARG; +} + + +Z7_COM7F_IMF(CHandler::SetProperties(const wchar_t * const *names, const PROPVARIANT *values, UInt32 numProps)) +{ + COM_TRY_BEGIN + + InitProps(); + + for (UInt32 i = 0; i < numProps; i++) + { + RINOK(SetProperty(names[i], values[i])) + } + return S_OK; + COM_TRY_END +} + +CHandler::CHandler() +{ + ClearVars(); + InitProps(); +} + +} + + + +static IInArchive *CreateHashHandler_In() { return new NHash::CHandler; } +static IOutArchive *CreateHashHandler_Out() { return new NHash::CHandler; } + +void Codecs_AddHashArcHandler(CCodecs *codecs) +{ + { + CArcInfoEx item; + + item.Name = "Hash"; + item.CreateInArchive = CreateHashHandler_In; + item.CreateOutArchive = CreateHashHandler_Out; + item.IsArcFunc = NULL; + item.Flags = + NArcInfoFlags::kKeepName + | NArcInfoFlags::kStartOpen + | NArcInfoFlags::kByExtOnlyOpen + // | NArcInfoFlags::kPureStartOpen + | NArcInfoFlags::kHashHandler + ; + + // ubuntu uses "SHA256SUMS" file + item.AddExts(UString ( + "sha256 sha512 sha224 sha384 sha1 sha md5" + // "b2sum" + " crc32 crc64" + " asc" + " cksum" + ), + UString()); + + item.UpdateEnabled = (item.CreateOutArchive != NULL); + item.SignatureOffset = 0; + // item.Version = MY_VER_MIX; + item.NewInterface = true; + + item.Signatures.AddNew().CopyFrom(NULL, 0); + + codecs->Formats.Add(item); + } +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/LoadCodecs.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/LoadCodecs.cpp index 585614c69..462836605 100644 --- a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/LoadCodecs.cpp +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/LoadCodecs.cpp @@ -68,18 +68,28 @@ using namespace NFile; #define kCodecsFolderName FTEXT("Codecs") #define kFormatsFolderName FTEXT("Formats") - +// **************** NanaZip Modification Start **************** +//static CFSTR const kMainDll = +// #ifdef _WIN32 +// FTEXT("7z.dll"); +// #else +// FTEXT("7z.so"); +// #endif static CFSTR const kMainDll = #ifdef _WIN32 - FTEXT("7z.dll"); + FTEXT("NanaZip.Core.dll"); #else - FTEXT("7z.so"); + FTEXT("NanaZip.Core.so"); #endif +// **************** NanaZip Modification End **************** #ifdef _WIN32 -static LPCTSTR const kRegistryPath = TEXT("Software") TEXT(STRING_PATH_SEPARATOR) TEXT("7-zip"); +// **************** NanaZip Modification Start **************** +//static LPCTSTR const kRegistryPath = TEXT("Software") TEXT(STRING_PATH_SEPARATOR) TEXT("7-zip"); +static LPCTSTR const kRegistryPath = TEXT("Software") TEXT(STRING_PATH_SEPARATOR) TEXT("NanaZip"); +// **************** NanaZip Modification End **************** static LPCWSTR const kProgramPathValue = L"Path"; static LPCWSTR const kProgramPath2Value = L"Path" #ifdef _WIN64 @@ -832,6 +842,14 @@ HRESULT CCodecs::Load() #ifdef Z7_EXTERNAL_CODECS const FString baseFolder = GetBaseFolderPrefixFromRegistry(); + // **************** NanaZip Modification Start **************** + { + bool loadedOK; + RINOK(LoadDll(baseFolder + L"NanaZip.Codecs.dll", false, &loadedOK)); + if (!loadedOK) + MainDll_ErrorPath = L"NanaZip.Codecs.dll"; + } + // **************** NanaZip Modification End **************** { bool loadedOK; RINOK(LoadDll(baseFolder + kMainDll, false, &loadedOK)) diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/SetProperties.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/SetProperties.cpp new file mode 100644 index 000000000..3364be73b --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/SetProperties.cpp @@ -0,0 +1,88 @@ +// SetProperties.cpp + +#include "StdAfx.h" + +#include "../../../Common/MyCom.h" +#include "../../../Common/MyString.h" +#include "../../../Common/StringToInt.h" + +#include "../../../Windows/PropVariant.h" + +#include "../../Archive/IArchive.h" + +#include "SetProperties.h" + +using namespace NWindows; +using namespace NCOM; + +static void ParseNumberString(const UString &s, NCOM::CPropVariant &prop) +{ + const wchar_t *end; + const UInt64 result = ConvertStringToUInt64(s, &end); + if (*end != 0 || s.IsEmpty()) + prop = s; + else if (result <= (UInt32)0xFFFFFFFF) + prop = (UInt32)result; + else + prop = result; +} + + +struct CPropPropetiesVector +{ + CPropVariant *values; + CPropPropetiesVector(unsigned num) + { + values = new CPropVariant[num]; + } + ~CPropPropetiesVector() + { + delete []values; + } +}; + + +HRESULT SetProperties(IUnknown *unknown, const CObjectVector &properties) +{ + if (properties.IsEmpty()) + return S_OK; + Z7_DECL_CMyComPtr_QI_FROM( + ISetProperties, + setProperties, unknown) + if (!setProperties) + return S_OK; + + UStringVector realNames; + CPropPropetiesVector values(properties.Size()); + { + unsigned i; + for (i = 0; i < properties.Size(); i++) + { + const CProperty &property = properties[i]; + NCOM::CPropVariant propVariant; + UString name = property.Name; + if (property.Value.IsEmpty()) + { + if (!name.IsEmpty()) + { + const wchar_t c = name.Back(); + if (c == L'-') + propVariant = false; + else if (c == L'+') + propVariant = true; + if (propVariant.vt != VT_EMPTY) + name.DeleteBack(); + } + } + else + ParseNumberString(property.Value, propVariant); + realNames.Add(name); + values.values[i] = propVariant; + } + CRecordVector names; + for (i = 0; i < realNames.Size(); i++) + names.Add((const wchar_t *)realNames[i]); + + return setProperties->SetProperties(names.ConstData(), values.values, names.Size()); + } +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.cpp new file mode 100644 index 000000000..16ea6cd15 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.cpp @@ -0,0 +1,19 @@ +// TempFiles.cpp + +#include "StdAfx.h" + +#include "../../../Windows/FileDir.h" + +#include "TempFiles.h" + +using namespace NWindows; +using namespace NFile; + +void CTempFiles::Clear() +{ + while (!Paths.IsEmpty()) + { + NDir::DeleteFileAlways(Paths.Back()); + Paths.DeleteBack(); + } +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.h b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.h new file mode 100644 index 000000000..c7b9ce67d --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/TempFiles.h @@ -0,0 +1,16 @@ +// TempFiles.h + +#ifndef ZIP7_INC_TEMP_FILES_H +#define ZIP7_INC_TEMP_FILES_H + +#include "../../../Common/MyString.h" + +class CTempFiles +{ + void Clear(); +public: + FStringVector Paths; + ~CTempFiles() { Clear(); } +}; + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Update.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Update.cpp new file mode 100644 index 000000000..06f1576bf --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/Update.cpp @@ -0,0 +1,1880 @@ +// Update.cpp + +#include "StdAfx.h" + +// #include + +#include "Update.h" + +#include "../../../Common/StringConvert.h" + +#include "../../../Windows/DLL.h" +#include "../../../Windows/FileDir.h" +#include "../../../Windows/FileFind.h" +#include "../../../Windows/FileName.h" +#include "../../../Windows/PropVariant.h" +#include "../../../Windows/PropVariantConv.h" +#include "../../../Windows/TimeUtils.h" + +#include "../../Common/FileStreams.h" +#include "../../Common/LimitedStreams.h" +#include "../../Common/MultiOutStream.h" +#include "../../Common/StreamUtils.h" + +#include "../../Compress/CopyCoder.h" + +#include "../Common/DirItem.h" +#include "../Common/EnumDirItems.h" +#include "../Common/OpenArchive.h" +#include "../Common/UpdateProduce.h" + +#include "EnumDirItems.h" +#include "SetProperties.h" +#include "TempFiles.h" +#include "UpdateCallback.h" + +static const char * const kUpdateIsNotSupoorted = + "update operations are not supported for this archive"; + +static const char * const kUpdateIsNotSupported_MultiVol = + "Updating for multivolume archives is not implemented"; + +using namespace NWindows; +using namespace NCOM; +using namespace NFile; +using namespace NDir; +using namespace NName; + +#ifdef _WIN32 +static CFSTR const kTempFolderPrefix = FTEXT("7zE"); +#endif + +void CUpdateErrorInfo::SetFromLastError(const char *message) +{ + SystemError = ::GetLastError(); + Message = message; +} + +HRESULT CUpdateErrorInfo::SetFromLastError(const char *message, const FString &fileName) +{ + SetFromLastError(message); + FileNames.Add(fileName); + return Get_HRESULT_Error(); +} + +HRESULT CUpdateErrorInfo::SetFromError_DWORD(const char *message, const FString &fileName, DWORD error) +{ + Message = message; + FileNames.Add(fileName); + SystemError = error; + return Get_HRESULT_Error(); +} + + +using namespace NUpdateArchive; + +struct CMultiOutStream_Rec +{ + CMultiOutStream *Spec; + CMyComPtr Ref; +}; + +struct CMultiOutStream_Bunch +{ + CObjectVector Items; + + HRESULT Destruct() + { + HRESULT hres = S_OK; + FOR_VECTOR (i, Items) + { + CMultiOutStream_Rec &rec = Items[i]; + if (rec.Ref) + { + const HRESULT hres2 = rec.Spec->Destruct(); + if (hres == S_OK) + hres = hres2; + } + } + Items.Clear(); + return hres; + } + + void DisableDeletion() + { + FOR_VECTOR (i, Items) + { + CMultiOutStream_Rec &rec = Items[i]; + if (rec.Ref) + rec.Spec->NeedDelete = false; + } + } +}; + + +void CArchivePath::ParseFromPath(const UString &path, EArcNameMode mode) +{ + OriginalPath = path; + + SplitPathToParts_2(path, Prefix, Name); + + if (mode == k_ArcNameMode_Add) + return; + + if (mode != k_ArcNameMode_Exact) + { + int dotPos = Name.ReverseFind_Dot(); + if (dotPos < 0) + return; + if ((unsigned)dotPos == Name.Len() - 1) + Name.DeleteBack(); + else + { + const UString ext = Name.Ptr((unsigned)(dotPos + 1)); + if (BaseExtension.IsEqualTo_NoCase(ext)) + { + BaseExtension = ext; + Name.DeleteFrom((unsigned)dotPos); + return; + } + } + } + + BaseExtension.Empty(); +} + +UString CArchivePath::GetFinalPath() const +{ + UString path = GetPathWithoutExt(); + if (!BaseExtension.IsEmpty()) + { + path.Add_Dot(); + path += BaseExtension; + } + return path; +} + +UString CArchivePath::GetFinalVolPath() const +{ + UString path = GetPathWithoutExt(); + // if BaseExtension is empty, we must ignore VolExtension also. + if (!BaseExtension.IsEmpty()) + { + path.Add_Dot(); + path += VolExtension; + } + return path; +} + +FString CArchivePath::GetTempPath() const +{ + FString path = TempPrefix; + path += us2fs(Name); + if (!BaseExtension.IsEmpty()) + { + path.Add_Dot(); + path += us2fs(BaseExtension); + } + path += ".tmp"; + path += TempPostfix; + return path; +} + +static const char * const kDefaultArcType = "7z"; +static const char * const kDefaultArcExt = "7z"; +static const char * const kSFXExtension = + #ifdef _WIN32 + "exe"; + #else + ""; + #endif + +bool CUpdateOptions::InitFormatIndex(const CCodecs *codecs, + const CObjectVector &types, const UString &arcPath) +{ + if (types.Size() > 1) + return false; + // int arcTypeIndex = -1; + if (types.Size() != 0) + { + MethodMode.Type = types[0]; + MethodMode.Type_Defined = true; + } + if (MethodMode.Type.FormatIndex < 0) + { + // MethodMode.Type = -1; + MethodMode.Type = COpenType(); + if (ArcNameMode != k_ArcNameMode_Add) + { + MethodMode.Type.FormatIndex = codecs->FindFormatForArchiveName(arcPath); + if (MethodMode.Type.FormatIndex >= 0) + MethodMode.Type_Defined = true; + } + } + return true; +} + +bool CUpdateOptions::SetArcPath(const CCodecs *codecs, const UString &arcPath) +{ + UString typeExt; + int formatIndex = MethodMode.Type.FormatIndex; + if (formatIndex < 0) + { + typeExt = kDefaultArcExt; + } + else + { + const CArcInfoEx &arcInfo = codecs->Formats[(unsigned)formatIndex]; + if (!arcInfo.UpdateEnabled) + return false; + typeExt = arcInfo.GetMainExt(); + } + UString ext = typeExt; + if (SfxMode) + ext = kSFXExtension; + ArchivePath.BaseExtension = ext; + ArchivePath.VolExtension = typeExt; + ArchivePath.ParseFromPath(arcPath, ArcNameMode); + FOR_VECTOR (i, Commands) + { + CUpdateArchiveCommand &uc = Commands[i]; + uc.ArchivePath.BaseExtension = ext; + uc.ArchivePath.VolExtension = typeExt; + uc.ArchivePath.ParseFromPath(uc.UserArchivePath, ArcNameMode); + } + return true; +} + + +struct CUpdateProduceCallbackImp Z7_final: public IUpdateProduceCallback +{ + const CObjectVector *_arcItems; + CDirItemsStat *_stat; + IUpdateCallbackUI *_callback; + + CUpdateProduceCallbackImp( + const CObjectVector *a, + CDirItemsStat *stat, + IUpdateCallbackUI *callback): + _arcItems(a), + _stat(stat), + _callback(callback) {} + + virtual HRESULT ShowDeleteFile(unsigned arcIndex) Z7_override; +}; + + +HRESULT CUpdateProduceCallbackImp::ShowDeleteFile(unsigned arcIndex) +{ + const CArcItem &ai = (*_arcItems)[arcIndex]; + { + CDirItemsStat &stat = *_stat; + if (ai.IsDir) + stat.NumDirs++; + else if (ai.IsAltStream) + { + stat.NumAltStreams++; + stat.AltStreamsSize += ai.Size; + } + else + { + stat.NumFiles++; + stat.FilesSize += ai.Size; + } + } + return _callback->ShowDeleteFile(ai.Name, ai.IsDir); +} + +bool CRenamePair::Prepare() +{ + if (RecursedType != NRecursedType::kNonRecursed) + return false; + if (!WildcardParsing) + return true; + return !DoesNameContainWildcard(OldName); +} + +extern bool g_CaseSensitive; + +static unsigned CompareTwoNames(const wchar_t *s1, const wchar_t *s2) +{ + for (unsigned i = 0;; i++) + { + wchar_t c1 = s1[i]; + wchar_t c2 = s2[i]; + if (c1 == 0 || c2 == 0) + return i; + if (c1 == c2) + continue; + if (!g_CaseSensitive && (MyCharUpper(c1) == MyCharUpper(c2))) + continue; + if (IsPathSepar(c1) && IsPathSepar(c2)) + continue; + return i; + } +} + +bool CRenamePair::GetNewPath(bool isFolder, const UString &src, UString &dest) const +{ + unsigned num = CompareTwoNames(OldName, src); + if (OldName[num] == 0) + { + if (src[num] != 0 && !IsPathSepar(src[num]) && num != 0 && !IsPathSepar(src[num - 1])) + return false; + } + else + { + // OldName[num] != 0 + // OldName = "1\1a.txt" + // src = "1" + + if (!isFolder + || src[num] != 0 + || !IsPathSepar(OldName[num]) + || OldName[num + 1] != 0) + return false; + } + dest = NewName + src.Ptr(num); + return true; +} + +#ifdef SUPPORT_ALT_STREAMS +int FindAltStreamColon_in_Path(const wchar_t *path); +#endif + + + +static HRESULT Compress( + const CUpdateOptions &options, + bool isUpdatingItself, + CCodecs *codecs, + const CActionSet &actionSet, + const CArc *arc, + CArchivePath &archivePath, + const CObjectVector &arcItems, + Byte *processedItemsStatuses, + const CDirItems &dirItems, + const CDirItem *parentDirItem, + CTempFiles &tempFiles, + CMultiOutStream_Bunch &multiStreams, + CUpdateErrorInfo &errorInfo, + IUpdateCallbackUI *callback, + CFinishArchiveStat &st) +{ + CMyComPtr outArchive; + int formatIndex = options.MethodMode.Type.FormatIndex; + + if (arc) + { + formatIndex = arc->FormatIndex; + if (formatIndex < 0) + return E_NOTIMPL; + CMyComPtr archive2 = arc->Archive; + HRESULT result = archive2.QueryInterface(IID_IOutArchive, &outArchive); + if (result != S_OK) + throw kUpdateIsNotSupoorted; + } + else + { + RINOK(codecs->CreateOutArchive((unsigned)formatIndex, outArchive)) + + #ifdef Z7_EXTERNAL_CODECS + { + CMyComPtr setCompressCodecsInfo; + outArchive.QueryInterface(IID_ISetCompressCodecsInfo, (void **)&setCompressCodecsInfo); + if (setCompressCodecsInfo) + { + RINOK(setCompressCodecsInfo->SetCompressCodecsInfo(codecs)) + } + } + #endif + } + + if (!outArchive) + throw kUpdateIsNotSupoorted; + + // we need to set properties to get fileTimeType. + RINOK(SetProperties(outArchive, options.MethodMode.Properties)) + + NFileTimeType::EEnum fileTimeType; + { + /* + how we compare file_in_archive::MTime with dirItem.MTime + for GetUpdatePairInfoList(): + + if (kpidMTime is not defined), external MTime of archive is used. + + before 22.00: + if (kpidTimeType is defined) + { + kpidTimeType is used as precision. + (kpidTimeType > kDOS) is not allowed. + } + else GetFileTimeType() value is used as precision. + + 22.00: + if (kpidMTime is defined) + { + if (kpidMTime::precision != 0), then kpidMTime::precision is used as precision. + else + { + if (kpidTimeType is defined), kpidTimeType is used as precision. + else GetFileTimeType() value is used as precision. + } + } + else external MTime of archive is used as precision. + */ + + UInt32 value; + RINOK(outArchive->GetFileTimeType(&value)) + + // we support any future fileType here. + fileTimeType = (NFileTimeType::EEnum)value; + + /* + old 21.07 code: + switch (value) + { + case NFileTimeType::kWindows: + case NFileTimeType::kUnix: + case NFileTimeType::kDOS: + fileTimeType = (NFileTimeType::EEnum)value; + break; + default: + return E_FAIL; + } + */ + } + + // bool noTimestampExpected = false; + { + const CArcInfoEx &arcInfo = codecs->Formats[(unsigned)formatIndex]; + + // if (arcInfo.Flags_KeepName()) noTimestampExpected = true; + if (arcInfo.Is_Xz() || + arcInfo.Is_BZip2()) + { + /* 7-zip before 22.00 returns NFileTimeType::kUnix for xz and bzip2, + but we want to set timestamp without reduction to unix. */ + // noTimestampExpected = true; + fileTimeType = NFileTimeType::kNotDefined; // it means not defined + } + + if (options.AltStreams.Val && !arcInfo.Flags_AltStreams()) + return E_NOTIMPL; + if (options.NtSecurity.Val && !arcInfo.Flags_NtSecurity()) + return E_NOTIMPL; + if (options.DeleteAfterCompressing && arcInfo.Flags_HashHandler()) + return E_NOTIMPL; + } + + CRecordVector updatePairs2; + + UStringVector newNames; + + CArcToDoStat stat2; + + if (options.RenamePairs.Size() != 0) + { + FOR_VECTOR (i, arcItems) + { + const CArcItem &ai = arcItems[i]; + bool needRename = false; + UString dest; + + if (ai.Censored) + { + FOR_VECTOR (j, options.RenamePairs) + { + const CRenamePair &rp = options.RenamePairs[j]; + if (rp.GetNewPath(ai.IsDir, ai.Name, dest)) + { + needRename = true; + break; + } + + #ifdef SUPPORT_ALT_STREAMS + if (ai.IsAltStream) + { + int colonPos = FindAltStreamColon_in_Path(ai.Name); + if (colonPos >= 0) + { + UString mainName = ai.Name.Left((unsigned)colonPos); + /* + actually we must improve that code to support cases + with folder renaming like: rn arc dir1\ dir2\ + */ + if (rp.GetNewPath(false, mainName, dest)) + { + needRename = true; + dest.Add_Colon(); + dest += ai.Name.Ptr((unsigned)(colonPos + 1)); + break; + } + } + } + #endif + } + } + + CUpdatePair2 up2; + up2.SetAs_NoChangeArcItem(ai.IndexInServer); + if (needRename) + { + up2.NewProps = true; + RINOK(arc->IsItem_Anti(i, up2.IsAnti)) + up2.NewNameIndex = (int)newNames.Add(dest); + } + updatePairs2.Add(up2); + } + } + else + { + CRecordVector updatePairs; + GetUpdatePairInfoList(dirItems, arcItems, fileTimeType, updatePairs); // must be done only once!!! + CUpdateProduceCallbackImp upCallback(&arcItems, &stat2.DeleteData, callback); + + UpdateProduce(updatePairs, actionSet, updatePairs2, isUpdatingItself ? &upCallback : NULL); + } + + { + FOR_VECTOR (i, updatePairs2) + { + const CUpdatePair2 &up = updatePairs2[i]; + + // 17.01: anti-item is (up.NewData && (p.UseArcProps in most cases)) + + if (up.NewData && !up.UseArcProps) + { + if (up.ExistOnDisk()) + { + CDirItemsStat2 &stat = stat2.NewData; + const CDirItem &di = dirItems.Items[(unsigned)up.DirIndex]; + if (di.IsDir()) + { + if (up.IsAnti) + stat.Anti_NumDirs++; + else + stat.NumDirs++; + } + #ifdef _WIN32 + else if (di.IsAltStream) + { + if (up.IsAnti) + stat.Anti_NumAltStreams++; + else + { + stat.NumAltStreams++; + stat.AltStreamsSize += di.Size; + } + } + #endif + else + { + if (up.IsAnti) + stat.Anti_NumFiles++; + else + { + stat.NumFiles++; + stat.FilesSize += di.Size; + } + } + } + } + else if (up.ArcIndex >= 0) + { + CDirItemsStat2 &stat = *(up.NewData ? &stat2.NewData : &stat2.OldData); + const CArcItem &ai = arcItems[(unsigned)up.ArcIndex]; + if (ai.IsDir) + { + if (up.IsAnti) + stat.Anti_NumDirs++; + else + stat.NumDirs++; + } + else if (ai.IsAltStream) + { + if (up.IsAnti) + stat.Anti_NumAltStreams++; + else + { + stat.NumAltStreams++; + stat.AltStreamsSize += ai.Size; + } + } + else + { + if (up.IsAnti) + stat.Anti_NumFiles++; + else + { + stat.NumFiles++; + stat.FilesSize += ai.Size; + } + } + } + } + RINOK(callback->SetNumItems(stat2)) + } + + CArchiveUpdateCallback *updateCallbackSpec = new CArchiveUpdateCallback; + CMyComPtr updateCallback(updateCallbackSpec); + + updateCallbackSpec->PreserveATime = options.PreserveATime; + updateCallbackSpec->ShareForWrite = options.OpenShareForWrite; + updateCallbackSpec->StopAfterOpenError = options.StopAfterOpenError; + updateCallbackSpec->StdInMode = options.StdInMode; + updateCallbackSpec->Callback = callback; + + if (arc) + { + // we set Archive to allow to transfer GetProperty requests back to DLL. + updateCallbackSpec->Archive = arc->Archive; + } + + updateCallbackSpec->DirItems = &dirItems; + updateCallbackSpec->ParentDirItem = parentDirItem; + + updateCallbackSpec->StoreNtSecurity = options.NtSecurity.Val; + updateCallbackSpec->StoreHardLinks = options.HardLinks.Val; + updateCallbackSpec->StoreSymLinks = options.SymLinks.Val; + updateCallbackSpec->StoreOwnerName = options.StoreOwnerName.Val; + updateCallbackSpec->StoreOwnerId = options.StoreOwnerId.Val; + + updateCallbackSpec->Arc = arc; + updateCallbackSpec->ArcItems = &arcItems; + updateCallbackSpec->UpdatePairs = &updatePairs2; + + updateCallbackSpec->ProcessedItemsStatuses = processedItemsStatuses; + + { + const UString arcPath = archivePath.GetFinalPath(); + updateCallbackSpec->ArcFileName = ExtractFileNameFromPath(arcPath); + } + + if (options.RenamePairs.Size() != 0) + updateCallbackSpec->NewNames = &newNames; + + if (options.SetArcMTime) + { + // updateCallbackSpec->Need_ArcMTime_Report = true; + updateCallbackSpec->Need_LatestMTime = true; + } + + CMyComPtr outSeekStream; + CMyComPtr outStream; + + if (!options.StdOutMode) + { + FString dirPrefix; + if (!GetOnlyDirPrefix(us2fs(archivePath.GetFinalPath()), dirPrefix)) + throw 1417161; + CreateComplexDir(dirPrefix); + } + + COutFileStream *outStreamSpec = NULL; + CStdOutFileStream *stdOutFileStreamSpec = NULL; + CMultiOutStream *volStreamSpec = NULL; + + if (options.VolumesSizes.Size() == 0) + { + if (options.StdOutMode) + { + stdOutFileStreamSpec = new CStdOutFileStream; + outStream = stdOutFileStreamSpec; + } + else + { + outStreamSpec = new COutFileStream; + outSeekStream = outStreamSpec; + outStream = outSeekStream; + bool isOK = false; + FString realPath; + + for (unsigned i = 0; i < (1 << 16); i++) + { + if (archivePath.Temp) + { + if (i > 0) + { + archivePath.TempPostfix.Empty(); + archivePath.TempPostfix.Add_UInt32(i); + } + realPath = archivePath.GetTempPath(); + } + else + realPath = us2fs(archivePath.GetFinalPath()); + if (outStreamSpec->Create_NEW(realPath)) + { + tempFiles.Paths.Add(realPath); + isOK = true; + break; + } + if (::GetLastError() != ERROR_FILE_EXISTS) + break; + if (!archivePath.Temp) + break; + } + + if (!isOK) + return errorInfo.SetFromLastError("cannot open file", realPath); + } + } + else + { + if (options.StdOutMode) + return E_FAIL; + if (arc && arc->GetGlobalOffset() > 0) + return E_NOTIMPL; + + volStreamSpec = new CMultiOutStream(); + outSeekStream = volStreamSpec; + outStream = outSeekStream; + volStreamSpec->Prefix = us2fs(archivePath.GetFinalVolPath()); + volStreamSpec->Prefix.Add_Dot(); + volStreamSpec->Init(options.VolumesSizes); + { + CMultiOutStream_Rec &rec = multiStreams.Items.AddNew(); + rec.Spec = volStreamSpec; + rec.Ref = rec.Spec; + } + + /* + updateCallbackSpec->VolumesSizes = volumesSizes; + updateCallbackSpec->VolName = archivePath.Prefix + archivePath.Name; + if (!archivePath.VolExtension.IsEmpty()) + updateCallbackSpec->VolExt = UString('.') + archivePath.VolExtension; + */ + } + + if (options.SfxMode) + { + CInFileStream *sfxStreamSpec = new CInFileStream; + CMyComPtr sfxStream(sfxStreamSpec); + if (!sfxStreamSpec->Open(options.SfxModule)) + return errorInfo.SetFromLastError("cannot open SFX module", options.SfxModule); + + CMyComPtr sfxOutStream; + COutFileStream *outStreamSpec2 = NULL; + if (options.VolumesSizes.Size() == 0) + sfxOutStream = outStream; + else + { + outStreamSpec2 = new COutFileStream; + sfxOutStream = outStreamSpec2; + const FString realPath = us2fs(archivePath.GetFinalPath()); + if (!outStreamSpec2->Create_NEW(realPath)) + return errorInfo.SetFromLastError("cannot open file", realPath); + } + + { + UInt64 sfxSize; + RINOK(sfxStreamSpec->GetSize(&sfxSize)) + RINOK(callback->WriteSfx(fs2us(options.SfxModule), sfxSize)) + } + + RINOK(NCompress::CopyStream(sfxStream, sfxOutStream, NULL)) + + if (outStreamSpec2) + { + RINOK(outStreamSpec2->Close()) + } + } + + CMyComPtr tailStream; + + if (options.SfxMode || !arc || arc->ArcStreamOffset == 0) + tailStream = outStream; + else + { + // Int64 globalOffset = arc->GetGlobalOffset(); + RINOK(InStream_SeekToBegin(arc->InStream)) + RINOK(NCompress::CopyStream_ExactSize(arc->InStream, outStream, arc->ArcStreamOffset, NULL)) + if (options.StdOutMode) + tailStream = outStream; + else + { + CTailOutStream *tailStreamSpec = new CTailOutStream; + tailStream = tailStreamSpec; + tailStreamSpec->Stream = outSeekStream; + tailStreamSpec->Offset = arc->ArcStreamOffset; + tailStreamSpec->Init(); + } + } + + CFiTime ft; + FiTime_Clear(ft); + bool ft_Defined = false; + { + FOR_VECTOR (i, updatePairs2) + { + const CUpdatePair2 &pair2 = updatePairs2[i]; + CFiTime ft2; + FiTime_Clear(ft2); + bool ft2_Defined = false; + /* we use full precision of dirItem, if dirItem is defined + and (dirItem will be used or dirItem is sameTime in dir and arc */ + if (pair2.DirIndex >= 0 && + (pair2.NewProps || pair2.IsSameTime)) + { + ft2 = dirItems.Items[(unsigned)pair2.DirIndex].MTime; + ft2_Defined = true; + } + else if (pair2.UseArcProps && pair2.ArcIndex >= 0) + { + const CArcItem &arcItem = arcItems[(unsigned)pair2.ArcIndex]; + if (arcItem.MTime.Def) + { + arcItem.MTime.Write_To_FiTime(ft2); + ft2_Defined = true; + } + } + if (ft2_Defined) + { + if (!ft_Defined || Compare_FiTime(&ft, &ft2) < 0) + { + ft = ft2; + ft_Defined = true; + } + } + } + /* + if (fileTimeType != NFileTimeType::kNotDefined) + FiTime_Normalize_With_Prec(ft, fileTimeType); + */ + } + + if (volStreamSpec && options.SetArcMTime && ft_Defined) + { + volStreamSpec->MTime = ft; + volStreamSpec->MTime_Defined = true; + } + + HRESULT result = outArchive->UpdateItems(tailStream, updatePairs2.Size(), updateCallback); + // callback->Finalize(); + RINOK(result) + + if (!updateCallbackSpec->AreAllFilesClosed()) + { + errorInfo.Message = "There are unclosed input files:"; + errorInfo.FileNames = updateCallbackSpec->_openFiles_Paths; + return E_FAIL; + } + + if (options.SetArcMTime) + { + // bool needNormalizeAfterStream; + // needParse; + /* + if (updateCallbackSpec->ArcMTime_WasReported) + { + isDefined = updateCallbackSpec->Reported_ArcMTime.Def; + if (isDefined) + updateCallbackSpec->Reported_ArcMTime.Write_To_FiTime(ft); + else + fileTimeType = NFileTimeType::kNotDefined; + } + if (!isDefined) + */ + { + if (updateCallbackSpec->LatestMTime_Defined) + { + // CArcTime at = StreamCallback_ArcMTime; + // updateCallbackSpec->StreamCallback_ArcMTime.Write_To_FiTime(ft); + // we must normalize with precision from archive; + if (!ft_Defined || Compare_FiTime(&ft, &updateCallbackSpec->LatestMTime) < 0) + ft = updateCallbackSpec->LatestMTime; + ft_Defined = true; + } + /* + if (fileTimeType != NFileTimeType::kNotDefined) + FiTime_Normalize_With_Prec(ft, fileTimeType); + */ + } + // if (ft.dwLowDateTime != 0 || ft.dwHighDateTime != 0) + if (ft_Defined) + { + // we ignore set time errors here. + // note that user could move some finished volumes to another folder. + if (outStreamSpec) + outStreamSpec->SetMTime(&ft); + else if (volStreamSpec) + volStreamSpec->SetMTime_Final(ft); + } + } + + if (callback) + { + UInt64 size = 0; + if (outStreamSpec) + outStreamSpec->GetSize(&size); + else if (stdOutFileStreamSpec) + size = stdOutFileStreamSpec->GetSize(); + else + size = volStreamSpec->GetSize(); + + st.OutArcFileSize = size; + } + + if (outStreamSpec) + result = outStreamSpec->Close(); + else if (volStreamSpec) + { + result = volStreamSpec->FinalFlush_and_CloseFiles(st.NumVolumes); + st.IsMultiVolMode = true; + } + + RINOK(result) + + if (processedItemsStatuses) + { + FOR_VECTOR (i, updatePairs2) + { + const CUpdatePair2 &up = updatePairs2[i]; + if (up.NewData && up.DirIndex >= 0) + { + const CDirItem &di = dirItems.Items[(unsigned)up.DirIndex]; + if (di.AreReparseData() || (!di.IsDir() && di.Size == 0)) + processedItemsStatuses[(unsigned)up.DirIndex] = 1; + } + } + } + + return result; +} + + + +static bool Censor_AreAllAllowed(const NWildcard::CCensor &censor) +{ + if (censor.Pairs.Size() != 1) + return false; + const NWildcard::CPair &pair = censor.Pairs[0]; + /* Censor_CheckPath() ignores (CPair::Prefix). + So we also ignore (CPair::Prefix) here */ + // if (!pair.Prefix.IsEmpty()) return false; + return pair.Head.AreAllAllowed(); +} + +bool CensorNode_CheckPath2(const NWildcard::CCensorNode &node, const CReadArcItem &item, bool &include); + +static bool Censor_CheckPath(const NWildcard::CCensor &censor, const CReadArcItem &item) +{ + bool finded = false; + FOR_VECTOR (i, censor.Pairs) + { + /* (CPair::Prefix) in not used for matching items in archive. + So we ignore (CPair::Prefix) here */ + bool include; + if (CensorNode_CheckPath2(censor.Pairs[i].Head, item, include)) + { + // Check it and FIXME !!!! + // here we can exclude item via some Pair, that is still allowed by another Pair + if (!include) + return false; + finded = true; + } + } + return finded; +} + +static HRESULT EnumerateInArchiveItems( + // bool storeStreamsMode, + const NWildcard::CCensor &censor, + const CArc &arc, + CObjectVector &arcItems) +{ + arcItems.Clear(); + UInt32 numItems; + IInArchive *archive = arc.Archive; + RINOK(archive->GetNumberOfItems(&numItems)) + arcItems.ClearAndReserve(numItems); + + CReadArcItem item; + + const bool allFilesAreAllowed = Censor_AreAllAllowed(censor); + + for (UInt32 i = 0; i < numItems; i++) + { + CArcItem ai; + + RINOK(arc.GetItem(i, item)) + ai.Name = item.Path; + ai.IsDir = item.IsDir; + ai.IsAltStream = + #ifdef SUPPORT_ALT_STREAMS + item.IsAltStream; + #else + false; + #endif + + /* + if (!storeStreamsMode && ai.IsAltStream) + continue; + */ + if (allFilesAreAllowed) + ai.Censored = true; + else + ai.Censored = Censor_CheckPath(censor, item); + + // ai.MTime will be set to archive MTime, if not present in archive item + RINOK(arc.GetItem_MTime(i, ai.MTime)) + RINOK(arc.GetItem_Size(i, ai.Size, ai.Size_Defined)) + + ai.IndexInServer = i; + arcItems.AddInReserved(ai); + } + return S_OK; +} + +#if defined(_WIN32) && !defined(UNDER_CE) + +#if defined(__MINGW32__) || defined(__MINGW64__) +#include +#else +#include +#endif + +extern "C" { + +#ifdef MAPI_FORCE_UNICODE + +#define Z7_WIN_LPMAPISENDMAILW LPMAPISENDMAILW +#define Z7_WIN_MapiFileDescW MapiFileDescW +#define Z7_WIN_MapiMessageW MapiMessageW +#define Z7_WIN_MapiRecipDescW MapiRecipDescW + +#else + +typedef struct +{ + ULONG ulReserved; + ULONG ulRecipClass; + PWSTR lpszName; + PWSTR lpszAddress; + ULONG ulEIDSize; + PVOID lpEntryID; +} Z7_WIN_MapiRecipDescW, *Z7_WIN_lpMapiRecipDescW; + +typedef struct +{ + ULONG ulReserved; + ULONG flFlags; + ULONG nPosition; + PWSTR lpszPathName; + PWSTR lpszFileName; + PVOID lpFileType; +} Z7_WIN_MapiFileDescW, *Z7_WIN_lpMapiFileDescW; + +typedef struct +{ + ULONG ulReserved; + PWSTR lpszSubject; + PWSTR lpszNoteText; + PWSTR lpszMessageType; + PWSTR lpszDateReceived; + PWSTR lpszConversationID; + FLAGS flFlags; + Z7_WIN_lpMapiRecipDescW lpOriginator; + ULONG nRecipCount; + Z7_WIN_lpMapiRecipDescW lpRecips; + ULONG nFileCount; + Z7_WIN_lpMapiFileDescW lpFiles; +} Z7_WIN_MapiMessageW, *Z7_WIN_lpMapiMessageW; + +typedef ULONG (FAR PASCAL Z7_WIN_MAPISENDMAILW)( + LHANDLE lhSession, + ULONG_PTR ulUIParam, + Z7_WIN_lpMapiMessageW lpMessage, + FLAGS flFlags, + ULONG ulReserved +); +typedef Z7_WIN_MAPISENDMAILW FAR *Z7_WIN_LPMAPISENDMAILW; + +#endif // MAPI_FORCE_UNICODE +} +#endif // _WIN32 + + +HRESULT UpdateArchive( + CCodecs *codecs, + const CObjectVector &types, + const UString &cmdArcPath2, + NWildcard::CCensor &censor, + CUpdateOptions &options, + CUpdateErrorInfo &errorInfo, + IOpenCallbackUI *openCallback, + IUpdateCallbackUI2 *callback, + bool needSetPath) +{ + if (options.StdOutMode && options.EMailMode) + return E_FAIL; + + if (types.Size() > 1) + return E_NOTIMPL; + + bool renameMode = !options.RenamePairs.IsEmpty(); + if (renameMode) + { + if (options.Commands.Size() != 1) + return E_FAIL; + } + + if (options.DeleteAfterCompressing) + { + if (options.Commands.Size() != 1) + return E_NOTIMPL; + const CActionSet &as = options.Commands[0].ActionSet; + for (unsigned i = 2; i < NPairState::kNumValues; i++) + if (as.StateActions[i] != NPairAction::kCompress) + return E_NOTIMPL; + } + + censor.AddPathsToCensor(options.PathMode); + #ifdef _WIN32 + ConvertToLongNames(censor); + #endif + censor.ExtendExclude(); + + + if (options.VolumesSizes.Size() > 0 && (options.EMailMode /* || options.SfxMode */)) + return E_NOTIMPL; + + if (options.SfxMode) + { + CProperty property; + property.Name = "rsfx"; + options.MethodMode.Properties.Add(property); + if (options.SfxModule.IsEmpty()) + { + errorInfo.Message = "SFX file is not specified"; + return E_FAIL; + } + bool found = false; + if (options.SfxModule.Find(FCHAR_PATH_SEPARATOR) < 0) + { + const FString fullName = NDLL::GetModuleDirPrefix() + options.SfxModule; + if (NFind::DoesFileExist_FollowLink(fullName)) + { + options.SfxModule = fullName; + found = true; + } + } + if (!found) + { + if (!NFind::DoesFileExist_FollowLink(options.SfxModule)) + return errorInfo.SetFromLastError("cannot find specified SFX module", options.SfxModule); + } + } + + CArchiveLink arcLink; + + + if (needSetPath) + { + if (!options.InitFormatIndex(codecs, types, cmdArcPath2) || + !options.SetArcPath(codecs, cmdArcPath2)) + return E_NOTIMPL; + } + + UString arcPath = options.ArchivePath.GetFinalPath(); + + if (!options.VolumesSizes.IsEmpty()) + { + arcPath = options.ArchivePath.GetFinalVolPath(); + arcPath += ".001"; + } + + if (cmdArcPath2.IsEmpty()) + { + if (options.MethodMode.Type.FormatIndex < 0) + throw "type of archive is not specified"; + } + else + { + NFind::CFileInfo fi; + if (!fi.Find_FollowLink(us2fs(arcPath))) + { + if (renameMode) + throw "can't find archive"; + if (options.MethodMode.Type.FormatIndex < 0) + { + if (!options.SetArcPath(codecs, cmdArcPath2)) + return E_NOTIMPL; + } + } + else + { + if (fi.IsDir()) + return errorInfo.SetFromError_DWORD("There is a folder with the name of archive", + us2fs(arcPath), + #ifdef _WIN32 + ERROR_ACCESS_DENIED + #else + EISDIR + #endif + ); + #ifdef _WIN32 + if (fi.IsDevice) + return E_NOTIMPL; + #endif + + if (!options.StdOutMode && options.UpdateArchiveItself) + if (fi.IsReadOnly()) + { + return errorInfo.SetFromError_DWORD("The file is read-only", + us2fs(arcPath), + #ifdef _WIN32 + ERROR_ACCESS_DENIED + #else + EACCES + #endif + ); + } + + if (options.VolumesSizes.Size() > 0) + { + errorInfo.FileNames.Add(us2fs(arcPath)); + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = kUpdateIsNotSupported_MultiVol; + return E_NOTIMPL; + } + CObjectVector types2; + // change it. + if (options.MethodMode.Type_Defined) + types2.Add(options.MethodMode.Type); + // We need to set Properties to open archive only in some cases (WIM archives). + + CIntVector excl; + COpenOptions op; + #ifndef Z7_SFX + op.props = &options.MethodMode.Properties; + #endif + op.codecs = codecs; + op.types = &types2; + op.excludedFormats = ! + op.stdInMode = false; + op.stream = NULL; + op.filePath = arcPath; + + RINOK(callback->StartOpenArchive(arcPath)) + + HRESULT result = arcLink.Open_Strict(op, openCallback); + + if (result == E_ABORT) + return result; + + HRESULT res2 = callback->OpenResult(codecs, arcLink, arcPath, result); + /* + if (result == S_FALSE) + return E_FAIL; + */ + RINOK(res2) + RINOK(result) + + if (arcLink.VolumePaths.Size() > 1) + { + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = kUpdateIsNotSupported_MultiVol; + return E_NOTIMPL; + } + + CArc &arc = arcLink.Arcs.Back(); + arc.MTime.Def = + #ifdef _WIN32 + !fi.IsDevice; + #else + true; + #endif + if (arc.MTime.Def) + arc.MTime.Set_From_FiTime(fi.MTime); + + if (arc.ErrorInfo.ThereIsTail) + { + // errorInfo.SystemError = (DWORD)E_NOTIMPL; + errorInfo.Message = "There is some data block after the end of the archive"; + return E_NOTIMPL; + } + if (options.MethodMode.Type.FormatIndex < 0) + { + options.MethodMode.Type.FormatIndex = arcLink.GetArc()->FormatIndex; + if (!options.SetArcPath(codecs, cmdArcPath2)) + return E_NOTIMPL; + } + } + } + + if (options.MethodMode.Type.FormatIndex < 0) + { + options.MethodMode.Type.FormatIndex = codecs->FindFormatForArchiveType((UString)kDefaultArcType); + if (options.MethodMode.Type.FormatIndex < 0) + return E_NOTIMPL; + } + + bool thereIsInArchive = arcLink.IsOpen; + if (!thereIsInArchive && renameMode) + return E_FAIL; + + CDirItems dirItems; + dirItems.Callback = callback; + + CDirItem parentDirItem; + CDirItem *parentDirItem_Ptr = NULL; + + /* + FStringVector requestedPaths; + FStringVector *requestedPaths_Ptr = NULL; + if (options.DeleteAfterCompressing) + requestedPaths_Ptr = &requestedPaths; + */ + + if (options.StdInMode) + { + CDirItem di; + // di.ClearBase(); + // di.Size = (UInt64)(Int64)-1; + if (!di.SetAs_StdInFile()) + return GetLastError_noZero_HRESULT(); + di.Name = options.StdInFileName; + // di.Attrib_IsDefined = false; + // NTime::GetCurUtc_FiTime(di.MTime); + // di.CTime = di.ATime = di.MTime; + dirItems.Items.Add(di); + } + else + { + bool needScanning = false; + + if (!renameMode) + FOR_VECTOR (i, options.Commands) + if (options.Commands[i].ActionSet.NeedScanning()) + needScanning = true; + + if (needScanning) + { + RINOK(callback->StartScanning()) + + dirItems.SymLinks = options.SymLinks.Val; + + #if defined(_WIN32) && !defined(UNDER_CE) + dirItems.ReadSecure = options.NtSecurity.Val; + #endif + + dirItems.ScanAltStreams = options.AltStreams.Val; + dirItems.ExcludeDirItems = censor.ExcludeDirItems; + dirItems.ExcludeFileItems = censor.ExcludeFileItems; + + dirItems.ShareForWrite = options.OpenShareForWrite; + + #ifndef _WIN32 + dirItems.StoreOwnerName = options.StoreOwnerName.Val; + #endif + + const HRESULT res = EnumerateItems(censor, + options.PathMode, + UString(), // options.AddPathPrefix, + dirItems); + + if (res != S_OK) + { + if (res != E_ABORT) + errorInfo.Message = "Scanning error"; + return res; + } + + RINOK(callback->FinishScanning(dirItems.Stat)) + + // 22.00: we don't need parent folder, if absolute path mode + if (options.PathMode != NWildcard::k_AbsPath) + if (censor.Pairs.Size() == 1) + { + NFind::CFileInfo fi; + FString prefix = us2fs(censor.Pairs[0].Prefix); + prefix.Add_Dot(); + // UString prefix = censor.Pairs[0].Prefix; + /* + if (prefix.Back() == WCHAR_PATH_SEPARATOR) + { + prefix.DeleteBack(); + } + */ + if (fi.Find(prefix)) + if (fi.IsDir()) + { + parentDirItem.Copy_From_FileInfoBase(fi); + parentDirItem_Ptr = &parentDirItem; + + int secureIndex = -1; + #if defined(_WIN32) && !defined(UNDER_CE) + if (options.NtSecurity.Val) + dirItems.AddSecurityItem(prefix, secureIndex); + #endif + parentDirItem.SecureIndex = secureIndex; + } + } + } + } + + FString tempDirPrefix; + bool usesTempDir = false; + + #ifdef _WIN32 + CTempDir tempDirectory; + if (options.EMailMode && options.EMailRemoveAfter) + { + tempDirectory.Create(kTempFolderPrefix); + tempDirPrefix = tempDirectory.GetPath(); + NormalizeDirPathPrefix(tempDirPrefix); + usesTempDir = true; + } + #endif + + CTempFiles tempFiles; + + bool createTempFile = false; + + if (!options.StdOutMode && options.UpdateArchiveItself) + { + CArchivePath &ap = options.Commands[0].ArchivePath; + ap = options.ArchivePath; + // if ((archive != 0 && !usesTempDir) || !options.WorkingDir.IsEmpty()) + if ((thereIsInArchive || !options.WorkingDir.IsEmpty()) && !usesTempDir && options.VolumesSizes.Size() == 0) + { + createTempFile = true; + ap.Temp = true; + if (!options.WorkingDir.IsEmpty()) + ap.TempPrefix = options.WorkingDir; + else + ap.TempPrefix = us2fs(ap.Prefix); + NormalizeDirPathPrefix(ap.TempPrefix); + } + } + + unsigned ci; + + + // self including protection + if (options.DeleteAfterCompressing) + { + for (ci = 0; ci < options.Commands.Size(); ci++) + { + CArchivePath &ap = options.Commands[ci].ArchivePath; + const FString path = us2fs(ap.GetFinalPath()); + // maybe we must compare absolute paths path here + FOR_VECTOR (i, dirItems.Items) + { + const FString phyPath = dirItems.GetPhyPath(i); + if (phyPath == path) + { + UString s; + s = "It is not allowed to include archive to itself"; + s.Add_LF(); + s += fs2us(path); + throw s; + } + } + } + } + + + for (ci = 0; ci < options.Commands.Size(); ci++) + { + CArchivePath &ap = options.Commands[ci].ArchivePath; + if (usesTempDir) + { + // Check it + ap.Prefix = fs2us(tempDirPrefix); + // ap.Temp = true; + // ap.TempPrefix = tempDirPrefix; + } + if (!options.StdOutMode && + (ci > 0 || !createTempFile)) + { + const FString path = us2fs(ap.GetFinalPath()); + if (NFind::DoesFileOrDirExist(path)) + { + errorInfo.SystemError = ERROR_FILE_EXISTS; + errorInfo.Message = "The file already exists"; + errorInfo.FileNames.Add(path); + return errorInfo.Get_HRESULT_Error(); + } + } + } + + CObjectVector arcItems; + if (thereIsInArchive) + { + RINOK(EnumerateInArchiveItems( + // options.StoreAltStreams, + censor, arcLink.Arcs.Back(), arcItems)) + } + + /* + FStringVector processedFilePaths; + FStringVector *processedFilePaths_Ptr = NULL; + if (options.DeleteAfterCompressing) + processedFilePaths_Ptr = &processedFilePaths; + */ + + CByteBuffer processedItems; + if (options.DeleteAfterCompressing) + { + const unsigned num = dirItems.Items.Size(); + processedItems.Alloc(num); + for (unsigned i = 0; i < num; i++) + processedItems[i] = 0; + } + + CMultiOutStream_Bunch multiStreams; + + /* + #ifndef Z7_NO_CRYPTO + if (arcLink.PasswordWasAsked) + { + // We set password, if open have requested password + RINOK(callback->SetPassword(arcLink.Password)); + } + #endif + */ + + for (ci = 0; ci < options.Commands.Size(); ci++) + { + const CArc *arc = thereIsInArchive ? arcLink.GetArc() : NULL; + CUpdateArchiveCommand &command = options.Commands[ci]; + UString name; + bool isUpdating; + + if (options.StdOutMode) + { + name = "stdout"; + isUpdating = thereIsInArchive; + } + else + { + name = command.ArchivePath.GetFinalPath(); + isUpdating = (ci == 0 && options.UpdateArchiveItself && thereIsInArchive); + } + + RINOK(callback->StartArchive(name, isUpdating)) + + CFinishArchiveStat st; + + RINOK(Compress(options, + isUpdating, + codecs, + command.ActionSet, + arc, + command.ArchivePath, + arcItems, + options.DeleteAfterCompressing ? (Byte *)processedItems : NULL, + + dirItems, + parentDirItem_Ptr, + + tempFiles, + multiStreams, + errorInfo, callback, st)) + + RINOK(callback->FinishArchive(st)) + } + + + if (thereIsInArchive) + { + RINOK(arcLink.Close()) + arcLink.Release(); + } + + multiStreams.DisableDeletion(); + RINOK(multiStreams.Destruct()) + + tempFiles.Paths.Clear(); + if (createTempFile) + { + try + { + CArchivePath &ap = options.Commands[0].ArchivePath; + const FString &tempPath = ap.GetTempPath(); + + // DWORD attrib = 0; + if (thereIsInArchive) + { + // attrib = NFind::GetFileAttrib(us2fs(arcPath)); + if (!DeleteFileAlways(us2fs(arcPath))) + return errorInfo.SetFromLastError("cannot delete the file", us2fs(arcPath)); + } + + if (!MyMoveFile(tempPath, us2fs(arcPath))) + { + errorInfo.SystemError = ::GetLastError(); + errorInfo.Message = "cannot move the file"; + if (errorInfo.SystemError == ERROR_INVALID_PARAMETER) + { + NFind::CFileInfo fi; + if (fi.Find(tempPath) && + fi.Size > (UInt32)(Int32)-1) + { + // bool isFsDetected = false; + // if (NSystem::Is_File_LimitedBy_4GB(us2fs(arcPath), isFsDetected) || !isFsDetected) + { + errorInfo.Message.Add_LF(); + errorInfo.Message += "Archive file size exceeds 4 GB"; + } + } + } + errorInfo.FileNames.Add(tempPath); + errorInfo.FileNames.Add(us2fs(arcPath)); + return errorInfo.Get_HRESULT_Error(); + } + + /* + if (attrib != INVALID_FILE_ATTRIBUTES && (attrib & FILE_ATTRIBUTE_READONLY)) + { + DWORD attrib2 = NFind::GetFileAttrib(us2fs(arcPath)); + if (attrib2 != INVALID_FILE_ATTRIBUTES) + NDir::SetFileAttrib(us2fs(arcPath), attrib2 | FILE_ATTRIBUTE_READONLY); + } + */ + } + catch(...) + { + throw; + } + } + + + #if defined(_WIN32) && !defined(UNDER_CE) + +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + + if (options.EMailMode) + { + NDLL::CLibrary mapiLib; + if (!mapiLib.Load(FTEXT("Mapi32.dll"))) + { + errorInfo.SetFromLastError("cannot load Mapi32.dll"); + return errorInfo.Get_HRESULT_Error(); + } + + FStringVector fullPaths; + unsigned i; + + for (i = 0; i < options.Commands.Size(); i++) + { + CArchivePath &ap = options.Commands[i].ArchivePath; + const FString finalPath = us2fs(ap.GetFinalPath()); + FString arcPath2; + if (!MyGetFullPathName(finalPath, arcPath2)) + return errorInfo.SetFromLastError("GetFullPathName error", finalPath); + fullPaths.Add(arcPath2); + } + + /* + LPMAPISENDDOCUMENTS fnSend = (LPMAPISENDDOCUMENTS)mapiLib.GetProc("MAPISendDocuments"); + if (fnSend == 0) + { + errorInfo.SetFromLastError)("7-Zip cannot find MAPISendDocuments function"); + return errorInfo.Get_HRESULT_Error(); + } + */ + const + Z7_WIN_LPMAPISENDMAILW sendMailW = Z7_GET_PROC_ADDRESS( + Z7_WIN_LPMAPISENDMAILW, mapiLib.Get_HMODULE(), + "MAPISendMailW"); + if (sendMailW) + { + + CCurrentDirRestorer curDirRestorer; + + UStringVector paths; + UStringVector names; + + for (i = 0; i < fullPaths.Size(); i++) + { + const UString arcPath2 = fs2us(fullPaths[i]); + const UString fileName = ExtractFileNameFromPath(arcPath2); + paths.Add(arcPath2); + names.Add(fileName); + // Warning!!! MAPISendDocuments function changes Current directory + // fnSend(0, ";", (LPSTR)(LPCSTR)path, (LPSTR)(LPCSTR)name, 0); + } + + CRecordVector files; + files.ClearAndSetSize(paths.Size()); + + for (i = 0; i < paths.Size(); i++) + { + Z7_WIN_MapiFileDescW &f = files[i]; + memset(&f, 0, sizeof(f)); + f.nPosition = 0xFFFFFFFF; + f.lpszPathName = paths[i].Ptr_non_const(); + f.lpszFileName = names[i].Ptr_non_const(); + } + + { + Z7_WIN_MapiMessageW m; + memset(&m, 0, sizeof(m)); + m.nFileCount = files.Size(); + m.lpFiles = files.NonConstData(); + + const UString addr (options.EMailAddress); + Z7_WIN_MapiRecipDescW rec; + if (!addr.IsEmpty()) + { + memset(&rec, 0, sizeof(rec)); + rec.ulRecipClass = MAPI_TO; + rec.lpszAddress = addr.Ptr_non_const(); + m.nRecipCount = 1; + m.lpRecips = &rec; + } + + sendMailW((LHANDLE)0, 0, &m, MAPI_DIALOG, 0); + } + } + else + { + const + LPMAPISENDMAIL sendMail = Z7_GET_PROC_ADDRESS( + LPMAPISENDMAIL, mapiLib.Get_HMODULE(), + "MAPISendMail"); + if (!sendMail) + { + // **************** NanaZip Modification Start **************** + //errorInfo.SetFromLastError("7-Zip cannot find MAPISendMail function"); + errorInfo.SetFromLastError("NanaZip cannot find MAPISendMail function"); + // **************** NanaZip Modification End **************** + return errorInfo.Get_HRESULT_Error(); + } + + CCurrentDirRestorer curDirRestorer; + + AStringVector paths; + AStringVector names; + + for (i = 0; i < fullPaths.Size(); i++) + { + const UString arcPath2 = fs2us(fullPaths[i]); + const UString fileName = ExtractFileNameFromPath(arcPath2); + paths.Add(GetAnsiString(arcPath2)); + names.Add(GetAnsiString(fileName)); + // const AString path (GetAnsiString(arcPath2)); + // const AString name (GetAnsiString(fileName)); + // Warning!!! MAPISendDocuments function changes Current directory + // fnSend(0, ";", (LPSTR)(LPCSTR)path, (LPSTR)(LPCSTR)name, 0); + } + + CRecordVector files; + files.ClearAndSetSize(paths.Size()); + + for (i = 0; i < paths.Size(); i++) + { + MapiFileDesc &f = files[i]; + memset(&f, 0, sizeof(f)); + f.nPosition = 0xFFFFFFFF; + f.lpszPathName = paths[i].Ptr_non_const(); + f.lpszFileName = names[i].Ptr_non_const(); + } + + { + MapiMessage m; + memset(&m, 0, sizeof(m)); + m.nFileCount = files.Size(); + m.lpFiles = files.NonConstData(); + + const AString addr (GetAnsiString(options.EMailAddress)); + MapiRecipDesc rec; + if (!addr.IsEmpty()) + { + memset(&rec, 0, sizeof(rec)); + rec.ulRecipClass = MAPI_TO; + rec.lpszAddress = addr.Ptr_non_const(); + m.nRecipCount = 1; + m.lpRecips = &rec; + } + + sendMail((LHANDLE)0, 0, &m, MAPI_DIALOG, 0); + } + } + } + + #endif + + if (options.DeleteAfterCompressing) + { + CRecordVector pairs; + FStringVector foldersNames; + + unsigned i; + + for (i = 0; i < dirItems.Items.Size(); i++) + { + const CDirItem &dirItem = dirItems.Items[i]; + const FString phyPath = dirItems.GetPhyPath(i); + if (dirItem.IsDir()) + { + CDirPathSortPair pair; + pair.Index = i; + pair.SetNumSlashes(phyPath); + pairs.Add(pair); + } + else + { + // 21.04: we have set processedItems[*] before for all required items + if (processedItems[i] != 0 + // || dirItem.Size == 0 + // || dirItem.AreReparseData() + ) + { + NFind::CFileInfo fileInfo; + /* if (!SymLinks), we follow link here, similar to (dirItem) filling */ + if (fileInfo.Find(phyPath, !options.SymLinks.Val)) + { + bool is_SameSize = false; + if (options.SymLinks.Val && dirItem.AreReparseData()) + { + /* (dirItem.Size = dirItem.ReparseData.Size()) was set before. + So we don't compare sizes for that case here */ + is_SameSize = fileInfo.IsOsSymLink(); + } + else + is_SameSize = (fileInfo.Size == dirItem.Size); + + if (is_SameSize + && Compare_FiTime(&fileInfo.MTime, &dirItem.MTime) == 0 + && Compare_FiTime(&fileInfo.CTime, &dirItem.CTime) == 0) + { + RINOK(callback->DeletingAfterArchiving(phyPath, false)) + DeleteFileAlways(phyPath); + } + } + } + else + { + // file was skipped by some reason. We can throw error for debug: + /* + errorInfo.SystemError = 0; + errorInfo.Message = "file was not processed"; + errorInfo.FileNames.Add(phyPath); + return E_FAIL; + */ + } + } + } + + pairs.Sort2(); + + for (i = 0; i < pairs.Size(); i++) + { + const FString phyPath = dirItems.GetPhyPath(pairs[i].Index); + if (NFind::DoesDirExist(phyPath)) + { + RINOK(callback->DeletingAfterArchiving(phyPath, true)) + RemoveDir(phyPath); + } + } + + RINOK(callback->FinishDeletingAfterArchiving()) + } + + return S_OK; +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateAction.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateAction.cpp new file mode 100644 index 000000000..59fded214 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateAction.cpp @@ -0,0 +1,64 @@ +// UpdateAction.cpp + +#include "StdAfx.h" + +#include "UpdateAction.h" + +namespace NUpdateArchive { + +const CActionSet k_ActionSet_Add = +{{ + NPairAction::kCopy, + NPairAction::kCopy, + NPairAction::kCompress, + NPairAction::kCompress, + NPairAction::kCompress, + NPairAction::kCompress, + NPairAction::kCompress +}}; + +const CActionSet k_ActionSet_Update = +{{ + NPairAction::kCopy, + NPairAction::kCopy, + NPairAction::kCompress, + NPairAction::kCopy, + NPairAction::kCompress, + NPairAction::kCopy, + NPairAction::kCompress +}}; + +const CActionSet k_ActionSet_Fresh = +{{ + NPairAction::kCopy, + NPairAction::kCopy, + NPairAction::kIgnore, + NPairAction::kCopy, + NPairAction::kCompress, + NPairAction::kCopy, + NPairAction::kCompress +}}; + +const CActionSet k_ActionSet_Sync = +{{ + NPairAction::kCopy, + NPairAction::kIgnore, + NPairAction::kCompress, + NPairAction::kCopy, + NPairAction::kCompress, + NPairAction::kCopy, + NPairAction::kCompress, +}}; + +const CActionSet k_ActionSet_Delete = +{{ + NPairAction::kCopy, + NPairAction::kIgnore, + NPairAction::kIgnore, + NPairAction::kIgnore, + NPairAction::kIgnore, + NPairAction::kIgnore, + NPairAction::kIgnore +}}; + +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateCallback.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateCallback.cpp new file mode 100644 index 000000000..d8d1499e7 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateCallback.cpp @@ -0,0 +1,1029 @@ +// UpdateCallback.cpp + +#include "StdAfx.h" + +// #include + +#ifndef _WIN32 +// #include +// #include +// for major()/minor(): +#if defined(__APPLE__) || defined(__DragonFly__) || \ + defined(BSD) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) +#include +#else +#include +#endif + +#endif // _WIN32 + +#ifndef Z7_ST +#include "../../../Windows/Synchronization.h" +#endif + +#include "../../../Common/ComTry.h" +#include "../../../Common/IntToString.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/Wildcard.h" +#include "../../../Common/UTFConvert.h" + +#include "../../../Windows/FileDir.h" +#include "../../../Windows/FileName.h" +#include "../../../Windows/PropVariant.h" + +#include "../../Common/StreamObjects.h" + +#include "UpdateCallback.h" + +#if defined(_WIN32) && !defined(UNDER_CE) +#define Z7_USE_SECURITY_CODE +#include "../../../Windows/SecurityUtils.h" +#endif + +using namespace NWindows; +using namespace NFile; + +#ifndef Z7_ST +static NSynchronization::CCriticalSection g_CriticalSection; +#define MT_LOCK NSynchronization::CCriticalSectionLock lock(g_CriticalSection); +#else +#define MT_LOCK +#endif + + +#ifdef Z7_USE_SECURITY_CODE +bool InitLocalPrivileges(); +#endif + +CArchiveUpdateCallback::CArchiveUpdateCallback(): + PreserveATime(false), + ShareForWrite(false), + StopAfterOpenError(false), + StdInMode(false), + + KeepOriginalItemNames(false), + StoreNtSecurity(false), + StoreHardLinks(false), + StoreSymLinks(false), + + #ifndef _WIN32 + StoreOwnerId(false), + StoreOwnerName(false), + #endif + + /* + , Need_ArcMTime_Report(false), + , ArcMTime_WasReported(false), + */ + Need_LatestMTime(false), + LatestMTime_Defined(false), + + Callback(NULL), + + DirItems(NULL), + ParentDirItem(NULL), + + Arc(NULL), + ArcItems(NULL), + UpdatePairs(NULL), + NewNames(NULL), + Comment(NULL), + CommentIndex(-1), + + ProcessedItemsStatuses(NULL), + _hardIndex_From((UInt32)(Int32)-1) +{ + #ifdef Z7_USE_SECURITY_CODE + _saclEnabled = InitLocalPrivileges(); + #endif +} + + +Z7_COM7F_IMF(CArchiveUpdateCallback::SetTotal(UInt64 size)) +{ + COM_TRY_BEGIN + return Callback->SetTotal(size); + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::SetCompleted(const UInt64 *completeValue)) +{ + COM_TRY_BEGIN + return Callback->SetCompleted(completeValue); + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::SetRatioInfo(const UInt64 *inSize, const UInt64 *outSize)) +{ + COM_TRY_BEGIN + return Callback->SetRatioInfo(inSize, outSize); + COM_TRY_END +} + + +/* +static const CStatProp kProps[] = +{ + { NULL, kpidPath, VT_BSTR}, + { NULL, kpidIsDir, VT_BOOL}, + { NULL, kpidSize, VT_UI8}, + { NULL, kpidCTime, VT_FILETIME}, + { NULL, kpidATime, VT_FILETIME}, + { NULL, kpidMTime, VT_FILETIME}, + { NULL, kpidAttrib, VT_UI4}, + { NULL, kpidIsAnti, VT_BOOL} +}; + +Z7_COM7F_IMF(CArchiveUpdateCallback::EnumProperties(IEnumSTATPROPSTG **) +{ + return CStatPropEnumerator::CreateEnumerator(kProps, Z7_ARRAY_SIZE(kProps), enumerator); +} +*/ + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetUpdateItemInfo(UInt32 index, + Int32 *newData, Int32 *newProps, UInt32 *indexInArchive)) +{ + COM_TRY_BEGIN + RINOK(Callback->CheckBreak()) + const CUpdatePair2 &up = (*UpdatePairs)[index]; + if (newData) *newData = BoolToInt(up.NewData); + if (newProps) *newProps = BoolToInt(up.NewProps); + if (indexInArchive) + { + *indexInArchive = (UInt32)(Int32)-1; + if (up.ExistInArchive()) + *indexInArchive = ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex; + } + return S_OK; + COM_TRY_END +} + + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetRootProp(PROPID propID, PROPVARIANT *value)) +{ + NCOM::CPropVariant prop; + switch (propID) + { + case kpidIsDir: prop = true; break; + case kpidAttrib: if (ParentDirItem) prop = ParentDirItem->GetWinAttrib(); break; + case kpidCTime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->CTime); break; + case kpidATime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->ATime); break; + case kpidMTime: if (ParentDirItem) PropVariant_SetFrom_FiTime(prop, ParentDirItem->MTime); break; + case kpidArcFileName: if (!ArcFileName.IsEmpty()) prop = ArcFileName; break; + default: break; + } + prop.Detach(value); + return S_OK; +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetParent(UInt32 /* index */, UInt32 *parent, UInt32 *parentType)) +{ + *parentType = NParentType::kDir; + *parent = (UInt32)(Int32)-1; + return S_OK; +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetNumRawProps(UInt32 *numProps)) +{ + *numProps = 0; + if (StoreNtSecurity) + *numProps = 1; + return S_OK; +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetRawPropInfo(UInt32 /* index */, BSTR *name, PROPID *propID)) +{ + *name = NULL; + *propID = kpidNtSecure; + return S_OK; +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetRootRawProp(PROPID + propID + , const void **data, UInt32 *dataSize, UInt32 *propType)) +{ + #ifndef Z7_USE_SECURITY_CODE + UNUSED_VAR(propID) + #endif + + *data = NULL; + *dataSize = 0; + *propType = 0; + if (!StoreNtSecurity) + return S_OK; + #ifdef Z7_USE_SECURITY_CODE + if (propID == kpidNtSecure) + { + if (StdInMode) + return S_OK; + + if (ParentDirItem) + { + if (ParentDirItem->SecureIndex < 0) + return S_OK; + const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[(unsigned)ParentDirItem->SecureIndex]; + *data = buf; + *dataSize = (UInt32)buf.Size(); + *propType = NPropDataType::kRaw; + return S_OK; + } + + if (Arc && Arc->GetRootProps) + return Arc->GetRootProps->GetRootRawProp(propID, data, dataSize, propType); + } + #endif + return S_OK; +} + + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetRawProp(UInt32 index, PROPID propID, const void **data, UInt32 *dataSize, UInt32 *propType)) +{ + *data = NULL; + *dataSize = 0; + *propType = 0; + + if (propID == kpidNtSecure || + propID == kpidNtReparse) + { + if (StdInMode) + return S_OK; + + const CUpdatePair2 &up = (*UpdatePairs)[index]; + if (up.UseArcProps && up.ExistInArchive() && Arc->GetRawProps) + return Arc->GetRawProps->GetRawProp( + ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex, + propID, data, dataSize, propType); + { + /* + if (!up.NewData) + return E_FAIL; + */ + if (up.IsAnti) + return S_OK; + + #if defined(_WIN32) && !defined(UNDER_CE) + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + #endif + + #ifdef Z7_USE_SECURITY_CODE + if (propID == kpidNtSecure) + { + if (!StoreNtSecurity) + return S_OK; + if (di.SecureIndex < 0) + return S_OK; + const CByteBuffer &buf = DirItems->SecureBlocks.Bufs[(unsigned)di.SecureIndex]; + *data = buf; + *dataSize = (UInt32)buf.Size(); + *propType = NPropDataType::kRaw; + } + else + #endif + if (propID == kpidNtReparse) + { + if (!StoreSymLinks) + return S_OK; + #if defined(_WIN32) && !defined(UNDER_CE) + // we use ReparseData2 instead of ReparseData for WIM format + const CByteBuffer *buf = &di.ReparseData2; + if (buf->Size() == 0) + buf = &di.ReparseData; + if (buf->Size() != 0) + { + *data = *buf; + *dataSize = (UInt32)buf->Size(); + *propType = NPropDataType::kRaw; + } + #endif + } + + return S_OK; + } + } + + return S_OK; +} + +#if defined(_WIN32) && !defined(UNDER_CE) + +static UString GetRelativePath(const UString &to, const UString &from) +{ + UStringVector partsTo, partsFrom; + SplitPathToParts(to, partsTo); + SplitPathToParts(from, partsFrom); + + unsigned i; + for (i = 0;; i++) + { + if (i + 1 >= partsFrom.Size() || + i + 1 >= partsTo.Size()) + break; + if (CompareFileNames(partsFrom[i], partsTo[i]) != 0) + break; + } + + if (i == 0) + { + #ifdef _WIN32 + if (NName::IsDrivePath(to) || + NName::IsDrivePath(from)) + return to; + #endif + } + + UString s; + unsigned k; + + for (k = i + 1; k < partsFrom.Size(); k++) + s += ".." STRING_PATH_SEPARATOR; + + for (k = i; k < partsTo.Size(); k++) + { + if (k != i) + s.Add_PathSepar(); + s += partsTo[k]; + } + + return s; +} + +#endif + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetProperty(UInt32 index, PROPID propID, PROPVARIANT *value)) +{ + COM_TRY_BEGIN + const CUpdatePair2 &up = (*UpdatePairs)[index]; + NCOM::CPropVariant prop; + + if (up.NewData) + { + /* + if (propID == kpidIsHardLink) + { + prop = _isHardLink; + prop.Detach(value); + return S_OK; + } + */ + if (propID == kpidSymLink) + { + if (index == _hardIndex_From) + { + prop.Detach(value); + return S_OK; + } + + #if !defined(UNDER_CE) + + if (up.DirIndex >= 0) + { + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + + #ifdef _WIN32 + // if (di.IsDir()) + { + CReparseAttr attr; + if (attr.Parse(di.ReparseData, di.ReparseData.Size())) + { + const UString simpleName = attr.GetPath(); + if (!attr.IsSymLink_WSL() && attr.IsRelative_Win()) + prop = simpleName; + else + { + const FString phyPath = DirItems->GetPhyPath((unsigned)up.DirIndex); + FString fullPath; + if (NDir::MyGetFullPathName(phyPath, fullPath)) + { + prop = GetRelativePath(simpleName, fs2us(fullPath)); + } + } + prop.Detach(value); + return S_OK; + } + } + + #else // _WIN32 + + if (di.ReparseData.Size() != 0) + { + AString utf; + utf.SetFrom_CalcLen((const char *)(const Byte *)di.ReparseData, (unsigned)di.ReparseData.Size()); + + UString us; + if (ConvertUTF8ToUnicode(utf, us)) + { + prop = us; + prop.Detach(value); + return S_OK; + } + } + + #endif // _WIN32 + } + #endif // !defined(UNDER_CE) + } + else if (propID == kpidHardLink) + { + if (index == _hardIndex_From) + { + const CKeyKeyValPair &pair = _map[_hardIndex_To]; + const CUpdatePair2 &up2 = (*UpdatePairs)[pair.Value]; + prop = DirItems->GetLogPath((unsigned)up2.DirIndex); + prop.Detach(value); + return S_OK; + } + if (up.DirIndex >= 0) + { + prop.Detach(value); + return S_OK; + } + } + } + + if (up.IsAnti + && propID != kpidIsDir + && propID != kpidPath + && propID != kpidIsAltStream) + { + switch (propID) + { + case kpidSize: prop = (UInt64)0; break; + case kpidIsAnti: prop = true; break; + default: break; + } + } + else if (propID == kpidPath && up.NewNameIndex >= 0) + prop = (*NewNames)[(unsigned)up.NewNameIndex]; + else if (propID == kpidComment + && CommentIndex >= 0 + && (unsigned)CommentIndex == index + && Comment) + prop = *Comment; + else if (propID == kpidShortName && up.NewNameIndex >= 0 && up.IsMainRenameItem) + { + // we can generate new ShortName here; + } + else if ((up.UseArcProps || (KeepOriginalItemNames && (propID == kpidPath || propID == kpidIsAltStream))) + && up.ExistInArchive() && Archive) + return Archive->GetProperty(ArcItems ? (*ArcItems)[(unsigned)up.ArcIndex].IndexInServer : (UInt32)(Int32)up.ArcIndex, propID, value); + else if (up.ExistOnDisk()) + { + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + switch (propID) + { + case kpidPath: prop = DirItems->GetLogPath((unsigned)up.DirIndex); break; + case kpidIsDir: prop = di.IsDir(); break; + case kpidSize: prop = (UInt64)(di.IsDir() ? (UInt64)0 : di.Size); break; + case kpidCTime: PropVariant_SetFrom_FiTime(prop, di.CTime); break; + case kpidATime: PropVariant_SetFrom_FiTime(prop, di.ATime); break; + case kpidMTime: PropVariant_SetFrom_FiTime(prop, di.MTime); break; + case kpidAttrib: /* if (di.Attrib_IsDefined) */ prop = (UInt32)di.GetWinAttrib(); break; + case kpidPosixAttrib: /* if (di.Attrib_IsDefined) */ prop = (UInt32)di.GetPosixAttrib(); break; + + #if defined(_WIN32) + case kpidIsAltStream: prop = di.IsAltStream; break; + // case kpidShortName: prop = di.ShortName; break; + #else + + #if defined(__APPLE__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wsign-conversion" + #endif + + case kpidDeviceMajor: + /* + printf("\ndi.mode = %o\n", di.mode); + printf("\nst.st_rdev major = %d\n", (unsigned)major(di.rdev)); + printf("\nst.st_rdev minor = %d\n", (unsigned)minor(di.rdev)); + */ + if (S_ISCHR(di.mode) || S_ISBLK(di.mode)) + prop = (UInt32)major(di.rdev); + break; + + case kpidDeviceMinor: + if (S_ISCHR(di.mode) || S_ISBLK(di.mode)) + prop = (UInt32)minor(di.rdev); + break; + + #if defined(__APPLE__) + #pragma GCC diagnostic pop + #endif + + // case kpidDevice: if (S_ISCHR(di.mode) || S_ISBLK(di.mode)) prop = (UInt64)(di.rdev); break; + + case kpidUserId: if (StoreOwnerId) prop = (UInt32)di.uid; break; + case kpidGroupId: if (StoreOwnerId) prop = (UInt32)di.gid; break; + case kpidUser: + if (di.OwnerNameIndex >= 0) + prop = DirItems->OwnerNameMap.Strings[(unsigned)di.OwnerNameIndex]; + break; + case kpidGroup: + if (di.OwnerGroupIndex >= 0) + prop = DirItems->OwnerGroupMap.Strings[(unsigned)di.OwnerGroupIndex]; + break; + #endif + default: break; + } + } + prop.Detach(value); + return S_OK; + COM_TRY_END +} + +#ifndef Z7_ST +static NSynchronization::CCriticalSection g_CS; +#endif + +void CArchiveUpdateCallback::UpdateProcessedItemStatus(unsigned dirIndex) +{ + if (ProcessedItemsStatuses) + { + #ifndef Z7_ST + NSynchronization::CCriticalSectionLock lock(g_CS); + #endif + ProcessedItemsStatuses[dirIndex] = 1; + } +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetStream2(UInt32 index, ISequentialInStream **inStream, UInt32 mode)) +{ + COM_TRY_BEGIN + *inStream = NULL; + const CUpdatePair2 &up = (*UpdatePairs)[index]; + if (!up.NewData) + return E_FAIL; + + RINOK(Callback->CheckBreak()) + // RINOK(Callback->Finalize()); + + bool isDir = IsDir(up); + + if (up.IsAnti) + { + UString name; + if (up.ArcIndex >= 0) + name = (*ArcItems)[(unsigned)up.ArcIndex].Name; + else if (up.DirIndex >= 0) + name = DirItems->GetLogPath((unsigned)up.DirIndex); + RINOK(Callback->GetStream(name, isDir, true, mode)) + + /* 9.33: fixed. Handlers expect real stream object for files, even for anti-file. + so we return empty stream */ + + if (!isDir) + { + CBufInStream *inStreamSpec = new CBufInStream(); + CMyComPtr inStreamLoc = inStreamSpec; + inStreamSpec->Init(NULL, 0); + *inStream = inStreamLoc.Detach(); + } + return S_OK; + } + + RINOK(Callback->GetStream(DirItems->GetLogPath((unsigned)up.DirIndex), isDir, false, mode)) + + if (isDir) + return S_OK; + + if (StdInMode) + { + if (mode != NUpdateNotifyOp::kAdd && + mode != NUpdateNotifyOp::kUpdate) + return S_OK; + +#if 1 + CStdInFileStream *inStreamSpec = new CStdInFileStream; + CMyComPtr inStreamLoc(inStreamSpec); +#else + CMyComPtr inStreamLoc; + if (!CreateStdInStream(inStreamLoc)) + return GetLastError_noZero_HRESULT(); +#endif + *inStream = inStreamLoc.Detach(); + } + else + { + #if !defined(UNDER_CE) + const CDirItem &di = DirItems->Items[(unsigned)up.DirIndex]; + if (di.AreReparseData()) + { + /* + // we still need DeviceIoControlOut() instead of Read + if (!inStreamSpec->File.OpenReparse(path)) + { + return Callback->OpenFileError(path, ::GetLastError()); + } + */ + // 20.03: we use Reparse Data instead of real data + + CBufInStream *inStreamSpec = new CBufInStream(); + CMyComPtr inStreamLoc = inStreamSpec; + inStreamSpec->Init(di.ReparseData, di.ReparseData.Size()); + *inStream = inStreamLoc.Detach(); + + UpdateProcessedItemStatus((unsigned)up.DirIndex); + return S_OK; + } + #endif // !defined(UNDER_CE) + + CInFileStream *inStreamSpec = new CInFileStream; + CMyComPtr inStreamLoc(inStreamSpec); + + /* + // for debug: + #ifdef _WIN32 + inStreamSpec->StoreOwnerName = true; + inStreamSpec->OwnerName = "user_name"; + inStreamSpec->OwnerName += di.Name; + inStreamSpec->OwnerName += "11111111112222222222222333333333333"; + inStreamSpec->OwnerGroup = "gname_"; + inStreamSpec->OwnerGroup += inStreamSpec->OwnerName; + #endif + */ + + #ifndef _WIN32 + inStreamSpec->StoreOwnerId = StoreOwnerId; + inStreamSpec->StoreOwnerName = StoreOwnerName; + + // if (StoreOwner) + { + inStreamSpec->_uid = di.uid; + inStreamSpec->_gid = di.gid; + if (di.OwnerNameIndex >= 0) + inStreamSpec->OwnerName = DirItems->OwnerNameMap.Strings[(unsigned)di.OwnerNameIndex]; + if (di.OwnerGroupIndex >= 0) + inStreamSpec->OwnerGroup = DirItems->OwnerGroupMap.Strings[(unsigned)di.OwnerGroupIndex]; + } + #endif + + inStreamSpec->SupportHardLinks = StoreHardLinks; + const bool preserveATime = (PreserveATime + || mode == NUpdateNotifyOp::kAnalyze); // 22.00 : we don't change access time in Analyze pass. + inStreamSpec->Set_PreserveATime(preserveATime); + + const FString path = DirItems->GetPhyPath((unsigned)up.DirIndex); + _openFiles_Indexes.Add(index); + _openFiles_Paths.Add(path); + // _openFiles_Streams.Add(inStreamSpec); + + /* 21.02 : we set Callback/CallbackRef after _openFiles_Indexes adding + for correct working if exception was raised in GetPhyPath */ + inStreamSpec->Callback = this; + inStreamSpec->CallbackRef = index; + + if (!inStreamSpec->OpenShared(path, ShareForWrite)) + { + bool isOpen = false; + if (preserveATime) + { + inStreamSpec->Set_PreserveATime(false); + isOpen = inStreamSpec->OpenShared(path, ShareForWrite); + } + if (!isOpen) + { + const DWORD error = ::GetLastError(); + const HRESULT hres = Callback->OpenFileError(path, error); + if (hres == S_OK || hres == S_FALSE) + if (StopAfterOpenError || + // v23: we check also for some critical errors: + #ifdef _WIN32 + error == ERROR_NO_SYSTEM_RESOURCES + #else + error == EMFILE + #endif + ) + { + if (error == 0) + return E_FAIL; + return HRESULT_FROM_WIN32(error); + } + return hres; + } + } + + /* + { + // for debug: + Byte b = 0; + UInt32 processedSize = 0; + if (inStreamSpec->Read(&b, 1, &processedSize) != S_OK || + processedSize != 1) + return E_FAIL; + } + */ + + if (Need_LatestMTime) + { + inStreamSpec->ReloadProps(); + } + + // #if defined(Z7_FILE_STREAMS_USE_WIN_FILE) || !defined(_WIN32) + if (StoreHardLinks) + { + CStreamFileProps props; + if (inStreamSpec->GetProps2(&props) == S_OK) + { + if (props.NumLinks > 1) + { + CKeyKeyValPair pair; + pair.Key1 = props.VolID; + pair.Key2 = props.FileID_Low; + pair.Value = index; + const unsigned numItems = _map.Size(); + const unsigned pairIndex = _map.AddToUniqueSorted2(pair); + if (numItems == _map.Size()) + { + // const CKeyKeyValPair &pair2 = _map.Pairs[pairIndex]; + _hardIndex_From = index; + _hardIndex_To = pairIndex; + // we could return NULL as stream, but it's better to return real stream + // return S_OK; + } + } + } + } + // #endif + + UpdateProcessedItemStatus((unsigned)up.DirIndex); + *inStream = inStreamLoc.Detach(); + } + + return S_OK; + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::SetOperationResult(Int32 opRes)) +{ + COM_TRY_BEGIN + return Callback->SetOperationResult(opRes); + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetStream(UInt32 index, ISequentialInStream **inStream)) +{ + COM_TRY_BEGIN + return GetStream2(index, inStream, + (*UpdatePairs)[index].ArcIndex < 0 ? + NUpdateNotifyOp::kAdd : + NUpdateNotifyOp::kUpdate); + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::ReportOperation(UInt32 indexType, UInt32 index, UInt32 op)) +{ + COM_TRY_BEGIN + + // if (op == NUpdateNotifyOp::kOpFinished) return Callback->ReportFinished(indexType, index); + + bool isDir = false; + + if (indexType == NArchive::NEventIndexType::kOutArcIndex) + { + UString name; + if (index != (UInt32)(Int32)-1) + { + const CUpdatePair2 &up = (*UpdatePairs)[index]; + if (up.ExistOnDisk()) + { + name = DirItems->GetLogPath((unsigned)up.DirIndex); + isDir = DirItems->Items[(unsigned)up.DirIndex].IsDir(); + } + } + return Callback->ReportUpdateOperation(op, name.IsEmpty() ? NULL : name.Ptr(), isDir); + } + + wchar_t temp[16]; + UString s2; + const wchar_t *s = NULL; + + if (indexType == NArchive::NEventIndexType::kInArcIndex) + { + if (index != (UInt32)(Int32)-1) + { + if (ArcItems) + { + const CArcItem &ai = (*ArcItems)[index]; + s = ai.Name; + isDir = ai.IsDir; + } + else if (Arc) + { + RINOK(Arc->GetItem_Path(index, s2)) + s = s2; + RINOK(Archive_IsItem_Dir(Arc->Archive, index, isDir)) + } + } + } + else if (indexType == NArchive::NEventIndexType::kBlockIndex) + { + temp[0] = '#'; + ConvertUInt32ToString(index, temp + 1); + s = temp; + } + + if (!s) + s = L""; + + return Callback->ReportUpdateOperation(op, s, isDir); + + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::ReportExtractResult(UInt32 indexType, UInt32 index, Int32 opRes)) +{ + COM_TRY_BEGIN + + bool isEncrypted = false; + wchar_t temp[16]; + UString s2; + const wchar_t *s = NULL; + + if (indexType == NArchive::NEventIndexType::kOutArcIndex) + { + /* + UString name; + if (index != (UInt32)(Int32)-1) + { + const CUpdatePair2 &up = (*UpdatePairs)[index]; + if (up.ExistOnDisk()) + { + s2 = DirItems->GetLogPath(up.DirIndex); + s = s2; + } + } + */ + return E_FAIL; + } + + if (indexType == NArchive::NEventIndexType::kInArcIndex) + { + if (index != (UInt32)(Int32)-1) + { + if (ArcItems) + s = (*ArcItems)[index].Name; + else if (Arc) + { + RINOK(Arc->GetItem_Path(index, s2)) + s = s2; + } + if (Archive) + { + RINOK(Archive_GetItemBoolProp(Archive, index, kpidEncrypted, isEncrypted)) + } + } + } + else if (indexType == NArchive::NEventIndexType::kBlockIndex) + { + temp[0] = '#'; + ConvertUInt32ToString(index, temp + 1); + s = temp; + } + + return Callback->ReportExtractResult(opRes, BoolToInt(isEncrypted), s); + + COM_TRY_END +} + + +/* +Z7_COM7F_IMF(CArchiveUpdateCallback::DoNeedArcProp(PROPID propID, Int32 *answer)) +{ + *answer = 0; + if (Need_ArcMTime_Report && propID == kpidComboMTime) + *answer = 1; + return S_OK; +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::ReportProp(UInt32 indexType, UInt32 index, PROPID propID, const PROPVARIANT *value)) +{ + if (indexType == NArchive::NEventIndexType::kArcProp) + { + if (propID == kpidComboMTime) + { + ArcMTime_WasReported = true; + if (value->vt == VT_FILETIME) + { + Reported_ArcMTime.Set_From_Prop(*value); + Reported_ArcMTime.Def = true; + } + else + { + Reported_ArcMTime.Clear(); + if (value->vt != VT_EMPTY) + return E_FAIL; // for debug + } + } + } + return Callback->ReportProp(indexType, index, propID, value); +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::ReportRawProp(UInt32 indexType, UInt32 index, + PROPID propID, const void *data, UInt32 dataSize, UInt32 propType)) +{ + return Callback->ReportRawProp(indexType, index, propID, data, dataSize, propType); +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::ReportFinished(UInt32 indexType, UInt32 index, Int32 opRes)) +{ + return Callback->ReportFinished(indexType, index, opRes); +} +*/ + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetVolumeSize(UInt32 index, UInt64 *size)) +{ + if (VolumesSizes.Size() == 0) + return S_FALSE; + if (index >= (UInt32)VolumesSizes.Size()) + index = VolumesSizes.Size() - 1; + *size = VolumesSizes[index]; + return S_OK; +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::GetVolumeStream(UInt32 index, ISequentialOutStream **volumeStream)) +{ + COM_TRY_BEGIN + char temp[16]; + ConvertUInt32ToString(index + 1, temp); + FString res (temp); + while (res.Len() < 2) + res.InsertAtFront(FTEXT('0')); + FString fileName = VolName; + fileName.Add_Dot(); + fileName += res; + fileName += VolExt; + COutFileStream *streamSpec = new COutFileStream; + CMyComPtr streamLoc(streamSpec); + if (!streamSpec->Create_NEW(fileName)) + return GetLastError_noZero_HRESULT(); + *volumeStream = streamLoc.Detach(); + return S_OK; + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::CryptoGetTextPassword2(Int32 *passwordIsDefined, BSTR *password)) +{ + COM_TRY_BEGIN + return Callback->CryptoGetTextPassword2(passwordIsDefined, password); + COM_TRY_END +} + +Z7_COM7F_IMF(CArchiveUpdateCallback::CryptoGetTextPassword(BSTR *password)) +{ + COM_TRY_BEGIN + return Callback->CryptoGetTextPassword(password); + COM_TRY_END +} + +HRESULT CArchiveUpdateCallback::InFileStream_On_Error(UINT_PTR val, DWORD error) +{ + #ifdef _WIN32 // FIX IT !!! + // why did we check only for ERROR_LOCK_VIOLATION ? + // if (error == ERROR_LOCK_VIOLATION) + #endif + { + MT_LOCK + const UInt32 index = (UInt32)val; + FOR_VECTOR(i, _openFiles_Indexes) + { + if (_openFiles_Indexes[i] == index) + { + RINOK(Callback->ReadingFileError(_openFiles_Paths[i], error)) + break; + } + } + } + return HRESULT_FROM_WIN32(error); +} + +void CArchiveUpdateCallback::InFileStream_On_Destroy(CInFileStream *stream, UINT_PTR val) +{ + MT_LOCK + if (Need_LatestMTime) + { + if (stream->_info_WasLoaded) + { + const CFiTime &ft = ST_MTIME(stream->_info); + if (!LatestMTime_Defined + || Compare_FiTime(&LatestMTime, &ft) < 0) + LatestMTime = ft; + LatestMTime_Defined = true; + } + } + const UInt32 index = (UInt32)val; + FOR_VECTOR(i, _openFiles_Indexes) + { + if (_openFiles_Indexes[i] == index) + { + _openFiles_Indexes.Delete(i); + _openFiles_Paths.Delete(i); + // _openFiles_Streams.Delete(i); + return; + } + } + /* 21.02 : this function can be called in destructor. + And destructor can be called after some exception. + If we don't want to throw exception in desctructors or after another exceptions, + we must disable the code below that raises new exception. + */ + // throw 20141125; +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdatePair.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdatePair.cpp new file mode 100644 index 000000000..cd736da56 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdatePair.cpp @@ -0,0 +1,302 @@ +// UpdatePair.cpp + +#include "StdAfx.h" + +#include +// #include + +#include "../../../Common/Wildcard.h" + +#include "../../../Windows/TimeUtils.h" + +#include "SortUtils.h" +#include "UpdatePair.h" + +using namespace NWindows; +using namespace NTime; + + +/* + a2.Prec = + { + 0 (k_PropVar_TimePrec_0): + if GetProperty(kpidMTime) returned 0 and + GetProperty(kpidTimeType) did not returned VT_UI4. + 7z, wim, tar in 7-Zip before v21) + in that case we use + (prec) that is set by IOutArchive::GetFileTimeType() + } +*/ + +static int MyCompareTime(unsigned prec, const CFiTime &f1, const CArcTime &a2) +{ + // except of precision, we also have limitation, when timestamp is out of range + + /* if (Prec) in archive item is defined, then use global (prec) */ + if (a2.Prec != k_PropVar_TimePrec_0) + prec = a2.Prec; + + CArcTime a1; + a1.Set_From_FiTime(f1); + /* Set_From_FiTime() must set full form precision: + k_PropVar_TimePrec_Base + numDigits + windows: 7 digits, non-windows: 9 digits */ + + if (prec == k_PropVar_TimePrec_DOS) + { + const UInt32 dosTime1 = a1.Get_DosTime(); + const UInt32 dosTime2 = a2.Get_DosTime(); + return MyCompare(dosTime1, dosTime2); + } + + if (prec == k_PropVar_TimePrec_Unix) + { + const Int64 u2 = FileTime_To_UnixTime64(a2.FT); + if (u2 == 0 || u2 == (UInt32)0xFFFFFFFF) + { + // timestamp probably was saturated in archive to 32-bit + // so we use saturated 32-bit value for disk file too. + UInt32 u1; + FileTime_To_UnixTime(a1.FT, u1); + const UInt32 u2_32 = (UInt32)u2; + return MyCompare(u1, u2_32); + } + + const Int64 u1 = FileTime_To_UnixTime64(a1.FT); + return MyCompare(u1, u2); + // prec = k_PropVar_TimePrec_Base; // for debug + } + + if (prec == k_PropVar_TimePrec_0) + prec = k_PropVar_TimePrec_Base + 7; + else if (prec == k_PropVar_TimePrec_HighPrec) + prec = k_PropVar_TimePrec_Base + 9; + else if (prec < k_PropVar_TimePrec_Base) + prec = k_PropVar_TimePrec_Base; + else if (prec > k_PropVar_TimePrec_Base + 9) + prec = k_PropVar_TimePrec_Base + 7; + + // prec now is full form: k_PropVar_TimePrec_Base + numDigits; + if (prec > a1.Prec && a1.Prec >= k_PropVar_TimePrec_Base) + prec = a1.Prec; + + const unsigned numDigits = prec - k_PropVar_TimePrec_Base; + if (numDigits >= 7) + { + const int comp = CompareFileTime(&a1.FT, &a2.FT); + if (comp != 0 || numDigits == 7) + return comp; + return MyCompare(a1.Ns100, a2.Ns100); + } + UInt32 d = 1; + for (unsigned k = numDigits; k < 7; k++) + d *= 10; + const UInt64 v1 = a1.Get_FILETIME_as_UInt64() / d * d; + const UInt64 v2 = a2.Get_FILETIME_as_UInt64() / d * d; + // printf("\ndelta=%d numDigits=%d\n", (unsigned)(v1- v2), numDigits); + return MyCompare(v1, v2); +} + + + +static const char * const k_Duplicate_inArc_Message = "Duplicate filename in archive:"; +static const char * const k_Duplicate_inDir_Message = "Duplicate filename on disk:"; +static const char * const k_NotCensoredCollision_Message = "Internal file name collision (file on disk, file in archive):"; + +Z7_ATTR_NORETURN +static +void ThrowError(const char *message, const UString &s1, const UString &s2) +{ + UString m (message); + m.Add_LF(); m += s1; + m.Add_LF(); m += s2; + throw m; +} + +static int CompareArcItemsBase(const CArcItem &ai1, const CArcItem &ai2) +{ + const int res = CompareFileNames(ai1.Name, ai2.Name); + if (res != 0) + return res; + if (ai1.IsDir != ai2.IsDir) + return ai1.IsDir ? -1 : 1; + return 0; +} + +static int CompareArcItems(const unsigned *p1, const unsigned *p2, void *param) +{ + const unsigned i1 = *p1; + const unsigned i2 = *p2; + const CObjectVector &arcItems = *(const CObjectVector *)param; + const int res = CompareArcItemsBase(arcItems[i1], arcItems[i2]); + if (res != 0) + return res; + return MyCompare(i1, i2); +} + +void GetUpdatePairInfoList( + const CDirItems &dirItems, + const CObjectVector &arcItems, + NFileTimeType::EEnum fileTimeType, + CRecordVector &updatePairs) +{ + CUIntVector dirIndices, arcIndices; + + const unsigned numDirItems = dirItems.Items.Size(); + const unsigned numArcItems = arcItems.Size(); + + CIntArr duplicatedArcItem(numArcItems); + { + int *vals = &duplicatedArcItem[0]; + for (unsigned i = 0; i < numArcItems; i++) + vals[i] = 0; + } + + { + arcIndices.ClearAndSetSize(numArcItems); + if (numArcItems != 0) + { + unsigned *vals = &arcIndices[0]; + for (unsigned i = 0; i < numArcItems; i++) + vals[i] = i; + } + arcIndices.Sort(CompareArcItems, (void *)&arcItems); + for (unsigned i = 0; i + 1 < numArcItems; i++) + if (CompareArcItemsBase( + arcItems[arcIndices[i]], + arcItems[arcIndices[i + 1]]) == 0) + { + duplicatedArcItem[i] = 1; + duplicatedArcItem[i + 1] = -1; + } + } + + UStringVector dirNames; + { + dirNames.ClearAndReserve(numDirItems); + unsigned i; + for (i = 0; i < numDirItems; i++) + dirNames.AddInReserved(dirItems.GetLogPath(i)); + SortFileNames(dirNames, dirIndices); + for (i = 0; i + 1 < numDirItems; i++) + { + const UString &s1 = dirNames[dirIndices[i]]; + const UString &s2 = dirNames[dirIndices[i + 1]]; + if (CompareFileNames(s1, s2) == 0) + ThrowError(k_Duplicate_inDir_Message, s1, s2); + } + } + + unsigned dirIndex = 0; + unsigned arcIndex = 0; + + int prevHostFile = -1; + const UString *prevHostName = NULL; + + while (dirIndex < numDirItems || arcIndex < numArcItems) + { + CUpdatePair pair; + + int dirIndex2 = -1; + int arcIndex2 = -1; + const CDirItem *di = NULL; + const CArcItem *ai = NULL; + + int compareResult = -1; + const UString *name = NULL; + + if (dirIndex < numDirItems) + { + dirIndex2 = (int)dirIndices[dirIndex]; + di = &dirItems.Items[(unsigned)dirIndex2]; + } + + if (arcIndex < numArcItems) + { + arcIndex2 = (int)arcIndices[arcIndex]; + ai = &arcItems[(unsigned)arcIndex2]; + compareResult = 1; + if (dirIndex < numDirItems) + { + compareResult = CompareFileNames(dirNames[(unsigned)dirIndex2], ai->Name); + if (compareResult == 0) + { + if (di->IsDir() != ai->IsDir) + compareResult = (ai->IsDir ? 1 : -1); + } + } + } + + if (compareResult < 0) + { + name = &dirNames[(unsigned)dirIndex2]; + pair.State = NUpdateArchive::NPairState::kOnlyOnDisk; + pair.DirIndex = dirIndex2; + dirIndex++; + } + else if (compareResult > 0) + { + name = &ai->Name; + pair.State = ai->Censored ? + NUpdateArchive::NPairState::kOnlyInArchive: + NUpdateArchive::NPairState::kNotMasked; + pair.ArcIndex = arcIndex2; + arcIndex++; + } + else + { + const int dupl = duplicatedArcItem[arcIndex]; + if (dupl != 0) + ThrowError(k_Duplicate_inArc_Message, ai->Name, arcItems[arcIndices[(unsigned)((int)arcIndex + dupl)]].Name); + + name = &dirNames[(unsigned)dirIndex2]; + if (!ai->Censored) + ThrowError(k_NotCensoredCollision_Message, *name, ai->Name); + + pair.DirIndex = dirIndex2; + pair.ArcIndex = arcIndex2; + + int compResult = 0; + if (ai->MTime.Def) + { + compResult = MyCompareTime((unsigned)fileTimeType, di->MTime, ai->MTime); + } + switch (compResult) + { + case -1: pair.State = NUpdateArchive::NPairState::kNewInArchive; break; + case 1: pair.State = NUpdateArchive::NPairState::kOldInArchive; break; + default: + pair.State = (ai->Size_Defined && di->Size == ai->Size) ? + NUpdateArchive::NPairState::kSameFiles : + NUpdateArchive::NPairState::kUnknowNewerFiles; + } + + dirIndex++; + arcIndex++; + } + + if ( + #ifdef _WIN32 + (di && di->IsAltStream) || + #endif + (ai && ai->IsAltStream)) + { + if (prevHostName) + { + const unsigned hostLen = prevHostName->Len(); + if (name->Len() > hostLen) + if ((*name)[hostLen] == ':' && CompareFileNames(*prevHostName, name->Left(hostLen)) == 0) + pair.HostIndex = prevHostFile; + } + } + else + { + prevHostFile = (int)updatePairs.Size(); + prevHostName = name; + } + + updatePairs.Add(pair); + } + + updatePairs.ReserveDown(); +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateProduce.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateProduce.cpp new file mode 100644 index 000000000..416cceba5 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Common/UpdateProduce.cpp @@ -0,0 +1,74 @@ +// UpdateProduce.cpp + +#include "StdAfx.h" + +#include "UpdateProduce.h" + +using namespace NUpdateArchive; + +static const char * const kUpdateActionSetCollision = "Internal collision in update action set"; + +void UpdateProduce( + const CRecordVector &updatePairs, + const CActionSet &actionSet, + CRecordVector &operationChain, + IUpdateProduceCallback *callback) +{ + FOR_VECTOR (i, updatePairs) + { + const CUpdatePair &pair = updatePairs[i]; + + CUpdatePair2 up2; + up2.DirIndex = pair.DirIndex; + up2.ArcIndex = pair.ArcIndex; + up2.NewData = up2.NewProps = true; + up2.UseArcProps = false; + + switch ((int)actionSet.StateActions[(unsigned)pair.State]) + { + case NPairAction::kIgnore: + if (pair.ArcIndex >= 0 && callback) + callback->ShowDeleteFile((unsigned)pair.ArcIndex); + continue; + + case NPairAction::kCopy: + if (pair.State == NPairState::kOnlyOnDisk) + throw kUpdateActionSetCollision; + if (pair.State == NPairState::kOnlyInArchive) + { + if (pair.HostIndex >= 0) + { + /* + ignore alt stream if + 1) no such alt stream in Disk + 2) there is Host file in disk + */ + if (updatePairs[(unsigned)pair.HostIndex].DirIndex >= 0) + continue; + } + } + up2.NewData = up2.NewProps = false; + up2.UseArcProps = true; + break; + + case NPairAction::kCompress: + if (pair.State == NPairState::kOnlyInArchive || + pair.State == NPairState::kNotMasked) + throw kUpdateActionSetCollision; + break; + + case NPairAction::kCompressAsAnti: + up2.IsAnti = true; + up2.UseArcProps = (pair.ArcIndex >= 0); + break; + + default: throw 123; // break; // is unexpected case + } + + up2.IsSameTime = ((unsigned)pair.State == NUpdateArchive::NPairState::kSameFiles); + + operationChain.Add(up2); + } + + operationChain.ReserveDown(); +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.cpp new file mode 100644 index 000000000..9b97fcc32 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.cpp @@ -0,0 +1,41 @@ +// BenchCon.cpp + +#include "StdAfx.h" + +#include "../Common/Bench.h" + +#include "BenchCon.h" +#include "ConsoleClose.h" + +struct CPrintBenchCallback Z7_final: public IBenchPrintCallback +{ + FILE *_file; + + void Print(const char *s) Z7_override; + void NewLine() Z7_override; + HRESULT CheckBreak() Z7_override; +}; + +void CPrintBenchCallback::Print(const char *s) +{ + fputs(s, _file); +} + +void CPrintBenchCallback::NewLine() +{ + fputc('\n', _file); +} + +HRESULT CPrintBenchCallback::CheckBreak() +{ + return NConsoleClose::TestBreakSignal() ? E_ABORT: S_OK; +} + +HRESULT BenchCon(DECL_EXTERNAL_CODECS_LOC_VARS + const CObjectVector &props, UInt32 numIterations, FILE *f) +{ + CPrintBenchCallback callback; + callback._file = f; + return Bench(EXTERNAL_CODECS_LOC_VARS + &callback, NULL, props, numIterations, true); +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.h b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.h new file mode 100644 index 000000000..09896d82c --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/BenchCon.h @@ -0,0 +1,14 @@ +// BenchCon.h + +#ifndef ZIP7_INC_BENCH_CON_H +#define ZIP7_INC_BENCH_CON_H + +#include + +#include "../../Common/CreateCoder.h" +#include "../../UI/Common/Property.h" + +HRESULT BenchCon(DECL_EXTERNAL_CODECS_LOC_VARS + const CObjectVector &props, UInt32 numIterations, FILE *f); + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.cpp new file mode 100644 index 000000000..be5510760 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.cpp @@ -0,0 +1,426 @@ +// HashCon.cpp + +#include "StdAfx.h" + +#include "../../../Common/IntToString.h" + +#include "../../../Windows/FileName.h" + +#include "ConsoleClose.h" +#include "HashCon.h" + +static const char * const kEmptyFileAlias = "[Content]"; + +static const char * const kScanningMessage = "Scanning"; + +static HRESULT CheckBreak2() +{ + return NConsoleClose::TestBreakSignal() ? E_ABORT : S_OK; +} + +HRESULT CHashCallbackConsole::CheckBreak() +{ + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::StartScanning() +{ + if (PrintHeaders && _so) + *_so << kScanningMessage << endl; + if (NeedPercents()) + { + _percent.ClearCurState(); + _percent.Command = "Scan"; + } + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::ScanProgress(const CDirItemsStat &st, const FString &path, bool isDir) +{ + if (NeedPercents()) + { + _percent.Files = st.NumDirs + st.NumFiles + st.NumAltStreams; + _percent.Completed = st.GetTotalBytes(); + _percent.FileName = fs2us(path); + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_percent.FileName); + _percent.Print(); + } + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::ScanError(const FString &path, DWORD systemError) +{ + return ScanError_Base(path, systemError); +} + +void Print_DirItemsStat(AString &s, const CDirItemsStat &st); + +HRESULT CHashCallbackConsole::FinishScanning(const CDirItemsStat &st) +{ + if (NeedPercents()) + { + _percent.ClosePrint(true); + _percent.ClearCurState(); + } + if (PrintHeaders && _so) + { + Print_DirItemsStat(_s, st); + *_so << _s << endl << endl; + } + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::SetNumFiles(UInt64 /* numFiles */) +{ + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::SetTotal(UInt64 size) +{ + if (NeedPercents()) + { + _percent.Total = size; + _percent.Print(); + } + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::SetCompleted(const UInt64 *completeValue) +{ + if (completeValue && NeedPercents()) + { + _percent.Completed = *completeValue; + _percent.Print(); + } + return CheckBreak2(); +} + +static void AddMinuses(AString &s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + s.Add_Minus(); +} + +static void AddSpaces_if_Positive(AString &s, int num) +{ + for (int i = 0; i < num; i++) + s.Add_Space(); +} + +static void SetSpacesAndNul(char *s, unsigned num) +{ + for (unsigned i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + +static void SetSpacesAndNul_if_Positive(char *s, int num) +{ + if (num < 0) + return; + for (int i = 0; i < num; i++) + s[i] = ' '; + s[num] = 0; +} + +static const unsigned kSizeField_Len = 13; +static const unsigned kNameField_Len = 12; + +static const unsigned kHashColumnWidth_Min = 4 * 2; + +static unsigned GetColumnWidth(unsigned digestSize) +{ + unsigned width = digestSize * 2; + return width < kHashColumnWidth_Min ? kHashColumnWidth_Min: width; +} + + +AString CHashCallbackConsole::GetFields() const +{ + AString s (PrintFields); + if (s.IsEmpty()) + s = "hsn"; + s.MakeLower_Ascii(); + return s; +} + + +void CHashCallbackConsole::PrintSeparatorLine(const CObjectVector &hashers) +{ + _s.Empty(); + const AString fields = GetFields(); + for (unsigned pos = 0; pos < fields.Len(); pos++) + { + const char c = fields[pos]; + if (c == 'h') + { + for (unsigned i = 0; i < hashers.Size(); i++) + { + AddSpace(); + const CHasherState &h = hashers[i]; + AddMinuses(_s, GetColumnWidth(h.DigestSize)); + } + } + else if (c == 's') + { + AddSpace(); + AddMinuses(_s, kSizeField_Len); + } + else if (c == 'n') + { + AddSpacesBeforeName(); + AddMinuses(_s, kNameField_Len); + } + } + + *_so << _s << endl; +} + + +HRESULT CHashCallbackConsole::BeforeFirstFile(const CHashBundle &hb) +{ + if (PrintHeaders && _so) + { + _s.Empty(); + ClosePercents_for_so(); + + const AString fields = GetFields(); + for (unsigned pos = 0; pos < fields.Len(); pos++) + { + const char c = fields[pos]; + if (c == 'h') + { + FOR_VECTOR (i, hb.Hashers) + { + AddSpace(); + const CHasherState &h = hb.Hashers[i]; + _s += h.Name; + AddSpaces_if_Positive(_s, (int)GetColumnWidth(h.DigestSize) - (int)h.Name.Len()); + } + } + + else if (c == 's') + { + AddSpace(); + const AString s2 ("Size"); + AddSpaces_if_Positive(_s, (int)kSizeField_Len - (int)s2.Len()); + _s += s2; + } + else if (c == 'n') + { + AddSpacesBeforeName(); + _s += "Name"; + } + } + + *_so << _s << endl; + PrintSeparatorLine(hb.Hashers); + } + + return CheckBreak2(); +} + +HRESULT CHashCallbackConsole::OpenFileError(const FString &path, DWORD systemError) +{ + return OpenFileError_Base(path, systemError); +} + +HRESULT CHashCallbackConsole::GetStream(const wchar_t *name, bool isDir) +{ + _fileName = name; + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_fileName); + + if (NeedPercents()) + { + if (PrintNameInPercents) + { + _percent.FileName.Empty(); + if (name) + _percent.FileName = name; + } + _percent.Print(); + } + return CheckBreak2(); +} + + +static const unsigned k_DigestStringSize = k_HashCalc_DigestSize_Max * 2 + k_HashCalc_ExtraSize * 2 + 16; + + + +void CHashCallbackConsole::PrintResultLine(UInt64 fileSize, + const CObjectVector &hashers, unsigned digestIndex, bool showHash, + const AString &path) +{ + ClosePercents_for_so(); + + _s.Empty(); + const AString fields = GetFields(); + + for (unsigned pos = 0; pos < fields.Len(); pos++) + { + const char c = fields[pos]; + if (c == 'h') + { + FOR_VECTOR (i, hashers) + { + AddSpace(); + const CHasherState &h = hashers[i]; + char s[k_DigestStringSize]; + s[0] = 0; + if (showHash) + h.WriteToString(digestIndex, s); + const unsigned len = (unsigned)strlen(s); + SetSpacesAndNul_if_Positive(s + len, (int)GetColumnWidth(h.DigestSize) - (int)len); + _s += s; + } + } + else if (c == 's') + { + AddSpace(); + char s[kSizeField_Len + 32]; + char *p = s; + SetSpacesAndNul(s, kSizeField_Len); + if (showHash) + { + p = s + kSizeField_Len; + ConvertUInt64ToString(fileSize, p); + const int numSpaces = (int)kSizeField_Len - (int)strlen(p); + if (numSpaces > 0) + p -= (unsigned)numSpaces; + } + _s += p; + } + else if (c == 'n') + { + AddSpacesBeforeName(); + _s += path; + } + } + + *_so << _s; +} + + +HRESULT CHashCallbackConsole::SetOperationResult(UInt64 fileSize, const CHashBundle &hb, bool showHash) +{ + if (_so) + { + AString s; + if (_fileName.IsEmpty()) + s = kEmptyFileAlias; + else + { + UString temp (_fileName); + _so->Normalize_UString_Path(temp); + _so->Convert_UString_to_AString(temp, s); + } + PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash, s); + + /* + PrintResultLine(fileSize, hb.Hashers, k_HashCalc_Index_Current, showHash); + if (PrintName) + { + if (_fileName.IsEmpty()) + *_so << kEmptyFileAlias; + else + _so->NormalizePrint_UString(_fileName); + } + */ + // if (PrintNewLine) + *_so << endl; + } + + if (NeedPercents()) + { + _percent.Files++; + _percent.Print(); + } + + return CheckBreak2(); +} + +static const char * const k_DigestTitles[] = +{ + " : " + , " for data: " + , " for data and names: " + , " for streams and names: " +}; + +static void PrintSum(CStdOutStream &so, const CHasherState &h, unsigned digestIndex) +{ + so << h.Name; + + { + AString temp; + AddSpaces_if_Positive(temp, 6 - (int)h.Name.Len()); + so << temp; + } + + so << k_DigestTitles[digestIndex]; + + char s[k_DigestStringSize]; + // s[0] = 0; + h.WriteToString(digestIndex, s); + so << s << endl; +} + +void PrintHashStat(CStdOutStream &so, const CHashBundle &hb) +{ + FOR_VECTOR (i, hb.Hashers) + { + const CHasherState &h = hb.Hashers[i]; + PrintSum(so, h, k_HashCalc_Index_DataSum); + if (hb.NumFiles != 1 || hb.NumDirs != 0) + PrintSum(so, h, k_HashCalc_Index_NamesSum); + if (hb.NumAltStreams != 0) + PrintSum(so, h, k_HashCalc_Index_StreamsSum); + so << endl; + } +} + +void CHashCallbackConsole::PrintProperty(const char *name, UInt64 value) +{ + char s[32]; + s[0] = ':'; + s[1] = ' '; + ConvertUInt64ToString(value, s + 2); + *_so << name << s << endl; +} + +HRESULT CHashCallbackConsole::AfterLastFile(CHashBundle &hb) +{ + ClosePercents2(); + + if (PrintHeaders && _so) + { + PrintSeparatorLine(hb.Hashers); + + PrintResultLine(hb.FilesSize, hb.Hashers, k_HashCalc_Index_DataSum, true, AString()); + + *_so << endl << endl; + + if (hb.NumFiles != 1 || hb.NumDirs != 0) + { + if (hb.NumDirs != 0) + PrintProperty("Folders", hb.NumDirs); + PrintProperty("Files", hb.NumFiles); + } + + PrintProperty("Size", hb.FilesSize); + + if (hb.NumAltStreams != 0) + { + PrintProperty("Alternate streams", hb.NumAltStreams); + PrintProperty("Alternate streams size", hb.AltStreamsSize); + } + + *_so << endl; + PrintHashStat(*_so, hb); + } + + return S_OK; +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.h b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.h new file mode 100644 index 000000000..6a4117af5 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/HashCon.h @@ -0,0 +1,58 @@ +// HashCon.h + +#ifndef ZIP7_INC_HASH_CON_H +#define ZIP7_INC_HASH_CON_H + +#include "../Common/HashCalc.h" + +#include "UpdateCallbackConsole.h" + +class CHashCallbackConsole Z7_final: + public IHashCallbackUI, + public CCallbackConsoleBase +{ + Z7_IFACE_IMP(IDirItemsCallback) + Z7_IFACE_IMP(IHashCallbackUI) + + UString _fileName; + AString _s; + + void AddSpace() + { + _s.Add_Space_if_NotEmpty(); + } + + void AddSpacesBeforeName() + { + if (!_s.IsEmpty()) + { + _s.Add_Space(); + _s.Add_Space(); + } + } + + void PrintSeparatorLine(const CObjectVector &hashers); + void PrintResultLine(UInt64 fileSize, + const CObjectVector &hashers, unsigned digestIndex, bool showHash, const AString &path); + void PrintProperty(const char *name, UInt64 value); + +public: + bool PrintNameInPercents; + bool PrintHeaders; + // bool PrintSize; + // bool PrintNewLine; // set it too (false), if you need only hash for single file without LF char. + AString PrintFields; + + AString GetFields() const; + + CHashCallbackConsole(): + PrintNameInPercents(true), + PrintHeaders(false) + // , PrintSize(true), + // , PrintNewLine(true) + {} +}; + +void PrintHashStat(CStdOutStream &so, const CHashBundle &hb); + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/Main.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/Main.cpp new file mode 100644 index 000000000..b8ab46ef7 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/Main.cpp @@ -0,0 +1,1643 @@ +// Main.cpp + +#include "StdAfx.h" + +#include "../../../Common/MyWindows.h" + +#ifdef _WIN32 + +#ifndef Z7_OLD_WIN_SDK + +#if defined(__MINGW32__) || defined(__MINGW64__) +#include +#else +#include +#endif + +#else // Z7_OLD_WIN_SDK + +typedef struct { + DWORD cb; + DWORD PageFaultCount; + SIZE_T PeakWorkingSetSize; + SIZE_T WorkingSetSize; + SIZE_T QuotaPeakPagedPoolUsage; + SIZE_T QuotaPagedPoolUsage; + SIZE_T QuotaPeakNonPagedPoolUsage; + SIZE_T QuotaNonPagedPoolUsage; + SIZE_T PagefileUsage; + SIZE_T PeakPagefileUsage; +} PROCESS_MEMORY_COUNTERS; +typedef PROCESS_MEMORY_COUNTERS *PPROCESS_MEMORY_COUNTERS; + +#endif // Z7_OLD_WIN_SDK + +#else // _WIN32 +#include +#include +#include +#include +#endif // _WIN32 + +#include "../../../../C/CpuArch.h" + +#include "../../../Common/MyInitGuid.h" + +#include "../../../Common/CommandLineParser.h" +#include "../../../Common/IntToString.h" +#include "../../../Common/MyException.h" +#include "../../../Common/StdInStream.h" +#include "../../../Common/StdOutStream.h" +#include "../../../Common/StringConvert.h" +#include "../../../Common/StringToInt.h" +#include "../../../Common/UTFConvert.h" + +#include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/TimeUtils.h" +#include "../../../Windows/FileDir.h" + +#include "../Common/ArchiveCommandLine.h" +#include "../Common/Bench.h" +#include "../Common/ExitCode.h" +#include "../Common/Extract.h" + +#ifdef Z7_EXTERNAL_CODECS +#include "../Common/LoadCodecs.h" +#endif + +#include "../../Common/RegisterCodec.h" + +#include "BenchCon.h" +#include "ConsoleClose.h" +#include "ExtractCallbackConsole.h" +#include "HashCon.h" +#include "List.h" +#include "OpenCallbackConsole.h" +#include "UpdateCallbackConsole.h" + +// **************** NanaZip Modification Start **************** +//#ifdef Z7_PROG_VARIANT_R +//#include "../../../../C/7zVersion.h" +//#else +//#include "../../MyVersion.h" +//#endif +#include +// **************** NanaZip Modification End **************** + +using namespace NWindows; +using namespace NFile; +using namespace NCommandLineParser; + +#ifdef _WIN32 +extern +HINSTANCE g_hInstance; +HINSTANCE g_hInstance = NULL; +#endif + +extern CStdOutStream *g_StdStream; +extern CStdOutStream *g_ErrStream; + +extern unsigned g_NumCodecs; +extern const CCodecInfo *g_Codecs[]; + +extern unsigned g_NumHashers; +extern const CHasherInfo *g_Hashers[]; + +#ifdef Z7_EXTERNAL_CODECS +extern +const CExternalCodecs *g_ExternalCodecs_Ptr; +const CExternalCodecs *g_ExternalCodecs_Ptr; +#endif + +DECLARE_AND_SET_CLIENT_VERSION_VAR + +#if defined(Z7_PROG_VARIANT_Z) + #define PROG_POSTFIX "z" + #define PROG_POSTFIX_2 " (z)" +#elif defined(Z7_PROG_VARIANT_R) + #define PROG_POSTFIX "r" + #define PROG_POSTFIX_2 " (r)" +#elif defined(Z7_PROG_VARIANT_A) || !defined(Z7_EXTERNAL_CODECS) + #define PROG_POSTFIX "a" + #define PROG_POSTFIX_2 " (a)" +#else + #define PROG_POSTFIX "" + #define PROG_POSTFIX_2 "" +#endif + +// **************** NanaZip Modification Start **************** +//static const char * const kCopyrightString = "\n7-Zip" +// PROG_POSTFIX_2 +// " " MY_VERSION_CPU +// " : " MY_COPYRIGHT_DATE "\n"; +static const char * const kCopyrightString = "\nNanaZip" + PROG_POSTFIX_2 + " " MILE_PROJECT_VERSION_UTF8_STRING " (" MY_CPU_NAME ")" + " : " "(c) M2-Team and Contributors. All rights reserved." "\n"; +// **************** NanaZip Modification End **************** + +static const char * const kHelpString = + // **************** NanaZip Modification Start **************** + // "Usage: 7z" + "Usage: NanaZipC" + // **************** NanaZip Modification End **************** + PROG_POSTFIX + " [...] [...] [@listfile]\n" + "\n" + "\n" + " a : Add files to archive\n" + " b : Benchmark\n" + " d : Delete files from archive\n" + " e : Extract files from archive (without using directory names)\n" + " h : Calculate hash values for files\n" + " i : Show information about supported formats\n" + " l : List contents of archive\n" + " rn : Rename files in archive\n" + " t : Test integrity of archive\n" + " u : Update files to archive\n" + " x : eXtract files with full paths\n" + "\n" + "\n" + " -- : Stop switches and @listfile parsing\n" + " -ai[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : Include archives\n" + " -ax[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : eXclude archives\n" + " -ao{a|s|t|u} : set Overwrite mode\n" + " -an : disable archive_name field\n" + " -bb[0-3] : set output log level\n" + " -bd : disable progress indicator\n" + " -bs{o|e|p}{0|1|2} : set output stream for output/error/progress line\n" + " -bt : show execution time statistics\n" + " -i[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : Include filenames\n" + " -m{Parameters} : set compression Method\n" + " -mmt[N] : set number of CPU threads\n" + " -mx[N] : set compression level: -mx1 (fastest) ... -mx9 (ultra)\n" + " -o{Directory} : set Output directory\n" + #ifndef Z7_NO_CRYPTO + " -p{Password} : set Password\n" + #endif + " -r[-|0] : Recurse subdirectories for name search\n" + " -sa{a|e|s} : set Archive name mode\n" + " -scc{UTF-8|WIN|DOS} : set charset for console input/output\n" + " -scs{UTF-8|UTF-16LE|UTF-16BE|WIN|DOS|{id}} : set charset for list files\n" + " -scrc[CRC32|CRC64|SHA256" +#ifndef Z7_PROG_VARIANT_R + "|SHA1|XXH64" +#ifdef Z7_PROG_VARIANT_Z + "|BLAKE2SP" +#endif +#endif + "|*] : set hash function for x, e, h commands\n" + " -sdel : delete files after compression\n" + " -seml[.] : send archive by email\n" + " -sfx[{name}] : Create SFX archive\n" + " -si[{name}] : read data from stdin\n" + " -slp : set Large Pages mode\n" + " -slt : show technical information for l (List) command\n" + " -snh : store hard links as links\n" + " -snl : store symbolic links as links\n" + " -sni : store NT security information\n" + " -sns[-] : store NTFS alternate streams\n" + " -so : write data to stdout\n" + " -spd : disable wildcard matching for file names\n" + " -spe : eliminate duplication of root folder for extract command\n" + " -spf[2] : use fully qualified file paths\n" + " -ssc[-] : set sensitive case mode\n" + " -sse : stop archive creating, if it can't open some input file\n" + " -ssp : do not change Last Access Time of source files while archiving\n" + " -ssw : compress shared files\n" + " -stl : set archive timestamp from the most recently modified file\n" + " -stm{HexMask} : set CPU thread affinity mask (hexadecimal number)\n" + " -stx{Type} : exclude archive type\n" + " -t{Type} : Set type of archive\n" + " -u[-][p#][q#][r#][x#][y#][z#][!newArchiveName] : Update options\n" + " -v{Size}[b|k|m|g] : Create volumes\n" + " -w[{path}] : assign Work directory. Empty path means a temporary directory\n" + " -x[r[-|0]][m[-|2]][w[-]]{@listfile|!wildcard} : eXclude filenames\n" + " -y : assume Yes on all queries\n"; + +// --------------------------- +// exception messages + +static const char * const kEverythingIsOk = "Everything is Ok"; +static const char * const kUserErrorMessage = "Incorrect command line"; +static const char * const kNoFormats = "7-Zip cannot find the code that works with archives."; +static const char * const kUnsupportedArcTypeMessage = "Unsupported archive type"; +// static const char * const kUnsupportedUpdateArcType = "Can't create archive for that type"; + +#ifndef Z7_EXTRACT_ONLY +#define kDefaultSfxModule "7zCon.sfx" +#endif + +Z7_ATTR_NORETURN +static void ShowMessageAndThrowException(LPCSTR message, NExitCode::EEnum code) +{ + if (g_ErrStream) + *g_ErrStream << endl << "ERROR: " << message << endl; + throw code; +} + + +#ifdef _WIN32 +#define ShowProgInfo(so) +#else +static void ShowProgInfo(CStdOutStream *so) +{ + if (!so) + return; + + *so + + /* + #ifdef __DATE__ + << " " << __DATE__ + #endif + #ifdef __TIME__ + << " " << __TIME__ + #endif + */ + + << " " << (unsigned)(sizeof(void *)) * 8 << "-bit" + + #ifdef __ILP32__ + << " ILP32" + #endif + + #ifdef __ARM_ARCH + << " arm_v:" << __ARM_ARCH + #if (__ARM_ARCH == 8) + // for macos: + #if defined(__ARM_ARCH_8_9__) + << ".9" + #elif defined(__ARM_ARCH_8_8__) + << ".8" + #elif defined(__ARM_ARCH_8_7__) + << ".7" + #elif defined(__ARM_ARCH_8_6__) + << ".6" + #elif defined(__ARM_ARCH_8_5__) + << ".5" + #elif defined(__ARM_ARCH_8_4__) + << ".4" + #elif defined(__ARM_ARCH_8_3__) + << ".3" + #elif defined(__ARM_ARCH_8_2__) + << ".2" + #elif defined(__ARM_ARCH_8_1__) + << ".1" + #endif + #endif + + #if defined(__ARM_ARCH_PROFILE) && \ + ( __ARM_ARCH_PROFILE >= 'A' && __ARM_ARCH_PROFILE <= 'Z' \ + || __ARM_ARCH_PROFILE >= 65 && __ARM_ARCH_PROFILE <= 65 + 25) + << "-" << (char)__ARM_ARCH_PROFILE + #endif + + #ifdef __ARM_ARCH_ISA_THUMB + << " thumb:" << __ARM_ARCH_ISA_THUMB + #endif + #endif + + #ifdef _MIPS_ARCH + << " mips_arch:" << _MIPS_ARCH + #endif + #ifdef __mips_isa_rev + << " mips_isa_rev:" << __mips_isa_rev + #endif + + #ifdef __iset__ + << " e2k_v:" << __iset__ + #endif + ; + + + + #ifdef ENV_HAVE_LOCALE + *so << " locale=" << GetLocale(); + #endif + #ifndef _WIN32 + { + const bool is_IsNativeUTF8 = IsNativeUTF8(); + if (!is_IsNativeUTF8) + *so << " UTF8=" << (is_IsNativeUTF8 ? "+" : "-"); + } + if (!g_ForceToUTF8) + *so << " use-UTF8=" << (g_ForceToUTF8 ? "+" : "-"); + { + const unsigned wchar_t_size = (unsigned)sizeof(wchar_t); + if (wchar_t_size != 4) + *so << " wchar_t=" << wchar_t_size * 8 << "-bit"; + } + { + const unsigned off_t_size = (unsigned)sizeof(off_t); + if (off_t_size != 8) + *so << " Files=" << off_t_size * 8 << "-bit"; + } + #endif + + { + const UInt32 numCpus = NWindows::NSystem::GetNumberOfProcessors(); + *so << " Threads:" << numCpus; + const UInt64 openMAX= NWindows::NSystem::Get_File_OPEN_MAX(); + *so << " OPEN_MAX:" << openMAX; + { + FString temp; + NDir::MyGetTempPath(temp); + if (!temp.IsEqualTo(STRING_PATH_SEPARATOR "tmp" STRING_PATH_SEPARATOR)) + *so << " temp_path:" << temp; + } + } + + #ifdef Z7_7ZIP_ASM + *so << ", ASM"; + #endif + + /* + { + AString s; + GetCpuName(s); + s.Trim(); + *so << ", " << s; + } + + #ifdef __ARM_FEATURE_CRC32 + << " CRC32" + #endif + + + #if (defined MY_CPU_X86_OR_AMD64 || defined(MY_CPU_ARM_OR_ARM64)) + if (CPU_IsSupported_AES()) *so << ",AES"; + #endif + + #ifdef MY_CPU_ARM_OR_ARM64 + if (CPU_IsSupported_CRC32()) *so << ",CRC32"; + #if defined(_WIN32) + if (CPU_IsSupported_CRYPTO()) *so << ",CRYPTO"; + #else + if (CPU_IsSupported_SHA1()) *so << ",SHA1"; + if (CPU_IsSupported_SHA2()) *so << ",SHA2"; + #endif + #endif + */ + + *so << endl; +} +#endif + +static void ShowCopyrightAndHelp(CStdOutStream *so, bool needHelp) +{ + if (!so) + return; + *so << kCopyrightString; + // *so << "# CPUs: " << (UInt64)NWindows::NSystem::GetNumberOfProcessors() << endl; + ShowProgInfo(so); + *so << endl; + if (needHelp) + *so << kHelpString; +} + + +static void PrintStringRight(CStdOutStream &so, const char *s, unsigned size) +{ + unsigned len = MyStringLen(s); + for (unsigned i = len; i < size; i++) + so << ' '; + so << s; +} + +static void PrintUInt32(CStdOutStream &so, UInt32 val, unsigned size) +{ + char s[16]; + ConvertUInt32ToString(val, s); + PrintStringRight(so, s, size); +} + +#ifdef Z7_EXTERNAL_CODECS +static void PrintNumber(CStdOutStream &so, UInt32 val, unsigned numDigits) +{ + AString s; + s.Add_UInt32(val); + while (s.Len() < numDigits) + s.InsertAtFront('0'); + so << s; +} +#endif + +static void PrintLibIndex(CStdOutStream &so, int libIndex) +{ + if (libIndex >= 0) + PrintUInt32(so, (UInt32)libIndex, 2); + else + so << " "; + so << ' '; +} + +static void PrintString(CStdOutStream &so, const UString &s, unsigned size) +{ + unsigned len = s.Len(); + so << s; + for (unsigned i = len; i < size; i++) + so << ' '; +} + +static void PrintWarningsPaths(const CErrorPathCodes &pc, CStdOutStream &so) +{ + FOR_VECTOR(i, pc.Paths) + { + so.NormalizePrint_UString_Path(fs2us(pc.Paths[i])); + so << " : "; + so << NError::MyFormatMessage(pc.Codes[i]) << endl; + } + so << "----------------" << endl; +} + +static int WarningsCheck(HRESULT result, const CCallbackConsoleBase &callback, + const CUpdateErrorInfo &errorInfo, + CStdOutStream *so, + CStdOutStream *se, + bool showHeaders) +{ + int exitCode = NExitCode::kSuccess; + + if (callback.ScanErrors.Paths.Size() != 0) + { + if (se) + { + *se << endl; + *se << "Scan WARNINGS for files and folders:" << endl << endl; + PrintWarningsPaths(callback.ScanErrors, *se); + *se << "Scan WARNINGS: " << callback.ScanErrors.Paths.Size(); + *se << endl; + } + exitCode = NExitCode::kWarning; + } + + if (result != S_OK || errorInfo.ThereIsError()) + { + if (se) + { + UString message; + if (!errorInfo.Message.IsEmpty()) + { + message += errorInfo.Message.Ptr(); + message.Add_LF(); + } + { + FOR_VECTOR(i, errorInfo.FileNames) + { + message += fs2us(errorInfo.FileNames[i]); + message.Add_LF(); + } + } + if (errorInfo.SystemError != 0) + { + message += NError::MyFormatMessage(errorInfo.SystemError); + message.Add_LF(); + } + if (!message.IsEmpty()) + *se << L"\nError:\n" << message; + } + + // we will work with (result) later + // throw CSystemException(result); + return NExitCode::kFatalError; + } + + unsigned numErrors = callback.FailedFiles.Paths.Size(); + if (numErrors == 0) + { + if (showHeaders) + if (callback.ScanErrors.Paths.Size() == 0) + if (so) + { + if (se) + se->Flush(); + *so << kEverythingIsOk << endl; + } + } + else + { + if (se) + { + *se << endl; + *se << "WARNINGS for files:" << endl << endl; + PrintWarningsPaths(callback.FailedFiles, *se); + *se << "WARNING: Cannot open " << numErrors << " file"; + if (numErrors > 1) + *se << 's'; + *se << endl; + } + exitCode = NExitCode::kWarning; + } + + return exitCode; +} + +static void ThrowException_if_Error(HRESULT res) +{ + if (res != S_OK) + throw CSystemException(res); +} + +static void PrintNum(UInt64 val, unsigned numDigits, char c = ' ') +{ + char temp[64]; + char *p = temp + 32; + ConvertUInt64ToString(val, p); + unsigned len = MyStringLen(p); + for (; len < numDigits; len++) + *--p = c; + *g_StdStream << p; +} + +#ifdef _WIN32 + +static void PrintTime(const char *s, UInt64 val, UInt64 total) +{ + *g_StdStream << endl << s << " Time ="; + const UInt32 kFreq = 10000000; + UInt64 sec = val / kFreq; + PrintNum(sec, 6); + *g_StdStream << '.'; + UInt32 ms = (UInt32)(val - (sec * kFreq)) / (kFreq / 1000); + PrintNum(ms, 3, '0'); + + while (val > ((UInt64)1 << 56)) + { + val >>= 1; + total >>= 1; + } + + UInt64 percent = 0; + if (total != 0) + percent = val * 100 / total; + *g_StdStream << " ="; + PrintNum(percent, 5); + *g_StdStream << '%'; +} + +#ifndef UNDER_CE + +#define SHIFT_SIZE_VALUE(x, num) (((x) + (1 << (num)) - 1) >> (num)) + +static void PrintMemUsage(const char *s, UInt64 val) +{ + *g_StdStream << " " << s << " Memory ="; + PrintNum(SHIFT_SIZE_VALUE(val, 20), 7); + *g_StdStream << " MB"; + /* + *g_StdStream << " ="; + PrintNum(SHIFT_SIZE_VALUE(val, 10), 9); + *g_StdStream << " KB"; + */ + #ifdef Z7_LARGE_PAGES + AString lp; + Add_LargePages_String(lp); + if (!lp.IsEmpty()) + *g_StdStream << lp; + #endif +} + +EXTERN_C_BEGIN +typedef BOOL (WINAPI *Func_GetProcessMemoryInfo)(HANDLE Process, + PPROCESS_MEMORY_COUNTERS ppsmemCounters, DWORD cb); +typedef BOOL (WINAPI *Func_QueryProcessCycleTime)(HANDLE Process, PULONG64 CycleTime); +EXTERN_C_END + +#endif + +static inline UInt64 GetTime64(const FILETIME &t) { return ((UInt64)t.dwHighDateTime << 32) | t.dwLowDateTime; } + +static void PrintStat() +{ + FILETIME creationTimeFT, exitTimeFT, kernelTimeFT, userTimeFT; + if (! + #ifdef UNDER_CE + ::GetThreadTimes(::GetCurrentThread() + #else + // NT 3.5 + ::GetProcessTimes(::GetCurrentProcess() + #endif + , &creationTimeFT, &exitTimeFT, &kernelTimeFT, &userTimeFT)) + return; + FILETIME curTimeFT; + NTime::GetCurUtc_FiTime(curTimeFT); + + #ifndef UNDER_CE + +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + + PROCESS_MEMORY_COUNTERS m; + memset(&m, 0, sizeof(m)); + BOOL memDefined = FALSE; + BOOL cycleDefined = FALSE; + ULONG64 cycleTime = 0; + { + /* NT 4.0: GetProcessMemoryInfo() in Psapi.dll + Win7: new function K32GetProcessMemoryInfo() in kernel32.dll + It's faster to call kernel32.dll code than Psapi.dll code + GetProcessMemoryInfo() requires Psapi.lib + Psapi.lib in SDK7+ can link to K32GetProcessMemoryInfo in kernel32.dll + The program with K32GetProcessMemoryInfo will not work on systems before Win7 + // memDefined = GetProcessMemoryInfo(GetCurrentProcess(), &m, sizeof(m)); + */ + const HMODULE kern = ::GetModuleHandleW(L"kernel32.dll"); + Func_GetProcessMemoryInfo + my_GetProcessMemoryInfo = Z7_GET_PROC_ADDRESS( + Func_GetProcessMemoryInfo, kern, + "K32GetProcessMemoryInfo"); + if (!my_GetProcessMemoryInfo) + { + const HMODULE lib = LoadLibraryW(L"Psapi.dll"); + if (lib) + my_GetProcessMemoryInfo = Z7_GET_PROC_ADDRESS( + Func_GetProcessMemoryInfo, lib, + "GetProcessMemoryInfo"); + } + if (my_GetProcessMemoryInfo) + memDefined = my_GetProcessMemoryInfo(GetCurrentProcess(), &m, sizeof(m)); + // FreeLibrary(lib); + const + Func_QueryProcessCycleTime + my_QueryProcessCycleTime = Z7_GET_PROC_ADDRESS( + Func_QueryProcessCycleTime, kern, + "QueryProcessCycleTime"); + if (my_QueryProcessCycleTime) + cycleDefined = my_QueryProcessCycleTime(GetCurrentProcess(), &cycleTime); + } + + #endif + + UInt64 curTime = GetTime64(curTimeFT); + UInt64 creationTime = GetTime64(creationTimeFT); + UInt64 kernelTime = GetTime64(kernelTimeFT); + UInt64 userTime = GetTime64(userTimeFT); + + UInt64 totalTime = curTime - creationTime; + + PrintTime("Kernel ", kernelTime, totalTime); + + const UInt64 processTime = kernelTime + userTime; + + #ifndef UNDER_CE + if (cycleDefined) + { + *g_StdStream << " Cnt:"; + PrintNum(cycleTime / 1000000, 15); + *g_StdStream << " MCycles"; + } + #endif + + PrintTime("User ", userTime, totalTime); + + #ifndef UNDER_CE + if (cycleDefined) + { + *g_StdStream << " Freq (cnt/ptime):"; + UInt64 us = processTime / 10; + if (us == 0) + us = 1; + PrintNum(cycleTime / us, 6); + *g_StdStream << " MHz"; + } + #endif + + PrintTime("Process", processTime, totalTime); + #ifndef UNDER_CE + if (memDefined) PrintMemUsage("Virtual ", m.PeakPagefileUsage); + #endif + + PrintTime("Global ", totalTime, totalTime); + #ifndef UNDER_CE + if (memDefined) PrintMemUsage("Physical", m.PeakWorkingSetSize); + #endif + *g_StdStream << endl; +} + + +#else // ! _WIN32 + +static UInt64 Get_timeofday_us() +{ + struct timeval now; + if (gettimeofday(&now, NULL) == 0) + return (UInt64)now.tv_sec * 1000000 + (UInt64)now.tv_usec; + return 0; +} + +static void PrintTime(const char *s, UInt64 val, UInt64 total_us, UInt64 kFreq) +{ + *g_StdStream << endl << s << " Time ="; + + { + UInt64 sec, ms; + + if (kFreq == 0) + { + sec = val / 1000000; + ms = val % 1000000 / 1000; + } + else + { + sec = val / kFreq; + ms = (UInt32)((val - (sec * kFreq)) * 1000 / kFreq); + } + + PrintNum(sec, 6); + *g_StdStream << '.'; + PrintNum(ms, 3, '0'); + } + + if (total_us == 0) + return; + + UInt64 percent = 0; + if (kFreq == 0) + percent = val * 100 / total_us; + else + { + const UInt64 kMaxVal = (UInt64)(Int64)-1; + UInt32 m = 100000000; + for (;;) + { + if (m == 0 || kFreq == 0) + break; + if (kMaxVal / m > val && + kMaxVal / kFreq > total_us) + break; + if (val > m) + val >>= 1; + else + m >>= 1; + if (kFreq > total_us) + kFreq >>= 1; + else + total_us >>= 1; + } + const UInt64 total = kFreq * total_us; + if (total != 0) + percent = val * m / total; + } + *g_StdStream << " ="; + PrintNum(percent, 5); + *g_StdStream << '%'; +} + +static void PrintStat(const UInt64 startTime) +{ + tms t; + /* clock_t res = */ times(&t); + const UInt64 totalTime = Get_timeofday_us() - startTime; + const UInt64 kFreq = (UInt64)sysconf(_SC_CLK_TCK); + PrintTime("Kernel ", (UInt64)t.tms_stime, totalTime, kFreq); + PrintTime("User ", (UInt64)t.tms_utime, totalTime, kFreq); + PrintTime("Process", (UInt64)t.tms_utime + (UInt64)t.tms_stime, totalTime, kFreq); + PrintTime("Global ", totalTime, totalTime, 0); + *g_StdStream << endl; +} + +#endif // ! _WIN32 + + + + + +static void PrintHexId(CStdOutStream &so, UInt64 id) +{ + char s[32]; + ConvertUInt64ToHex(id, s); + PrintStringRight(so, s, 8); +} + +#ifndef _WIN32 +void Set_ModuleDirPrefix_From_ProgArg0(const char *s); +#endif + +int Main2( + #ifndef _WIN32 + int numArgs, char *args[] + #endif +); +int Main2( + #ifndef _WIN32 + int numArgs, char *args[] + #endif +) +{ + #if defined(MY_CPU_SIZEOF_POINTER) + { unsigned k = sizeof(void *); if (k != MY_CPU_SIZEOF_POINTER) throw "incorrect MY_CPU_PTR_SIZE"; } + #endif + + #if defined(_WIN32) && !defined(UNDER_CE) + SetFileApisToOEM(); + #endif + + #ifdef ENV_HAVE_LOCALE + // printf("\nBefore SetLocale() : %s\n", IsNativeUtf8() ? "NATIVE UTF-8" : "IS NOT NATIVE UTF-8"); + MY_SetLocale(); + // printf("\nAfter SetLocale() : %s\n", IsNativeUtf8() ? "NATIVE UTF-8" : "IS NOT NATIVE UTF-8"); + #endif + + #ifndef _WIN32 + const UInt64 startTime = Get_timeofday_us(); + #endif + + /* + { + g_StdOut << "DWORD:" << (unsigned)sizeof(DWORD); + g_StdOut << " LONG:" << (unsigned)sizeof(LONG); + g_StdOut << " long:" << (unsigned)sizeof(long); + #ifdef _WIN64 + // g_StdOut << " long long:" << (unsigned)sizeof(long long); + #endif + g_StdOut << " int:" << (unsigned)sizeof(int); + g_StdOut << " void*:" << (unsigned)sizeof(void *); + g_StdOut << endl; + } + */ + + UStringVector commandStrings; + + #ifdef _WIN32 + NCommandLineParser::SplitCommandLine(GetCommandLineW(), commandStrings); + #else + { + if (numArgs > 0) + Set_ModuleDirPrefix_From_ProgArg0(args[0]); + + for (int i = 0; i < numArgs; i++) + { + AString a (args[i]); +#if 0 + printf("\n%d %s :", i, a.Ptr()); + for (unsigned k = 0; k < a.Len(); k++) + printf(" %2x", (unsigned)(Byte)a[k]); +#endif + const UString s = MultiByteToUnicodeString(a); + commandStrings.Add(s); + } + // printf("\n"); + } + + #endif + + #ifndef UNDER_CE + if (commandStrings.Size() > 0) + commandStrings.Delete(0); + #endif + + if (commandStrings.Size() == 0) + { + ShowCopyrightAndHelp(g_StdStream, true); + return 0; + } + + CArcCmdLineOptions options; + + CArcCmdLineParser parser; + + parser.Parse1(commandStrings, options); + + g_StdOut.IsTerminalMode = options.IsStdOutTerminal; + g_StdErr.IsTerminalMode = options.IsStdErrTerminal; + + if (options.Number_for_Out != k_OutStream_stdout) + g_StdStream = (options.Number_for_Out == k_OutStream_stderr ? &g_StdErr : NULL); + + if (options.Number_for_Errors != k_OutStream_stderr) + g_ErrStream = (options.Number_for_Errors == k_OutStream_stdout ? &g_StdOut : NULL); + + CStdOutStream *percentsStream = NULL; + if (options.Number_for_Percents != k_OutStream_disabled) + percentsStream = (options.Number_for_Percents == k_OutStream_stderr) ? &g_StdErr : &g_StdOut; + + if (options.HelpMode) + { + ShowCopyrightAndHelp(g_StdStream, true); + return 0; + } + + if (options.EnableHeaders) + { + ShowCopyrightAndHelp(g_StdStream, false); + if (!parser.Parse1Log.IsEmpty()) + *g_StdStream << parser.Parse1Log; + } + + parser.Parse2(options); + + { + int cp = options.ConsoleCodePage; + + int stdout_cp = cp; + int stderr_cp = cp; + int stdin_cp = cp; + + /* + // these cases are complicated. + // maybe we must use CRT functions instead of console WIN32. + // different Windows/CRT versions also can work different ways. + // so the following code was not enabled: + if (cp == -1) + { + // we set CodePage only if stream is attached to terminal + // maybe we should set CodePage even if is not terminal? + #ifdef _WIN32 + { + UINT ccp = GetConsoleOutputCP(); + if (ccp != 0) + { + if (options.IsStdOutTerminal) stdout_cp = ccp; + if (options.IsStdErrTerminal) stderr_cp = ccp; + } + } + if (options.IsInTerminal) + { + UINT ccp = GetConsoleCP(); + if (ccp != 0) stdin_cp = ccp; + } + #endif + } + */ + + if (stdout_cp != -1) g_StdOut.CodePage = stdout_cp; + if (stderr_cp != -1) g_StdErr.CodePage = stderr_cp; + if (stdin_cp != -1) g_StdIn.CodePage = stdin_cp; + } + g_StdOut.ListPathSeparatorSlash = options.ListPathSeparatorSlash; + g_StdErr.ListPathSeparatorSlash = options.ListPathSeparatorSlash; + + unsigned percentsNameLevel = 1; + if (options.LogLevel == 0 || options.Number_for_Percents != options.Number_for_Out) + percentsNameLevel = 2; + + unsigned consoleWidth = 80; + + if (percentsStream) + { + #ifdef _WIN32 + + #if !defined(UNDER_CE) + CONSOLE_SCREEN_BUFFER_INFO consoleInfo; + if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &consoleInfo)) + consoleWidth = (USHORT)consoleInfo.dwSize.X; + #endif + + #else + +#if !defined(__sun) + struct winsize w; + if (ioctl(0, TIOCGWINSZ, &w) == 0) + consoleWidth = w.ws_col; +#endif + #endif + } + + CREATE_CODECS_OBJECT + + codecs->CaseSensitive_Change = options.CaseSensitive_Change; + codecs->CaseSensitive = options.CaseSensitive; + ThrowException_if_Error(codecs->Load()); + Codecs_AddHashArcHandler(codecs); + + #ifdef Z7_EXTERNAL_CODECS + { + g_ExternalCodecs_Ptr = &_externalCodecs; + UString s; + codecs->GetCodecsErrorMessage(s); + if (!s.IsEmpty()) + { + CStdOutStream &so = (g_StdStream ? *g_StdStream : g_StdOut); + so << endl << s << endl; + } + } + #endif + + const bool isExtractGroupCommand = options.Command.IsFromExtractGroup(); + + if (codecs->Formats.Size() == 0 && + (isExtractGroupCommand + || options.Command.CommandType == NCommandType::kList + || options.Command.IsFromUpdateGroup())) + { + #ifdef Z7_EXTERNAL_CODECS + if (!codecs->MainDll_ErrorPath.IsEmpty()) + { + UString s ("Can't load module: "); + s += fs2us(codecs->MainDll_ErrorPath); + throw s; + } + #endif + throw kNoFormats; + } + + CObjectVector types; + if (!ParseOpenTypes(*codecs, options.ArcType, types)) + { + throw kUnsupportedArcTypeMessage; + } + + + CIntVector excludedFormats; + FOR_VECTOR (k, options.ExcludedArcTypes) + { + CIntVector tempIndices; + if (!codecs->FindFormatForArchiveType(options.ExcludedArcTypes[k], tempIndices) + || tempIndices.Size() != 1) + throw kUnsupportedArcTypeMessage; + + + + excludedFormats.AddToUniqueSorted(tempIndices[0]); + // excludedFormats.Sort(); + } + + #ifdef Z7_EXTERNAL_CODECS + if (isExtractGroupCommand + || options.Command.IsFromUpdateGroup() + || options.Command.CommandType == NCommandType::kHash + || options.Command.CommandType == NCommandType::kBenchmark) + ThrowException_if_Error(_externalCodecs.Load()); + #endif + + int retCode = NExitCode::kSuccess; + HRESULT hresultMain = S_OK; + + // bool showStat = options.ShowTime; + + /* + if (!options.EnableHeaders || + options.TechMode) + showStat = false; + */ + + + if (options.Command.CommandType == NCommandType::kInfo) + { + CStdOutStream &so = (g_StdStream ? *g_StdStream : g_StdOut); + unsigned i; + + #ifdef Z7_EXTERNAL_CODECS + so << endl << "Libs:" << endl; + for (i = 0; i < codecs->Libs.Size(); i++) + { + PrintLibIndex(so, (int)i); + const CCodecLib &lib = codecs->Libs[i]; + // if (lib.Version != 0) + so << ": " << (lib.Version >> 16) << "."; + PrintNumber(so, lib.Version & 0xffff, 2); + so << " : " << lib.Path << endl; + } + #endif + + so << endl << "Formats:" << endl; + + const char * const kArcFlags = "KSNFMGOPBELHXCc+a+m+r+"; + const char * const kArcTimeFlags = "wudn"; + const unsigned kNumArcFlags = (unsigned)strlen(kArcFlags); + const unsigned kNumArcTimeFlags = (unsigned)strlen(kArcTimeFlags); + + for (i = 0; i < codecs->Formats.Size(); i++) + { + const CArcInfoEx &arc = codecs->Formats[i]; + + #ifdef Z7_EXTERNAL_CODECS + PrintLibIndex(so, arc.LibIndex); + #else + so << " "; + #endif + + so << (char)(arc.UpdateEnabled ? 'C' : ' '); + + { + unsigned b; + for (b = 0; b < kNumArcFlags; b++) + so << (char)((arc.Flags & ((UInt32)1 << b)) != 0 ? kArcFlags[b] : '.'); + so << ' '; + } + + if (arc.TimeFlags != 0) + { + unsigned b; + for (b = 0; b < kNumArcTimeFlags; b++) + so << (char)((arc.TimeFlags & ((UInt32)1 << b)) != 0 ? kArcTimeFlags[b] : '.'); + so << arc.Get_DefaultTimePrec(); + so << ' '; + } + + so << ' '; + PrintString(so, arc.Name, 8); + so << ' '; + UString s; + + FOR_VECTOR (t, arc.Exts) + { + if (t != 0) + s.Add_Space(); + const CArcExtInfo &ext = arc.Exts[t]; + s += ext.Ext; + if (!ext.AddExt.IsEmpty()) + { + s += " ("; + s += ext.AddExt; + s.Add_Char(')'); + } + } + + PrintString(so, s, 13); + so << ' '; + + if (arc.SignatureOffset != 0) + so << "offset=" << arc.SignatureOffset << ' '; + + // so << "numSignatures = " << arc.Signatures.Size() << " "; + + FOR_VECTOR(si, arc.Signatures) + { + if (si != 0) + so << " || "; + + const CByteBuffer &sig = arc.Signatures[si]; + + for (size_t j = 0; j < sig.Size(); j++) + { + if (j != 0) + so << ' '; + const unsigned b = sig.ConstData()[j]; + if (b > 0x20 && b < 0x80) + { + so << (char)b; + } + else + { + so << GET_HEX_CHAR_UPPER(b >> 4); + so << GET_HEX_CHAR_UPPER(b & 15); + } + } + } + so << endl; + } + + so << endl << "Codecs:" << endl; // << "Lib ID Name" << endl; + + for (i = 0; i < g_NumCodecs; i++) + { + const CCodecInfo &cod = *g_Codecs[i]; + + PrintLibIndex(so, -1); + + if (cod.NumStreams == 1) + so << ' '; + else + so << cod.NumStreams; + + so << (char)(cod.CreateEncoder ? 'E' : ' '); + so << (char)(cod.CreateDecoder ? 'D' : ' '); + so << (char)(cod.IsFilter ? 'F' : ' '); + + so << ' '; + PrintHexId(so, cod.Id); + so << ' ' << cod.Name << endl; + } + + + #ifdef Z7_EXTERNAL_CODECS + + UInt32 numMethods; + if (_externalCodecs.GetCodecs->GetNumMethods(&numMethods) == S_OK) + for (UInt32 j = 0; j < numMethods; j++) + { + PrintLibIndex(so, codecs->GetCodec_LibIndex(j)); + + UInt32 numStreams = codecs->GetCodec_NumStreams(j); + if (numStreams == 1) + so << ' '; + else + so << numStreams; + + so << (char)(codecs->GetCodec_EncoderIsAssigned(j) ? 'E' : ' '); + so << (char)(codecs->GetCodec_DecoderIsAssigned(j) ? 'D' : ' '); + { + bool isFilter_Assigned; + const bool isFilter = codecs->GetCodec_IsFilter(j, isFilter_Assigned); + so << (char)(isFilter ? 'F' : isFilter_Assigned ? ' ' : '*'); + } + + + so << ' '; + UInt64 id; + HRESULT res = codecs->GetCodec_Id(j, id); + if (res != S_OK) + id = (UInt64)(Int64)-1; + PrintHexId(so, id); + so << ' ' << codecs->GetCodec_Name(j) << endl; + } + + #endif + + + so << endl << "Hashers:" << endl; // << " L Size ID Name" << endl; + + for (i = 0; i < g_NumHashers; i++) + { + const CHasherInfo &codec = *g_Hashers[i]; + PrintLibIndex(so, -1); + PrintUInt32(so, codec.DigestSize, 4); + so << ' '; + PrintHexId(so, codec.Id); + so << ' ' << codec.Name << endl; + } + + #ifdef Z7_EXTERNAL_CODECS + + numMethods = _externalCodecs.GetHashers->GetNumHashers(); + for (UInt32 j = 0; j < numMethods; j++) + { + PrintLibIndex(so, codecs->GetHasherLibIndex(j)); + PrintUInt32(so, codecs->GetHasherDigestSize(j), 4); + so << ' '; + PrintHexId(so, codecs->GetHasherId(j)); + so << ' ' << codecs->GetHasherName(j) << endl; + } + + #endif + + } + else if (options.Command.CommandType == NCommandType::kBenchmark) + { + CStdOutStream &so = (g_StdStream ? *g_StdStream : g_StdOut); + hresultMain = BenchCon(EXTERNAL_CODECS_VARS_L + options.Properties, options.NumIterations, (FILE *)so); + if (hresultMain == S_FALSE) + { + so << endl; + if (g_ErrStream) + *g_ErrStream << "\nDecoding ERROR\n"; + retCode = NExitCode::kFatalError; + hresultMain = S_OK; + } + } + else if (isExtractGroupCommand || options.Command.CommandType == NCommandType::kList) + { + UStringVector ArchivePathsSorted; + UStringVector ArchivePathsFullSorted; + + if (options.StdInMode) + { + ArchivePathsSorted.Add(options.ArcName_for_StdInMode); + ArchivePathsFullSorted.Add(options.ArcName_for_StdInMode); + } + else + { + CExtractScanConsole scan; + + scan.Init(options.EnableHeaders ? g_StdStream : NULL, + g_ErrStream, percentsStream, + options.DisablePercents); + scan.SetWindowWidth(consoleWidth); + + if (g_StdStream && options.EnableHeaders) + *g_StdStream << "Scanning the drive for archives:" << endl; + + CDirItemsStat st; + + scan.StartScanning(); + + hresultMain = EnumerateDirItemsAndSort( + options.arcCensor, + NWildcard::k_RelatPath, + UString(), // addPathPrefix + ArchivePathsSorted, + ArchivePathsFullSorted, + st, + &scan); + + scan.CloseScanning(); + + if (hresultMain == S_OK) + { + if (options.EnableHeaders) + scan.PrintStat(st); + } + else + { + /* + if (res != E_ABORT) + { + throw CSystemException(res); + // errorInfo.Message = "Scanning error"; + } + return res; + */ + } + } + + if (hresultMain == S_OK) { + if (isExtractGroupCommand) + { + CExtractCallbackConsole *ecs = new CExtractCallbackConsole; + CMyComPtr extractCallback = ecs; + + #ifndef Z7_NO_CRYPTO + ecs->PasswordIsDefined = options.PasswordEnabled; + ecs->Password = options.Password; + #endif + + ecs->Init(g_StdStream, g_ErrStream, percentsStream, options.DisablePercents); + ecs->MultiArcMode = (ArchivePathsSorted.Size() > 1); + + ecs->LogLevel = options.LogLevel; + ecs->PercentsNameLevel = percentsNameLevel; + + if (percentsStream) + ecs->SetWindowWidth(consoleWidth); + + /* + COpenCallbackConsole openCallback; + openCallback.Init(g_StdStream, g_ErrStream); + + #ifndef Z7_NO_CRYPTO + openCallback.PasswordIsDefined = options.PasswordEnabled; + openCallback.Password = options.Password; + #endif + */ + + CExtractOptions eo; + (CExtractOptionsBase &)eo = options.ExtractOptions; + + eo.StdInMode = options.StdInMode; + eo.StdOutMode = options.StdOutMode; + eo.YesToAll = options.YesToAll; + eo.TestMode = options.Command.IsTestCommand(); + + #ifndef Z7_SFX + eo.Properties = options.Properties; + #endif + + UString errorMessage; + CDecompressStat stat; + CHashBundle hb; + IHashCalc *hashCalc = NULL; + + if (!options.HashMethods.IsEmpty()) + { + hashCalc = &hb; + ThrowException_if_Error(hb.SetMethods(EXTERNAL_CODECS_VARS_L options.HashMethods)); + // hb.Init(); + } + + hresultMain = Extract( + // EXTERNAL_CODECS_VARS_L + codecs, + types, + excludedFormats, + ArchivePathsSorted, + ArchivePathsFullSorted, + options.Censor.Pairs.Front().Head, + eo, + ecs, ecs, ecs, + hashCalc, errorMessage, stat); + + ecs->ClosePercents(); + + if (!errorMessage.IsEmpty()) + { + if (g_ErrStream) + *g_ErrStream << endl << "ERROR:" << endl << errorMessage << endl; + if (hresultMain == S_OK) + hresultMain = E_FAIL; + } + + CStdOutStream *so = g_StdStream; + + bool isError = false; + + if (so) + { + *so << endl; + + if (ecs->NumTryArcs > 1) + { + *so << "Archives: " << ecs->NumTryArcs << endl; + *so << "OK archives: " << ecs->NumOkArcs << endl; + } + } + + if (ecs->NumCantOpenArcs != 0) + { + isError = true; + if (so) + *so << "Can't open as archive: " << ecs->NumCantOpenArcs << endl; + } + + if (ecs->NumArcsWithError != 0) + { + isError = true; + if (so) + *so << "Archives with Errors: " << ecs->NumArcsWithError << endl; + } + + if (so) + { + if (ecs->NumArcsWithWarnings != 0) + *so << "Archives with Warnings: " << ecs->NumArcsWithWarnings << endl; + + if (ecs->NumOpenArcWarnings != 0) + { + *so << endl; + if (ecs->NumOpenArcWarnings != 0) + *so << "Warnings: " << ecs->NumOpenArcWarnings << endl; + } + } + + if (ecs->NumOpenArcErrors != 0) + { + isError = true; + if (so) + { + *so << endl; + if (ecs->NumOpenArcErrors != 0) + *so << "Open Errors: " << ecs->NumOpenArcErrors << endl; + } + } + + if (isError) + retCode = NExitCode::kFatalError; + + if (so) { + if (ecs->NumArcsWithError != 0 || ecs->NumFileErrors != 0) + { + // if (ecs->NumArchives > 1) + { + *so << endl; + if (ecs->NumFileErrors != 0) + *so << "Sub items Errors: " << ecs->NumFileErrors << endl; + } + } + else if (hresultMain == S_OK) + { + if (stat.NumFolders != 0) + *so << "Folders: " << stat.NumFolders << endl; + if (stat.NumFiles != 1 || stat.NumFolders != 0 || stat.NumAltStreams != 0) + *so << "Files: " << stat.NumFiles << endl; + if (stat.NumAltStreams != 0) + { + *so << "Alternate Streams: " << stat.NumAltStreams << endl; + *so << "Alternate Streams Size: " << stat.AltStreams_UnpackSize << endl; + } + + *so + << "Size: " << stat.UnpackSize << endl + << "Compressed: " << stat.PackSize << endl; + if (hashCalc) + { + *so << endl; + PrintHashStat(*so, hb); + } + } + } // if (so) + } + else // if_(!isExtractGroupCommand) + { + UInt64 numErrors = 0; + UInt64 numWarnings = 0; + + // options.ExtractNtOptions.StoreAltStreams = true, if -sns[-] is not definmed + + CListOptions lo; + lo.ExcludeDirItems = options.Censor.ExcludeDirItems; + lo.ExcludeFileItems = options.Censor.ExcludeFileItems; + lo.DisablePercents = options.DisablePercents; + + hresultMain = ListArchives( + lo, + codecs, + types, + excludedFormats, + options.StdInMode, + ArchivePathsSorted, + ArchivePathsFullSorted, + options.ExtractOptions.NtOptions.AltStreams.Val, + options.AltStreams.Val, // we don't want to show AltStreams by default + options.Censor.Pairs.Front().Head, + options.EnableHeaders, + options.TechMode, + #ifndef Z7_NO_CRYPTO + options.PasswordEnabled, + options.Password, + #endif + &options.Properties, + numErrors, numWarnings); + + if (options.EnableHeaders) + if (numWarnings > 0) + g_StdOut << endl << "Warnings: " << numWarnings << endl; + + if (numErrors > 0) + { + if (options.EnableHeaders) + g_StdOut << endl << "Errors: " << numErrors << endl; + retCode = NExitCode::kFatalError; + } + } // if_(isExtractGroupCommand) + } // if_(hresultMain == S_OK) + } + else if (options.Command.IsFromUpdateGroup()) + { + #ifdef Z7_EXTRACT_ONLY + throw "update commands are not implemented"; + #else + CUpdateOptions &uo = options.UpdateOptions; + if (uo.SfxMode && uo.SfxModule.IsEmpty()) + uo.SfxModule = kDefaultSfxModule; + + COpenCallbackConsole openCallback; + openCallback.Init(g_StdStream, g_ErrStream, percentsStream, options.DisablePercents); + + #ifndef Z7_NO_CRYPTO + bool passwordIsDefined = + (options.PasswordEnabled && !options.Password.IsEmpty()); + openCallback.PasswordIsDefined = passwordIsDefined; + openCallback.Password = options.Password; + #endif + + CUpdateCallbackConsole callback; + callback.LogLevel = options.LogLevel; + callback.PercentsNameLevel = percentsNameLevel; + + if (percentsStream) + callback.SetWindowWidth(consoleWidth); + + #ifndef Z7_NO_CRYPTO + callback.PasswordIsDefined = passwordIsDefined; + callback.AskPassword = (options.PasswordEnabled && options.Password.IsEmpty()); + callback.Password = options.Password; + #endif + + callback.StdOutMode = uo.StdOutMode; + callback.Init( + // NULL, + g_StdStream, g_ErrStream, percentsStream, options.DisablePercents); + + CUpdateErrorInfo errorInfo; + + /* + if (!uo.Init(codecs, types, options.ArchiveName)) + throw kUnsupportedUpdateArcType; + */ + hresultMain = UpdateArchive(codecs, + types, + options.ArchiveName, + options.Censor, + uo, + errorInfo, &openCallback, &callback, true); + + callback.ClosePercents2(); + + CStdOutStream *se = g_StdStream; + if (!se) + se = g_ErrStream; + + retCode = WarningsCheck(hresultMain, callback, errorInfo, + g_StdStream, se, + true // options.EnableHeaders + ); + #endif + } + else if (options.Command.CommandType == NCommandType::kHash) + { + const CHashOptions &uo = options.HashOptions; + + CHashCallbackConsole callback; + if (percentsStream) + callback.SetWindowWidth(consoleWidth); + + callback.Init(g_StdStream, g_ErrStream, percentsStream, options.DisablePercents); + callback.PrintHeaders = options.EnableHeaders; + callback.PrintFields = options.ListFields; + + AString errorInfoString; + hresultMain = HashCalc(EXTERNAL_CODECS_VARS_L + options.Censor, uo, + errorInfoString, &callback); + CUpdateErrorInfo errorInfo; + errorInfo.Message = errorInfoString; + CStdOutStream *se = g_StdStream; + if (!se) + se = g_ErrStream; + retCode = WarningsCheck(hresultMain, callback, errorInfo, g_StdStream, se, options.EnableHeaders); + } + else + ShowMessageAndThrowException(kUserErrorMessage, NExitCode::kUserError); + + if (options.ShowTime && g_StdStream) + PrintStat( + #ifndef _WIN32 + startTime + #endif + ); + + ThrowException_if_Error(hresultMain); + + return retCode; +} diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp new file mode 100644 index 000000000..15fd7ac4b --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.cpp @@ -0,0 +1,885 @@ +// UpdateCallbackConsole.cpp + +#include "StdAfx.h" + +#include "../../../Common/IntToString.h" + +#include "../../../Windows/ErrorMsg.h" +#include "../../../Windows/FileName.h" + +#ifndef Z7_ST +#include "../../../Windows/Synchronization.h" +#endif + +// #include "../Common/PropIDUtils.h" + +#include "ConsoleClose.h" +#include "UserInputUtils.h" +#include "UpdateCallbackConsole.h" + +using namespace NWindows; + +#ifndef Z7_ST +static NSynchronization::CCriticalSection g_CriticalSection; +#define MT_LOCK NSynchronization::CCriticalSectionLock lock(g_CriticalSection); +#else +#define MT_LOCK +#endif + +static const wchar_t * const kEmptyFileAlias = L"[Content]"; + +static const char * const kOpenArchiveMessage = "Open archive: "; +static const char * const kCreatingArchiveMessage = "Creating archive: "; +static const char * const kUpdatingArchiveMessage = "Updating archive: "; +static const char * const kScanningMessage = "Scanning the drive:"; + +static const char * const kError = "ERROR: "; +static const char * const kWarning = "WARNING: "; + +static HRESULT CheckBreak2() +{ + return NConsoleClose::TestBreakSignal() ? E_ABORT : S_OK; +} + +HRESULT Print_OpenArchive_Props(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink); +HRESULT Print_OpenArchive_Error(CStdOutStream &so, const CCodecs *codecs, const CArchiveLink &arcLink); + +void PrintErrorFlags(CStdOutStream &so, const char *s, UInt32 errorFlags); + +void Print_ErrorFormatIndex_Warning(CStdOutStream *_so, const CCodecs *codecs, const CArc &arc); + +HRESULT CUpdateCallbackConsole::OpenResult( + const CCodecs *codecs, const CArchiveLink &arcLink, + const wchar_t *name, HRESULT result) +{ + ClosePercents2(); + + FOR_VECTOR (level, arcLink.Arcs) + { + const CArc &arc = arcLink.Arcs[level]; + const CArcErrorInfo &er = arc.ErrorInfo; + + UInt32 errorFlags = er.GetErrorFlags(); + + if (errorFlags != 0 || !er.ErrorMessage.IsEmpty()) + { + if (_se) + { + *_se << endl; + if (level != 0) + *_se << arc.Path << endl; + } + + if (errorFlags != 0) + { + if (_se) + PrintErrorFlags(*_se, "ERRORS:", errorFlags); + } + + if (!er.ErrorMessage.IsEmpty()) + { + if (_se) + *_se << "ERRORS:" << endl << er.ErrorMessage << endl; + } + + if (_se) + { + *_se << endl; + _se->Flush(); + } + } + + UInt32 warningFlags = er.GetWarningFlags(); + + if (warningFlags != 0 || !er.WarningMessage.IsEmpty()) + { + if (_so) + { + *_so << endl; + if (level != 0) + *_so << arc.Path << endl; + } + + if (warningFlags != 0) + { + if (_so) + PrintErrorFlags(*_so, "WARNINGS:", warningFlags); + } + + if (!er.WarningMessage.IsEmpty()) + { + if (_so) + *_so << "WARNINGS:" << endl << er.WarningMessage << endl; + } + + if (_so) + { + *_so << endl; + if (NeedFlush) + _so->Flush(); + } + } + + + if (er.ErrorFormatIndex >= 0) + { + if (_so) + { + Print_ErrorFormatIndex_Warning(_so, codecs, arc); + if (NeedFlush) + _so->Flush(); + } + } + } + + if (result == S_OK) + { + if (_so) + { + RINOK(Print_OpenArchive_Props(*_so, codecs, arcLink)) + *_so << endl; + } + } + else + { + if (_so) + _so->Flush(); + if (_se) + { + *_se << kError; + _se->NormalizePrint_wstr_Path(name); + *_se << endl; + HRESULT res = Print_OpenArchive_Error(*_se, codecs, arcLink); + RINOK(res) + _se->Flush(); + } + } + + return S_OK; +} + +HRESULT CUpdateCallbackConsole::StartScanning() +{ + if (_so) + *_so << kScanningMessage << endl; + _percent.Command = "Scan "; + return S_OK; +} + +HRESULT CUpdateCallbackConsole::ScanProgress(const CDirItemsStat &st, const FString &path, bool /* isDir */) +{ + if (NeedPercents()) + { + _percent.Files = st.NumDirs + st.NumFiles + st.NumAltStreams; + _percent.Completed = st.GetTotalBytes(); + _percent.FileName = fs2us(path); + _percent.Print(); + } + + return CheckBreak(); +} + +void CCallbackConsoleBase::CommonError(const FString &path, DWORD systemError, bool isWarning) +{ + ClosePercents2(); + + if (_se) + { + if (_so) + _so->Flush(); + + *_se << endl << (isWarning ? kWarning : kError) + << NError::MyFormatMessage(systemError) + << endl; + _se->NormalizePrint_UString_Path(fs2us(path)); + *_se << endl << endl; + _se->Flush(); + } +} + +/* +void CCallbackConsoleBase::CommonError(const char *message) +{ + ClosePercents2(); + + if (_se) + { + if (_so) + _so->Flush(); + + *_se << endl << kError << message << endl; + _se->Flush(); + } +} +*/ + + +HRESULT CCallbackConsoleBase::ScanError_Base(const FString &path, DWORD systemError) +{ + MT_LOCK + + ScanErrors.AddError(path, systemError); + CommonError(path, systemError, true); + + return S_OK; +} + +HRESULT CCallbackConsoleBase::OpenFileError_Base(const FString &path, DWORD systemError) +{ + MT_LOCK + FailedFiles.AddError(path, systemError); + NumNonOpenFiles++; + /* + if (systemError == ERROR_SHARING_VIOLATION) + { + */ + CommonError(path, systemError, true); + return S_FALSE; + /* + } + return systemError; + */ +} + +HRESULT CCallbackConsoleBase::ReadingFileError_Base(const FString &path, DWORD systemError) +{ + MT_LOCK + CommonError(path, systemError, false); + return HRESULT_FROM_WIN32(systemError); +} + +HRESULT CUpdateCallbackConsole::ScanError(const FString &path, DWORD systemError) +{ + return ScanError_Base(path, systemError); +} + + +static void PrintPropPair(AString &s, const char *name, UInt64 val) +{ + char temp[32]; + ConvertUInt64ToString(val, temp); + s += name; + s += ": "; + s += temp; +} + +void PrintSize_bytes_Smart(AString &s, UInt64 val); +void Print_DirItemsStat(AString &s, const CDirItemsStat &st); +void Print_DirItemsStat2(AString &s, const CDirItemsStat2 &st); + +HRESULT CUpdateCallbackConsole::FinishScanning(const CDirItemsStat &st) +{ + if (NeedPercents()) + { + _percent.ClosePrint(true); + _percent.ClearCurState(); + } + + if (_so) + { + AString s; + Print_DirItemsStat(s, st); + *_so << s << endl << endl; + } + return S_OK; +} + +static const char * const k_StdOut_ArcName = "StdOut"; + +HRESULT CUpdateCallbackConsole::StartOpenArchive(const wchar_t *name) +{ + if (_so) + { + *_so << kOpenArchiveMessage; + if (name) + *_so << name; + else + *_so << k_StdOut_ArcName; + *_so << endl; + } + return S_OK; +} + +HRESULT CUpdateCallbackConsole::StartArchive(const wchar_t *name, bool updating) +{ + if (NeedPercents()) + _percent.ClosePrint(true); + + _percent.ClearCurState(); + NumNonOpenFiles = 0; + + if (_so) + { + *_so << (updating ? kUpdatingArchiveMessage : kCreatingArchiveMessage); + if (name) + _so->NormalizePrint_wstr_Path(name); + else + *_so << k_StdOut_ArcName; + *_so << endl << endl; + } + return S_OK; +} + +HRESULT CUpdateCallbackConsole::FinishArchive(const CFinishArchiveStat &st) +{ + ClosePercents2(); + + if (_so) + { + AString s; + // Print_UInt64_and_String(s, _percent.Files == 1 ? "file" : "files", _percent.Files); + PrintPropPair(s, "Files read from disk", _percent.Files - NumNonOpenFiles); + s.Add_LF(); + s += "Archive size: "; + PrintSize_bytes_Smart(s, st.OutArcFileSize); + s.Add_LF(); + if (st.IsMultiVolMode) + { + s += "Volumes: "; + s.Add_UInt32(st.NumVolumes); + s.Add_LF(); + } + *_so << endl; + *_so << s; + // *_so << endl; + } + + return S_OK; +} + +HRESULT CUpdateCallbackConsole::WriteSfx(const wchar_t *name, UInt64 size) +{ + if (_so) + { + *_so << "Write SFX: "; + *_so << name; + AString s (" : "); + PrintSize_bytes_Smart(s, size); + *_so << s << endl; + } + return S_OK; +} + + +HRESULT CUpdateCallbackConsole::DeletingAfterArchiving(const FString &path, bool /* isDir */) +{ + if (LogLevel > 0 && _so) + { + ClosePercents_for_so(); + + if (!DeleteMessageWasShown) + { + if (_so) + *_so << endl << ": Removing files after including to archive" << endl; + } + + { + { + _tempA = "Removing"; + _tempA.Add_Space(); + *_so << _tempA; + _tempU = fs2us(path); + _so->Normalize_UString_Path(_tempU); + _so->PrintUString(_tempU, _tempA); + *_so << endl; + if (NeedFlush) + _so->Flush(); + } + } + } + + if (!DeleteMessageWasShown) + { + if (NeedPercents()) + { + _percent.ClearCurState(); + } + DeleteMessageWasShown = true; + } + else + { + _percent.Files++; + } + + if (NeedPercents()) + { + // if (!FullLog) + { + _percent.Command = "Removing"; + _percent.FileName = fs2us(path); + } + _percent.Print(); + } + + return S_OK; +} + + +HRESULT CUpdateCallbackConsole::FinishDeletingAfterArchiving() +{ + ClosePercents2(); + if (_so && DeleteMessageWasShown) + *_so << endl; + return S_OK; +} + +HRESULT CUpdateCallbackConsole::CheckBreak() +{ + return CheckBreak2(); +} + +/* +HRESULT CUpdateCallbackConsole::Finalize() +{ + // MT_LOCK + return S_OK; +} +*/ + + +void static PrintToDoStat(CStdOutStream *_so, const CDirItemsStat2 &stat, const char *name) +{ + AString s; + Print_DirItemsStat2(s, stat); + *_so << name << ": " << s << endl; +} + +HRESULT CUpdateCallbackConsole::SetNumItems(const CArcToDoStat &stat) +{ + if (_so) + { + ClosePercents_for_so(); + if (!stat.DeleteData.IsEmpty()) + { + *_so << endl; + PrintToDoStat(_so, stat.DeleteData, "Delete data from archive"); + } + if (!stat.OldData.IsEmpty()) + PrintToDoStat(_so, stat.OldData, "Keep old data in archive"); + // if (!stat.NewData.IsEmpty()) + { + PrintToDoStat(_so, stat.NewData, "Add new data to archive"); + } + *_so << endl; + } + return S_OK; +} + +HRESULT CUpdateCallbackConsole::SetTotal(UInt64 size) +{ + MT_LOCK + if (NeedPercents()) + { + _percent.Total = size; + _percent.Print(); + } + return S_OK; +} + +HRESULT CUpdateCallbackConsole::SetCompleted(const UInt64 *completeValue) +{ + MT_LOCK + if (completeValue) + { + if (NeedPercents()) + { + _percent.Completed = *completeValue; + _percent.Print(); + } + } + return CheckBreak2(); +} + +HRESULT CUpdateCallbackConsole::SetRatioInfo(const UInt64 * /* inSize */, const UInt64 * /* outSize */) +{ + return CheckBreak2(); +} + +HRESULT CCallbackConsoleBase::PrintProgress(const wchar_t *name, bool isDir, const char *command, bool showInLog) +{ + MT_LOCK + + bool show2 = (showInLog && _so); + + if (show2) + { + ClosePercents_for_so(); + + _tempA = command; + if (name) + _tempA.Add_Space(); + *_so << _tempA; + + _tempU.Empty(); + if (name) + { + _tempU = name; + if (isDir) + NWindows::NFile::NName::NormalizeDirPathPrefix(_tempU); + _so->Normalize_UString_Path(_tempU); + } + _so->PrintUString(_tempU, _tempA); + *_so << endl; + if (NeedFlush) + _so->Flush(); + } + + if (NeedPercents()) + { + if (PercentsNameLevel >= 1) + { + _percent.FileName.Empty(); + _percent.Command.Empty(); + if (PercentsNameLevel > 1 || !show2) + { + _percent.Command = command; + if (name) + _percent.FileName = name; + } + } + _percent.Print(); + } + + return CheckBreak2(); +} + + +/* +void CCallbackConsoleBase::PrintInfoLine(const UString &s) +{ + if (LogLevel < 1000) + return; + + MT_LOCK + + const bool show2 = (_so != NULL); + + if (show2) + { + ClosePercents_for_so(); + _so->PrintUString(s, _tempA); + *_so << endl; + if (NeedFlush) + _so->Flush(); + } +} +*/ + +HRESULT CUpdateCallbackConsole::GetStream(const wchar_t *name, bool isDir, bool isAnti, UInt32 mode) +{ + if (StdOutMode) + return S_OK; + + if (!name || name[0] == 0) + name = kEmptyFileAlias; + + unsigned requiredLevel = 1; + + const char *s; + if (mode == NUpdateNotifyOp::kAdd || + mode == NUpdateNotifyOp::kUpdate) + { + if (isAnti) + s = "Anti"; + else if (mode == NUpdateNotifyOp::kAdd) + s = "+"; + else + s = "U"; + } + else + { + requiredLevel = 3; + if (mode == NUpdateNotifyOp::kAnalyze) + s = "A"; + else + s = "Reading"; + } + + return PrintProgress(name, isDir, s, LogLevel >= requiredLevel); +} + +HRESULT CUpdateCallbackConsole::OpenFileError(const FString &path, DWORD systemError) +{ + return OpenFileError_Base(path, systemError); +} + +HRESULT CUpdateCallbackConsole::ReadingFileError(const FString &path, DWORD systemError) +{ + return ReadingFileError_Base(path, systemError); +} + +HRESULT CUpdateCallbackConsole::SetOperationResult(Int32 /* opRes */) +{ + MT_LOCK + _percent.Files++; + /* + if (opRes != NArchive::NUpdate::NOperationResult::kOK) + { + if (opRes == NArchive::NUpdate::NOperationResult::kError_FileChanged) + { + CommonError("Input file changed"); + } + } + */ + return S_OK; +} + +void SetExtractErrorMessage(Int32 opRes, Int32 encrypted, AString &dest); + +HRESULT CUpdateCallbackConsole::ReportExtractResult(Int32 opRes, Int32 isEncrypted, const wchar_t *name) +{ + // if (StdOutMode) return S_OK; + + if (opRes != NArchive::NExtract::NOperationResult::kOK) + { + ClosePercents2(); + + if (_se) + { + if (_so) + _so->Flush(); + + AString s; + SetExtractErrorMessage(opRes, isEncrypted, s); + *_se << s << " : " << endl; + _se->NormalizePrint_wstr_Path(name); + *_se << endl << endl; + _se->Flush(); + } + return S_OK; + } + return S_OK; +} + + +HRESULT CUpdateCallbackConsole::ReportUpdateOperation(UInt32 op, const wchar_t *name, bool isDir) +{ + // if (StdOutMode) return S_OK; + + char temp[16]; + const char *s; + + unsigned requiredLevel = 1; + + switch (op) + { + case NUpdateNotifyOp::kAdd: s = "+"; break; + case NUpdateNotifyOp::kUpdate: s = "U"; break; + case NUpdateNotifyOp::kAnalyze: s = "A"; requiredLevel = 3; break; + case NUpdateNotifyOp::kReplicate: s = "="; requiredLevel = 3; break; + case NUpdateNotifyOp::kRepack: s = "R"; requiredLevel = 2; break; + case NUpdateNotifyOp::kSkip: s = "."; requiredLevel = 2; break; + case NUpdateNotifyOp::kDelete: s = "D"; requiredLevel = 3; break; + case NUpdateNotifyOp::kHeader: s = "Header creation"; requiredLevel = 100; break; + case NUpdateNotifyOp::kInFileChanged: s = "Size of input file was changed:"; requiredLevel = 10; break; + // case NUpdateNotifyOp::kOpFinished: s = "Finished"; requiredLevel = 100; break; + default: + { + temp[0] = 'o'; + temp[1] = 'p'; + ConvertUInt64ToString(op, temp + 2); + s = temp; + } + } + + return PrintProgress(name, isDir, s, LogLevel >= requiredLevel); +} + +/* +HRESULT CUpdateCallbackConsole::SetPassword(const UString & + #ifndef Z7_NO_CRYPTO + password + #endif + ) +{ + #ifndef Z7_NO_CRYPTO + PasswordIsDefined = true; + Password = password; + #endif + return S_OK; +} +*/ + +HRESULT CUpdateCallbackConsole::CryptoGetTextPassword2(Int32 *passwordIsDefined, BSTR *password) +{ + COM_TRY_BEGIN + + *password = NULL; + + #ifdef Z7_NO_CRYPTO + + *passwordIsDefined = false; + return S_OK; + + #else + + if (!PasswordIsDefined) + { + if (AskPassword) + { + RINOK(GetPassword_HRESULT(_so, Password)) + PasswordIsDefined = true; + } + } + *passwordIsDefined = BoolToInt(PasswordIsDefined); + return StringToBstr(Password, password); + + #endif + + COM_TRY_END +} + +HRESULT CUpdateCallbackConsole::CryptoGetTextPassword(BSTR *password) +{ + COM_TRY_BEGIN + + *password = NULL; + + #ifdef Z7_NO_CRYPTO + + return E_NOTIMPL; + + #else + + if (!PasswordIsDefined) + { + { + RINOK(GetPassword_HRESULT(_so, Password)) + PasswordIsDefined = true; + } + } + return StringToBstr(Password, password); + + #endif + COM_TRY_END +} + +HRESULT CUpdateCallbackConsole::ShowDeleteFile(const wchar_t *name, bool isDir) +{ + if (StdOutMode) + return S_OK; + + if (LogLevel > 7) + { + if (!name || name[0] == 0) + name = kEmptyFileAlias; + return PrintProgress(name, isDir, "D", true); + } + return S_OK; +} + +/* +void GetPropName(PROPID propID, const wchar_t *name, AString &nameA, UString &nameU); + +static void GetPropName(PROPID propID, UString &nameU) +{ + AString nameA; + GetPropName(propID, NULL, nameA, nameU); + // if (!nameA.IsEmpty()) + nameU = nameA; +} + + +static void AddPropNamePrefix(UString &s, PROPID propID) +{ + UString name; + GetPropName(propID, name); + s += name; + s += " = "; +} + +void CCallbackConsoleBase::PrintPropInfo(UString &s, PROPID propID, const PROPVARIANT *value) +{ + AddPropNamePrefix(s, propID); + { + UString dest; + const int level = 9; // we show up to ns precision level + ConvertPropertyToString2(dest, *value, propID, level); + s += dest; + } + PrintInfoLine(s); +} + +static void Add_IndexType_Index(UString &s, UInt32 indexType, UInt32 index) +{ + if (indexType == NArchive::NEventIndexType::kArcProp) + { + } + else + { + if (indexType == NArchive::NEventIndexType::kBlockIndex) + { + s += "#"; + } + else if (indexType == NArchive::NEventIndexType::kOutArcIndex) + { + } + else + { + s += "indexType_"; + s.Add_UInt32(indexType); + s.Add_Space(); + } + s.Add_UInt32(index); + } + s += ": "; +} + +HRESULT CUpdateCallbackConsole::ReportProp(UInt32 indexType, UInt32 index, PROPID propID, const PROPVARIANT *value) +{ + UString s; + Add_IndexType_Index(s, indexType, index); + PrintPropInfo(s, propID, value); + return S_OK; +} + +static inline char GetHex(Byte value) +{ + return (char)((value < 10) ? ('0' + value) : ('a' + (value - 10))); +} + +static void AddHexToString(UString &dest, const Byte *data, UInt32 size) +{ + for (UInt32 i = 0; i < size; i++) + { + Byte b = data[i]; + dest += GetHex((Byte)((b >> 4) & 0xF)); + dest += GetHex((Byte)(b & 0xF)); + } +} + +void HashHexToString(char *dest, const Byte *data, UInt32 size); + +HRESULT CUpdateCallbackConsole::ReportRawProp(UInt32 indexType, UInt32 index, + PROPID propID, const void *data, UInt32 dataSize, UInt32 propType) +{ + UString s; + propType = propType; + Add_IndexType_Index(s, indexType, index); + AddPropNamePrefix(s, propID); + if (propID == kpidChecksum) + { + char temp[k_HashCalc_DigestSize_Max + 8]; + HashHexToString(temp, (const Byte *)data, dataSize); + s += temp; + } + else + AddHexToString(s, (const Byte *)data, dataSize); + PrintInfoLine(s); + return S_OK; +} + +HRESULT CUpdateCallbackConsole::ReportFinished(UInt32 indexType, UInt32 index, Int32 opRes) +{ + UString s; + Add_IndexType_Index(s, indexType, index); + s += "finished"; + if (opRes != NArchive::NUpdate::NOperationResult::kOK) + { + s += ": "; + s.Add_UInt32(opRes); + } + PrintInfoLine(s); + return S_OK; +} +*/ diff --git a/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.h b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.h new file mode 100644 index 000000000..51234ad56 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/7zip/UI/Console/UpdateCallbackConsole.h @@ -0,0 +1,138 @@ +// UpdateCallbackConsole.h + +#ifndef ZIP7_INC_UPDATE_CALLBACK_CONSOLE_H +#define ZIP7_INC_UPDATE_CALLBACK_CONSOLE_H + +#include "../../../Common/StdOutStream.h" + +#include "../Common/Update.h" + +#include "PercentPrinter.h" + +struct CErrorPathCodes +{ + FStringVector Paths; + CRecordVector Codes; + + void AddError(const FString &path, DWORD systemError) + { + Paths.Add(path); + Codes.Add(systemError); + } + void Clear() + { + Paths.Clear(); + Codes.Clear(); + } +}; + + +class CCallbackConsoleBase +{ +protected: + CPercentPrinter _percent; + + CStdOutStream *_so; + CStdOutStream *_se; + + void CommonError(const FString &path, DWORD systemError, bool isWarning); + // void CommonError(const char *message); + + HRESULT ScanError_Base(const FString &path, DWORD systemError); + HRESULT OpenFileError_Base(const FString &name, DWORD systemError); + HRESULT ReadingFileError_Base(const FString &name, DWORD systemError); + +public: + bool NeedPercents() const { return _percent._so != NULL; } + + bool StdOutMode; + + bool NeedFlush; + unsigned PercentsNameLevel; + unsigned LogLevel; + + AString _tempA; + UString _tempU; + + CCallbackConsoleBase(): + StdOutMode(false), + NeedFlush(false), + PercentsNameLevel(1), + LogLevel(0), + NumNonOpenFiles(0) + {} + + void SetWindowWidth(unsigned width) { _percent.MaxLen = width - 1; } + + void Init( + CStdOutStream *outStream, + CStdOutStream *errorStream, + CStdOutStream *percentStream, + bool disablePercents) + { + FailedFiles.Clear(); + + _so = outStream; + _se = errorStream; + _percent._so = percentStream; + _percent.DisablePrint = disablePercents; + } + + void ClosePercents2() + { + if (NeedPercents()) + _percent.ClosePrint(true); + } + + void ClosePercents_for_so() + { + if (NeedPercents() && _so == _percent._so) + _percent.ClosePrint(false); + } + + CErrorPathCodes FailedFiles; + CErrorPathCodes ScanErrors; + UInt64 NumNonOpenFiles; + + HRESULT PrintProgress(const wchar_t *name, bool isDir, const char *command, bool showInLog); + + // void PrintInfoLine(const UString &s); + // void PrintPropInfo(UString &s, PROPID propID, const PROPVARIANT *value); +}; + + +class CUpdateCallbackConsole Z7_final: + public IUpdateCallbackUI2, + public CCallbackConsoleBase +{ + // void PrintPropPair(const char *name, const wchar_t *val); + Z7_IFACE_IMP(IUpdateCallbackUI) + Z7_IFACE_IMP(IDirItemsCallback) + Z7_IFACE_IMP(IUpdateCallbackUI2) +public: + bool DeleteMessageWasShown; + + #ifndef Z7_NO_CRYPTO + bool PasswordIsDefined; + bool AskPassword; + UString Password; + #endif + + CUpdateCallbackConsole(): + DeleteMessageWasShown(false) + #ifndef Z7_NO_CRYPTO + , PasswordIsDefined(false) + , AskPassword(false) + #endif + {} + + /* + void Init(CStdOutStream *outStream) + { + CCallbackConsoleBase::Init(outStream); + } + */ + // ~CUpdateCallbackConsole() { if (NeedPercents()) _percent.ClosePrint(); } +}; + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/Windows/FileSystem.cpp b/NanaZip.Core/SevenZip/CPP/Windows/FileSystem.cpp new file mode 100644 index 000000000..38c691d86 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/Windows/FileSystem.cpp @@ -0,0 +1,187 @@ +// Windows/FileSystem.cpp + +#include "StdAfx.h" + +#ifndef UNDER_CE + +#ifndef _UNICODE +#include "../Common/StringConvert.h" +#endif + +#include "FileSystem.h" +#include "Defs.h" + +#ifndef _UNICODE +extern bool g_IsNT; +#endif + +namespace NWindows { +namespace NFile { +namespace NSystem { + +#ifdef _WIN32 + +bool MyGetVolumeInformation( + CFSTR rootPath, + UString &volumeName, + LPDWORD volumeSerialNumber, + LPDWORD maximumComponentLength, + LPDWORD fileSystemFlags, + UString &fileSystemName) +{ + BOOL res; + #ifndef _UNICODE + if (!g_IsNT) + { + TCHAR v[MAX_PATH + 2]; v[0] = 0; + TCHAR f[MAX_PATH + 2]; f[0] = 0; + res = GetVolumeInformation(fs2fas(rootPath), + v, MAX_PATH, + volumeSerialNumber, maximumComponentLength, fileSystemFlags, + f, MAX_PATH); + volumeName = MultiByteToUnicodeString(v); + fileSystemName = MultiByteToUnicodeString(f); + } + else + #endif + { + WCHAR v[MAX_PATH + 2]; v[0] = 0; + WCHAR f[MAX_PATH + 2]; f[0] = 0; + res = GetVolumeInformationW(fs2us(rootPath), + v, MAX_PATH, + volumeSerialNumber, maximumComponentLength, fileSystemFlags, + f, MAX_PATH); + volumeName = v; + fileSystemName = f; + } + return BOOLToBool(res); +} + +UINT MyGetDriveType(CFSTR pathName) +{ + #ifndef _UNICODE + if (!g_IsNT) + { + return GetDriveType(fs2fas(pathName)); + } + else + #endif + { + return GetDriveTypeW(fs2us(pathName)); + } +} + +#if !defined(Z7_WIN32_WINNT_MIN) || Z7_WIN32_WINNT_MIN < 0x0400 +// GetDiskFreeSpaceEx requires Windows95-OSR2, NT4 +#define Z7_USE_DYN_GetDiskFreeSpaceEx +#endif + +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx +typedef BOOL (WINAPI * Func_GetDiskFreeSpaceExA)( + LPCSTR lpDirectoryName, // directory name + PULARGE_INTEGER lpFreeBytesAvailable, // bytes available to caller + PULARGE_INTEGER lpTotalNumberOfBytes, // bytes on disk + PULARGE_INTEGER lpTotalNumberOfFreeBytes // free bytes on disk +); + +typedef BOOL (WINAPI * Func_GetDiskFreeSpaceExW)( + LPCWSTR lpDirectoryName, // directory name + PULARGE_INTEGER lpFreeBytesAvailable, // bytes available to caller + PULARGE_INTEGER lpTotalNumberOfBytes, // bytes on disk + PULARGE_INTEGER lpTotalNumberOfFreeBytes // free bytes on disk +); +#endif + +bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize) +{ + DWORD numSectorsPerCluster, bytesPerSector, numFreeClusters, numClusters; + bool sizeIsDetected = false; + #ifndef _UNICODE + if (!g_IsNT) + { +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx + const + Func_GetDiskFreeSpaceExA f = Z7_GET_PROC_ADDRESS( + Func_GetDiskFreeSpaceExA, GetModuleHandle(TEXT("kernel32.dll")), + "GetDiskFreeSpaceExA"); + if (f) +#endif + { + ULARGE_INTEGER freeBytesToCaller2, totalSize2, freeSize2; + sizeIsDetected = BOOLToBool( +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx + f +#else + GetDiskFreeSpaceExA +#endif + (fs2fas(rootPath), &freeBytesToCaller2, &totalSize2, &freeSize2)); + totalSize = totalSize2.QuadPart; + freeSize = freeSize2.QuadPart; + } + if (!::GetDiskFreeSpace(fs2fas(rootPath), &numSectorsPerCluster, &bytesPerSector, &numFreeClusters, &numClusters)) + return false; + } + else + #endif + { +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx + const + Func_GetDiskFreeSpaceExW f = Z7_GET_PROC_ADDRESS( + Func_GetDiskFreeSpaceExW, GetModuleHandle(TEXT("kernel32.dll")), + "GetDiskFreeSpaceExW"); + if (f) +#endif + { + ULARGE_INTEGER freeBytesToCaller2, totalSize2, freeSize2; + sizeIsDetected = BOOLToBool( +#ifdef Z7_USE_DYN_GetDiskFreeSpaceEx + f +#else + GetDiskFreeSpaceExW +#endif + (fs2us(rootPath), &freeBytesToCaller2, &totalSize2, &freeSize2)); + totalSize = totalSize2.QuadPart; + freeSize = freeSize2.QuadPart; + } + if (!::GetDiskFreeSpaceW(fs2us(rootPath), &numSectorsPerCluster, &bytesPerSector, &numFreeClusters, &numClusters)) + return false; + } + clusterSize = (UInt64)bytesPerSector * (UInt64)numSectorsPerCluster; + if (!sizeIsDetected) + { + totalSize = clusterSize * (UInt64)numClusters; + freeSize = clusterSize * (UInt64)numFreeClusters; + } + return true; +} + +#endif + +/* +bool Is_File_LimitedBy_4GB(CFSTR _path, bool &isFsDetected) +{ + isFsDetected = false; + FString path (_path); + path.DeleteFrom(NName::GetRootPrefixSize(path)); + // GetVolumeInformation supports super paths. + // NName::If_IsSuperPath_RemoveSuperPrefix(path); + if (!path.IsEmpty()) + { + DWORD volumeSerialNumber, maximumComponentLength, fileSystemFlags; + UString volName, fileSystemName; + if (MyGetVolumeInformation(path, volName, + &volumeSerialNumber, &maximumComponentLength, &fileSystemFlags, + fileSystemName)) + { + isFsDetected = true; + if (fileSystemName.IsPrefixedBy_Ascii_NoCase("fat")) + return true; + } + } + return false; +} +*/ + +}}} + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/Windows/FileSystem.h b/NanaZip.Core/SevenZip/CPP/Windows/FileSystem.h new file mode 100644 index 000000000..5516756df --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/Windows/FileSystem.h @@ -0,0 +1,31 @@ +// Windows/FileSystem.h + +#ifndef ZIP7_INC_WINDOWS_FILE_SYSTEM_H +#define ZIP7_INC_WINDOWS_FILE_SYSTEM_H + +#include "../Common/MyString.h" +#include "../Common/MyTypes.h" + +namespace NWindows { +namespace NFile { +namespace NSystem { + +#ifdef _WIN32 + +bool MyGetVolumeInformation( + CFSTR rootPath , + UString &volumeName, + LPDWORD volumeSerialNumber, + LPDWORD maximumComponentLength, + LPDWORD fileSystemFlags, + UString &fileSystemName); + +UINT MyGetDriveType(CFSTR pathName); + +bool MyGetDiskFreeSpace(CFSTR rootPath, UInt64 &clusterSize, UInt64 &totalSize, UInt64 &freeSize); + +#endif + +}}} + +#endif diff --git a/NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.cpp b/NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.cpp new file mode 100644 index 000000000..2deeda109 --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.cpp @@ -0,0 +1,1170 @@ +// Windows/SystemInfo.cpp + +#include "StdAfx.h" + +#include "../../C/CpuArch.h" + +#include "../Common/IntToString.h" +#include "../Common/StringConvert.h" + +#ifdef _WIN32 + +#include "Registry.h" + +#else + +#include +#include +#ifdef __APPLE__ +#include + +#elif !defined(_AIX) + +#if defined(__GLIBC__) && (__GLIBC__ * 100 + __GLIBC_MINOR__ >= 216) + #define Z7_GETAUXV_AVAILABLE +#else +// #pragma message("=== is not NEW GLIBC === ") + #if defined __has_include + #if __has_include () +// #pragma message("=== sys/auxv.h is avail=== ") + #define Z7_GETAUXV_AVAILABLE + #endif + #endif +#endif + +#ifdef Z7_GETAUXV_AVAILABLE +// #if defined __has_include +// #if __has_include () +#include +#define USE_HWCAP +// #endif +// #endif + +// #undef AT_HWCAP // to debug +// #undef AT_HWCAP2 // to debug + +/* the following patch for some debian systems. + Is it OK to define AT_HWCAP and AT_HWCAP2 here with these constant numbers? */ +/* +#if defined(__FreeBSD_kernel__) && defined(__GLIBC__) + #ifndef AT_HWCAP + #define AT_HWCAP 16 + #endif + #ifndef AT_HWCAP2 + #define AT_HWCAP2 26 + #endif +#endif +*/ + +#ifdef USE_HWCAP + +#if defined(__FreeBSD__) + +// #if (__FreeBSD__ >= 13) // (FreeBSD 12.01 is required for elf_aux_info() ???) +static unsigned long MY_getauxval(int aux) +{ + unsigned long val; + if (elf_aux_info(aux, &val, sizeof(val))) + return 0; + return val; +} + +#else // ! __FreeBSD__ + +#ifdef MY_CPU_ARM_OR_ARM64 + #if defined __has_include + #if __has_include () +#include + #endif + #endif +#endif + +#if defined(AT_HWCAP) || defined(AT_HWCAP2) +#define MY_getauxval getauxval +#endif + +#endif // ! __FreeBSD__ +#endif // USE_HWCAP +#endif // Z7_GETAUXV_AVAILABLE + +#endif // !defined(_AIX) + +#ifdef __linux__ +#include "../Windows/FileIO.h" +#endif + +#endif // WIN32 + +#include "SystemInfo.h" +#include "System.h" + +using namespace NWindows; + +#ifdef __linux__ + +static bool ReadFile_to_Buffer(CFSTR fileName, CByteBuffer &buf) +{ + NWindows::NFile::NIO::CInFile file; + if (!file.Open(fileName)) + return false; + /* + UInt64 size; + if (!file.GetLength(size)) + { + // GetLength() doesn't work "/proc/cpuinfo" + return false; + } + if (size >= ((UInt32)1 << 29)) + return false; + */ + size_t size = 0; + size_t addSize = (size_t)1 << 12; + for (;;) + { + // printf("\nsize = %d\n", (unsigned)size); + buf.ChangeSize_KeepData(size + addSize, size); + size_t processed; + if (!file.ReadFull(buf.NonConstData() + size, addSize, processed)) + return false; + if (processed == 0) + { + buf.ChangeSize_KeepData(size, size); + return true; + } + size += processed; + addSize *= 2; + } +} + +#endif + + +#if defined(_WIN32) || defined(AT_HWCAP) || defined(AT_HWCAP2) +static void PrintHex(AString &s, UInt64 v) +{ + char temp[32]; + ConvertUInt64ToHex(v, temp); + s += temp; +} +#endif + +#ifdef MY_CPU_X86_OR_AMD64 + +Z7_NO_INLINE +static void PrintCpuChars(AString &s, UInt32 v) +{ + for (unsigned j = 0; j < 4; j++) + { + const Byte b = (Byte)(v & 0xFF); + v >>= 8; + if (b == 0) + break; + if (b >= 0x20 && b <= 0x7f) + s.Add_Char((char)b); + else + { + s.Add_Char('['); + char temp[16]; + ConvertUInt32ToHex(b, temp); + s += temp; + s.Add_Char(']'); + } + } +} + + +static void x86cpuid_to_String(AString &s) +{ + s.Empty(); + + UInt32 a[4]; + // cpuid was called already. So we don't check for cpuid availability here + z7_x86_cpuid(a, 0x80000000); + + if (a[0] >= 0x80000004) // if (maxFunc2 >= hi+4) the full name is available + { + for (unsigned i = 0; i < 3; i++) + { + z7_x86_cpuid(a, (UInt32)(0x80000002 + i)); + for (unsigned j = 0; j < 4; j++) + PrintCpuChars(s, a[j]); + } + } + + s.Trim(); + + if (s.IsEmpty()) + { + z7_x86_cpuid(a, 0); + for (unsigned i = 1; i < 4; i++) + { + const unsigned j = (i ^ (i >> 1)); + PrintCpuChars(s, a[j]); + } + s.Trim(); + } +} + +/* +static void x86cpuid_all_to_String(AString &s) +{ + Cx86cpuid p; + if (!x86cpuid_CheckAndRead(&p)) + return; + s += "x86cpuid maxFunc = "; + s.Add_UInt32(p.maxFunc); + for (unsigned j = 0; j <= p.maxFunc; j++) + { + s.Add_LF(); + // s.Add_UInt32(j); // align + { + char temp[32]; + ConvertUInt32ToString(j, temp); + unsigned len = (unsigned)strlen(temp); + while (len < 8) + { + len++; + s.Add_Space(); + } + s += temp; + } + + s += ":"; + UInt32 d[4] = { 0 }; + MyCPUID(j, &d[0], &d[1], &d[2], &d[3]); + for (unsigned i = 0; i < 4; i++) + { + char temp[32]; + ConvertUInt32ToHex8Digits(d[i], temp); + s.Add_Space(); + s += temp; + } + } +} +*/ + +#endif + + + +#ifdef _WIN32 + +static const char * const k_PROCESSOR_ARCHITECTURE[] = +{ + "x86" // "INTEL" + , "MIPS" + , "ALPHA" + , "PPC" + , "SHX" + , "ARM" + , "IA64" + , "ALPHA64" + , "MSIL" + , "x64" // "AMD64" + , "IA32_ON_WIN64" + , "NEUTRAL" + , "ARM64" + , "ARM32_ON_WIN64" +}; + +#define Z7_WIN_PROCESSOR_ARCHITECTURE_INTEL 0 +#define Z7_WIN_PROCESSOR_ARCHITECTURE_AMD64 9 + + +#define Z7_WIN_PROCESSOR_INTEL_PENTIUM 586 +#define Z7_WIN_PROCESSOR_AMD_X8664 8664 + +/* +static const CUInt32PCharPair k_PROCESSOR[] = +{ + { 2200, "IA64" }, + { 8664, "x64" } +}; + +#define PROCESSOR_INTEL_386 386 +#define PROCESSOR_INTEL_486 486 +#define PROCESSOR_INTEL_PENTIUM 586 +#define PROCESSOR_INTEL_860 860 +#define PROCESSOR_INTEL_IA64 2200 +#define PROCESSOR_AMD_X8664 8664 +#define PROCESSOR_MIPS_R2000 2000 +#define PROCESSOR_MIPS_R3000 3000 +#define PROCESSOR_MIPS_R4000 4000 +#define PROCESSOR_ALPHA_21064 21064 +#define PROCESSOR_PPC_601 601 +#define PROCESSOR_PPC_603 603 +#define PROCESSOR_PPC_604 604 +#define PROCESSOR_PPC_620 620 +#define PROCESSOR_HITACHI_SH3 10003 +#define PROCESSOR_HITACHI_SH3E 10004 +#define PROCESSOR_HITACHI_SH4 10005 +#define PROCESSOR_MOTOROLA_821 821 +#define PROCESSOR_SHx_SH3 103 +#define PROCESSOR_SHx_SH4 104 +#define PROCESSOR_STRONGARM 2577 // 0xA11 +#define PROCESSOR_ARM720 1824 // 0x720 +#define PROCESSOR_ARM820 2080 // 0x820 +#define PROCESSOR_ARM920 2336 // 0x920 +#define PROCESSOR_ARM_7TDMI 70001 +#define PROCESSOR_OPTIL 18767 // 0x494f +*/ + + +/* +static const char * const k_PF[] = +{ + "FP_ERRATA" + , "FP_EMU" + , "CMPXCHG" + , "MMX" + , "PPC_MOVEMEM_64BIT" + , "ALPHA_BYTE" + , "SSE" + , "3DNOW" + , "RDTSC" + , "PAE" + , "SSE2" + , "SSE_DAZ" + , "NX" + , "SSE3" + , "CMPXCHG16B" + , "CMP8XCHG16" + , "CHANNELS" + , "XSAVE" + , "ARM_VFP_32" + , "ARM_NEON" + , "L2AT" + , "VIRT_FIRMWARE" + , "RDWRFSGSBASE" + , "FASTFAIL" + , "ARM_DIVIDE" + , "ARM_64BIT_LOADSTORE_ATOMIC" + , "ARM_EXTERNAL_CACHE" + , "ARM_FMAC" + , "RDRAND" + , "ARM_V8" + , "ARM_V8_CRYPTO" + , "ARM_V8_CRC32" + , "RDTSCP" + , "RDPID" + , "ARM_V81_ATOMIC" + , "MONITORX" +}; +*/ + +#endif + + +static void PrintPage(AString &s, UInt64 v) +{ + const char *t = "B"; + if ((v & 0x3ff) == 0) + { + v >>= 10; + t = "KB"; + } + s.Add_UInt64(v); + s += t; +} + +#ifdef _WIN32 + +static AString TypeToString2(const char * const table[], unsigned num, UInt32 value) +{ + char sz[16]; + const char *p = NULL; + if (value < num) + p = table[value]; + if (!p) + { + ConvertUInt32ToString(value, sz); + p = sz; + } + return (AString)p; +} + +// #if defined(Z7_LARGE_PAGES) || defined(_WIN32) +// #ifdef _WIN32 +void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v) +{ + char c = 0; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'K'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'M'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'G'; + if ((v & 0x3FF) == 0) { v >>= 10; c = 'T'; + }}}} + else + { + // s += "0x"; + PrintHex(s, v); + return; + } + s.Add_UInt64(v); + if (c) + s.Add_Char(c); + s.Add_Char('B'); +} +// #endif +// #endif + +static void SysInfo_To_String(AString &s, const SYSTEM_INFO &si) +{ + s += TypeToString2(k_PROCESSOR_ARCHITECTURE, Z7_ARRAY_SIZE(k_PROCESSOR_ARCHITECTURE), si.wProcessorArchitecture); + + if (!( (si.wProcessorArchitecture == Z7_WIN_PROCESSOR_ARCHITECTURE_INTEL && si.dwProcessorType == Z7_WIN_PROCESSOR_INTEL_PENTIUM) + || (si.wProcessorArchitecture == Z7_WIN_PROCESSOR_ARCHITECTURE_AMD64 && si.dwProcessorType == Z7_WIN_PROCESSOR_AMD_X8664))) + { + s.Add_Space(); + // s += TypePairToString(k_PROCESSOR, Z7_ARRAY_SIZE(k_PROCESSOR), si.dwProcessorType); + s.Add_UInt32(si.dwProcessorType); + } + s.Add_Space(); + PrintHex(s, si.wProcessorLevel); + s.Add_Dot(); + PrintHex(s, si.wProcessorRevision); + if ((UInt64)si.dwActiveProcessorMask + 1 != ((UInt64)1 << si.dwNumberOfProcessors)) + if ((UInt64)si.dwActiveProcessorMask + 1 != 0 || si.dwNumberOfProcessors != sizeof(UInt64) * 8) + { + s += " act:"; + PrintHex(s, si.dwActiveProcessorMask); + } + s += " threads:"; + s.Add_UInt32(si.dwNumberOfProcessors); + if (si.dwPageSize != 1 << 12) + { + s += " page:"; + PrintPage(s, si.dwPageSize); + } + if (si.dwAllocationGranularity != 1 << 16) + { + s += " gran:"; + PrintPage(s, si.dwAllocationGranularity); + } + s.Add_Space(); + + const DWORD_PTR minAdd = (DWORD_PTR)si.lpMinimumApplicationAddress; + UInt64 maxSize = (UInt64)(DWORD_PTR)si.lpMaximumApplicationAddress + 1; + const UInt32 kReserveSize = ((UInt32)1 << 16); + if (minAdd != kReserveSize) + { + PrintSize_KMGT_Or_Hex(s, minAdd); + s.Add_Minus(); + } + else + { + if ((maxSize & (kReserveSize - 1)) == 0) + maxSize += kReserveSize; + } + PrintSize_KMGT_Or_Hex(s, maxSize); +} + +#ifndef _WIN64 +EXTERN_C_BEGIN +typedef VOID (WINAPI *Func_GetNativeSystemInfo)(LPSYSTEM_INFO lpSystemInfo); +EXTERN_C_END +#endif + +#endif + +#ifdef __APPLE__ +#ifndef MY_CPU_X86_OR_AMD64 +static void Add_sysctlbyname_to_String(const char *name, AString &s) +{ + size_t bufSize = 256; + char buf[256]; + if (z7_sysctlbyname_Get(name, &buf, &bufSize) == 0) + s += buf; +} +#endif +#endif + +void GetSysInfo(AString &s1, AString &s2); +void GetSysInfo(AString &s1, AString &s2) +{ + s1.Empty(); + s2.Empty(); + + #ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + { + SysInfo_To_String(s1, si); + // s += " : "; + } + + #if !defined(_WIN64) && !defined(UNDER_CE) + const + Func_GetNativeSystemInfo fn = Z7_GET_PROC_ADDRESS( + Func_GetNativeSystemInfo, GetModuleHandleA("kernel32.dll"), + "GetNativeSystemInfo"); + if (fn) + { + SYSTEM_INFO si2; + fn(&si2); + // if (memcmp(&si, &si2, sizeof(si)) != 0) + { + // s += " - "; + SysInfo_To_String(s2, si2); + } + } + #endif + #endif +} + + +static void AddBracedString(AString &dest, AString &src) +{ + if (!src.IsEmpty()) + { + dest.Add_Space_if_NotEmpty(); + dest.Add_Char('('); + dest += src; + dest.Add_Char(')'); + } +} + +struct CCpuName +{ + AString CpuName; + AString Revision; + AString Microcode; + AString LargePages; + + void Fill(); + + void Get_Revision_Microcode_LargePages(AString &s) + { + s.Empty(); + AddBracedString(s, Revision); + AddBracedString(s, Microcode); + s.Add_OptSpaced(LargePages); + } +}; + +void CCpuName::Fill() +{ + CpuName.Empty(); + Revision.Empty(); + Microcode.Empty(); + LargePages.Empty(); + + AString &s = CpuName; + + #ifdef MY_CPU_X86_OR_AMD64 + { + #if !defined(MY_CPU_AMD64) + if (z7_x86_cpuid_GetMaxFunc()) + #endif + { + x86cpuid_to_String(s); + { + UInt32 a[4]; + z7_x86_cpuid(a, 1); + char temp[16]; + ConvertUInt32ToHex(a[0], temp); + Revision += temp; + } + } + } + #elif defined(__APPLE__) + { + Add_sysctlbyname_to_String("machdep.cpu.brand_string", s); + } + #elif defined(MY_CPU_E2K) && defined(Z7_MCST_LCC_VERSION) && (Z7_MCST_LCC_VERSION >= 12323) + { + s += "mcst "; + s += __builtin_cpu_name(); + s.Add_Space(); + s += __builtin_cpu_arch(); + } + #endif + + +#ifdef _WIN32 + { + NRegistry::CKey key; + if (key.Open(HKEY_LOCAL_MACHINE, TEXT("HARDWARE\\DESCRIPTION\\System\\CentralProcessor\\0"), KEY_READ) == ERROR_SUCCESS) + { + // s.Empty(); // for debug + { + CSysString name; + if (s.IsEmpty()) + if (key.QueryValue(TEXT("ProcessorNameString"), name) == ERROR_SUCCESS) + { + s += GetAnsiString(name); + } + if (key.QueryValue(TEXT("Identifier"), name) == ERROR_SUCCESS) + { + if (!Revision.IsEmpty()) + Revision += " : "; + Revision += GetAnsiString(name); + } + } + LONG res[2]; + CByteBuffer bufs[2]; + { + for (unsigned i = 0; i < 2; i++) + { + UInt32 size = 0; + res[i] = key.QueryValue(i == 0 ? + TEXT("Previous Update Revision") : + TEXT("Update Revision"), + bufs[i], size); + if (res[i] == ERROR_SUCCESS) + if (size != bufs[i].Size()) + res[i] = ERROR_SUCCESS + 1; + } + } + if (res[0] == ERROR_SUCCESS || res[1] == ERROR_SUCCESS) + { + for (unsigned i = 0; i < 2; i++) + { + if (i == 1) + Microcode += "->"; + if (res[i] != ERROR_SUCCESS) + continue; + const CByteBuffer &buf = bufs[i]; + if (buf.Size() == 8) + { + const UInt32 high = GetUi32(buf); + if (high != 0) + { + PrintHex(Microcode, high); + Microcode.Add_Dot(); + } + PrintHex(Microcode, GetUi32(buf + 4)); + } + } + } + } + } +#endif + + if (s.IsEmpty()) + { + #ifdef MY_CPU_NAME + s += MY_CPU_NAME; + #endif + } + + #ifdef __APPLE__ + { + AString s2; + UInt32 v = 0; + if (z7_sysctlbyname_Get_UInt32("machdep.cpu.core_count", &v) == 0) + { + s2.Add_UInt32(v); + s2.Add_Char('C'); + } + if (z7_sysctlbyname_Get_UInt32("machdep.cpu.thread_count", &v) == 0) + { + s2.Add_UInt32(v); + s2.Add_Char('T'); + } + if (!s2.IsEmpty()) + { + s.Add_Space_if_NotEmpty(); + s += s2; + } + } + #endif + + #ifdef Z7_LARGE_PAGES + Add_LargePages_String(LargePages); + #endif +} + +void AddCpuFeatures(AString &s); +void AddCpuFeatures(AString &s) +{ + #ifdef _WIN32 + // const unsigned kNumFeatures_Extra = 32; // we check also for unknown features + // const unsigned kNumFeatures = Z7_ARRAY_SIZE(k_PF) + kNumFeatures_Extra; + const unsigned kNumFeatures = 64; + UInt64 flags = 0; + for (unsigned i = 0; i < kNumFeatures; i++) + { + if (IsProcessorFeaturePresent((DWORD)i)) + { + flags += (UInt64)1 << i; + // s.Add_Space_if_NotEmpty(); + // s += TypeToString2(k_PF, Z7_ARRAY_SIZE(k_PF), i); + } + } + s.Add_OptSpaced("f:"); + PrintHex(s, flags); + + #elif defined(__APPLE__) + { + UInt32 v = 0; + if (z7_sysctlbyname_Get_UInt32("hw.pagesize", &v) == 0) + { + s.Add_OptSpaced("PageSize:"); + PrintPage(s, v); + } + } + + #else + + const long v = sysconf(_SC_PAGESIZE); + if (v != -1) + { + s.Add_OptSpaced("PageSize:"); + PrintPage(s, (unsigned long)v); + } + + #if !defined(_AIX) + + #ifdef __linux__ + + CByteBuffer buf; + if (ReadFile_to_Buffer("/sys/kernel/mm/transparent_hugepage/enabled", buf)) + // if (ReadFile_to_Buffer("/proc/cpuinfo", buf)) + { + s.Add_OptSpaced("THP:"); + AString s2; + s2.SetFrom_CalcLen((const char *)(const void *)(const Byte *)buf, (unsigned)buf.Size()); + const int pos = s2.Find('['); + if (pos >= 0) + { + const int pos2 = s2.Find(']', (unsigned)pos + 1); + if (pos2 >= 0) + { + s2.DeleteFrom((unsigned)pos2); + s2.DeleteFrontal((unsigned)pos + 1); + } + } + s += s2; + } + // else throw CSystemException(MY_SRes_HRESULT_FROM_WRes(errno)); + + #endif + + + #ifdef AT_HWCAP + s.Add_OptSpaced("hwcap:"); + { + unsigned long h = MY_getauxval(AT_HWCAP); + PrintHex(s, h); + #ifdef MY_CPU_ARM64 + if (h & HWCAP_CRC32) s += ":CRC32"; + if (h & HWCAP_SHA1) s += ":SHA1"; + if (h & HWCAP_SHA2) s += ":SHA2"; + if (h & HWCAP_AES) s += ":AES"; + if (h & HWCAP_ASIMD) s += ":ASIMD"; + #elif defined(MY_CPU_ARM) + if (h & HWCAP_NEON) s += ":NEON"; + #endif + } + #endif // AT_HWCAP + + #ifdef AT_HWCAP2 + { + unsigned long h = MY_getauxval(AT_HWCAP2); + #ifndef MY_CPU_ARM + if (h != 0) + #endif + { + s += " hwcap2:"; + PrintHex(s, h); + #ifdef MY_CPU_ARM + if (h & HWCAP2_CRC32) s += ":CRC32"; + if (h & HWCAP2_SHA1) s += ":SHA1"; + if (h & HWCAP2_SHA2) s += ":SHA2"; + if (h & HWCAP2_AES) s += ":AES"; + #endif + } + } + #endif // AT_HWCAP2 + #endif // _AIX + #endif // _WIN32 +} + + +#ifdef _WIN32 +#ifndef UNDER_CE + +Z7_DIAGNOSTIC_IGNORE_CAST_FUNCTION + +EXTERN_C_BEGIN +typedef void (WINAPI * Func_RtlGetVersion) (OSVERSIONINFOEXW *); +EXTERN_C_END + +static BOOL My_RtlGetVersion(OSVERSIONINFOEXW *vi) +{ + const HMODULE ntdll = ::GetModuleHandleW(L"ntdll.dll"); + if (!ntdll) + return FALSE; + const + Func_RtlGetVersion func = Z7_GET_PROC_ADDRESS( + Func_RtlGetVersion, ntdll, + "RtlGetVersion"); + if (!func) + return FALSE; + func(vi); + return TRUE; +} + +#endif +#endif + + +void GetOsInfoText(AString &sRes) +{ + sRes.Empty(); + AString s; + + #ifdef _WIN32 + #ifndef UNDER_CE + // OSVERSIONINFO vi; + OSVERSIONINFOEXW vi; + vi.dwOSVersionInfoSize = sizeof(vi); + // if (::GetVersionEx(&vi)) + if (My_RtlGetVersion(&vi)) + { + s += "Windows"; + if (vi.dwPlatformId != VER_PLATFORM_WIN32_NT) + s.Add_UInt32(vi.dwPlatformId); + s.Add_Space(); s.Add_UInt32(vi.dwMajorVersion); + s.Add_Dot(); s.Add_UInt32(vi.dwMinorVersion); + s.Add_Space(); s.Add_UInt32(vi.dwBuildNumber); + + if (vi.wServicePackMajor != 0 || vi.wServicePackMinor != 0) + { + s += " SP:"; s.Add_UInt32(vi.wServicePackMajor); + s.Add_Dot(); s.Add_UInt32(vi.wServicePackMinor); + } + // s += " Suite:"; PrintHex(s, vi.wSuiteMask); + // s += " Type:"; s.Add_UInt32(vi.wProductType); + // s.Add_Space(); s += GetOemString(vi.szCSDVersion); + } + /* + { + s += " OEMCP:"; s.Add_UInt32(GetOEMCP()); + s += " ACP:"; s.Add_UInt32(GetACP()); + } + */ + #endif + #else // _WIN32 + + if (!s.IsEmpty()) + s.Add_LF(); + struct utsname un; + if (uname(&un) == 0) + { + s += un.sysname; + // s += " : "; s += un.nodename; // we don't want to show name of computer + s += " : "; s += un.release; + s += " : "; s += un.version; + s += " : "; s += un.machine; + + #ifdef __APPLE__ + // Add_sysctlbyname_to_String("kern.version", s); + // it's same as "utsname.version" + #endif + } + #endif // _WIN32 + + sRes += s; + #ifdef MY_CPU_X86_OR_AMD64 + { + AString s2; + GetVirtCpuid(s2); + if (!s2.IsEmpty()) + { + sRes += " : "; + sRes += s2; + } + } + #endif +} + + + +void GetSystemInfoText(AString &sRes) +{ + GetOsInfoText(sRes); + sRes.Add_LF(); + + { + AString s, s1, s2; + GetSysInfo(s1, s2); + if (!s1.IsEmpty() || !s2.IsEmpty()) + { + s = s1; + if (s1 != s2 && !s2.IsEmpty()) + { + s += " - "; + s += s2; + } + } + { + AddCpuFeatures(s); + if (!s.IsEmpty()) + { + sRes += s; + sRes.Add_LF(); + } + } + } + { + AString s; + GetCpuName_MultiLine(s); + if (!s.IsEmpty()) + { + sRes += s; + sRes.Add_LF(); + } + } + /* + #ifdef MY_CPU_X86_OR_AMD64 + { + AString s; + x86cpuid_all_to_String(s); + if (!s.IsEmpty()) + { + printCallback->Print(s); + printCallback->NewLine(); + } + } + #endif + */ +} + + +void GetCpuName_MultiLine(AString &s); +void GetCpuName_MultiLine(AString &s) +{ + CCpuName cpuName; + cpuName.Fill(); + s = cpuName.CpuName; + AString s2; + cpuName.Get_Revision_Microcode_LargePages(s2); + if (!s2.IsEmpty()) + { + s.Add_LF(); + s += s2; + } +} + + +#ifdef MY_CPU_X86_OR_AMD64 + +void GetVirtCpuid(AString &s) +{ + const UInt32 kHv = 0x40000000; + + Z7_IF_X86_CPUID_SUPPORTED + { + UInt32 a[4]; + z7_x86_cpuid(a, kHv); + + if (a[0] < kHv || a[0] >= kHv + (1 << 16)) + return; + { + { + for (unsigned j = 1; j < 4; j++) + PrintCpuChars(s, a[j]); + } + } + if (a[0] >= kHv + 1) + { + UInt32 d[4]; + z7_x86_cpuid(d, kHv + 1); + s += " : "; + PrintCpuChars(s, d[0]); + if (a[0] >= kHv + 2) + { + z7_x86_cpuid(d, kHv + 2); + s += " : "; + s.Add_UInt32(d[1] >> 16); + s.Add_Dot(); s.Add_UInt32(d[1] & 0xffff); + s.Add_Dot(); s.Add_UInt32(d[0]); + s.Add_Dot(); s.Add_UInt32(d[2]); + s.Add_Dot(); s.Add_UInt32(d[3] >> 24); + s.Add_Dot(); s.Add_UInt32(d[3] & 0xffffff); + } + /* + if (a[0] >= kHv + 5) + { + z7_x86_cpuid(d, kHv + 5); + s += " : "; + s.Add_UInt32(d[0]); + s += "p"; + s.Add_UInt32(d[1]); + s += "t"; + } + */ + } + } +} + +#endif + + +void GetCompiler(AString &s) +{ + #ifdef __clang__ + s += " CLANG "; + s.Add_UInt32(__clang_major__); + s.Add_Dot(); + s.Add_UInt32(__clang_minor__); + s.Add_Dot(); + s.Add_UInt32(__clang_patchlevel__); + #endif + + #ifdef __xlC__ + s += " XLC "; + s.Add_UInt32(__xlC__ >> 8); + s.Add_Dot(); + s.Add_UInt32(__xlC__ & 0xFF); + #ifdef __xlC_ver__ + s.Add_Dot(); + s.Add_UInt32(__xlC_ver__ >> 8); + s.Add_Dot(); + s.Add_UInt32(__xlC_ver__ & 0xFF); + #endif + #endif + + // #define __LCC__ 126 + // #define __LCC_MINOR__ 20 + // #define __MCST__ 1 + #ifdef __MCST__ + s += " MCST"; + #endif + #ifdef __LCC__ + s += " LCC "; + s.Add_UInt32(__LCC__ / 100); + s.Add_Dot(); + s.Add_UInt32(__LCC__ % 100 / 10); + s.Add_UInt32(__LCC__ % 10); + #ifdef __LCC_MINOR__ + s.Add_Dot(); + s.Add_UInt32(__LCC_MINOR__ / 10); + s.Add_UInt32(__LCC_MINOR__ % 10); + #endif + #endif + + // #define __EDG_VERSION__ 602 + #ifdef __EDG_VERSION__ + s += " EDG "; + s.Add_UInt32(__EDG_VERSION__ / 100); + s.Add_Dot(); + s.Add_UInt32(__EDG_VERSION__ % 100 / 10); + s.Add_UInt32(__EDG_VERSION__ % 10); + #endif + + #ifdef __VERSION__ + s.Add_Space(); + s += "ver:"; + s += __VERSION__; + #endif + + #ifdef __GNUC__ + s += " GCC "; + s.Add_UInt32(__GNUC__); + s.Add_Dot(); + s.Add_UInt32(__GNUC_MINOR__); + s.Add_Dot(); + s.Add_UInt32(__GNUC_PATCHLEVEL__); + #endif + + + #ifdef _MSC_VER + s += " MSC "; + s.Add_UInt32(_MSC_VER); + #ifdef _MSC_FULL_VER + s.Add_Dot(); + s.Add_UInt32(_MSC_FULL_VER); + #endif + + #endif + + #if defined(__AVX512F__) + #if defined(__AVX512VL__) + #define MY_CPU_COMPILE_ISA "AVX512VL" + #else + #define MY_CPU_COMPILE_ISA "AVX512F" + #endif + #elif defined(__AVX2__) + #define MY_CPU_COMPILE_ISA "AVX2" + #elif defined(__AVX__) + #define MY_CPU_COMPILE_ISA "AVX" + #elif defined(__SSE2__) + #define MY_CPU_COMPILE_ISA "SSE2" + #elif defined(_M_IX86_FP) && (_M_IX86_FP >= 2) + #define MY_CPU_COMPILE_ISA "SSE2" + #elif defined(__SSE__) + #define MY_CPU_COMPILE_ISA "SSE" + #elif defined(_M_IX86_FP) && (_M_IX86_FP >= 1) + #define MY_CPU_COMPILE_ISA "SSE" + #elif defined(__i686__) + #define MY_CPU_COMPILE_ISA "i686" + #elif defined(__i586__) + #define MY_CPU_COMPILE_ISA "i586" + #elif defined(__i486__) + #define MY_CPU_COMPILE_ISA "i486" + #elif defined(__i386__) + #define MY_CPU_COMPILE_ISA "i386" + #elif defined(_M_IX86_FP) + #define MY_CPU_COMPILE_ISA "IA32" + #endif + + AString s2; + + #ifdef MY_CPU_COMPILE_ISA + s2.Add_OptSpaced(MY_CPU_COMPILE_ISA); + #endif + +#ifndef MY_CPU_ARM64 + #ifdef __ARM_FP + s2.Add_OptSpaced("FP"); + #endif + #ifdef __ARM_NEON + s2.Add_OptSpaced("NEON"); + #endif + #ifdef __NEON__ + s2.Add_OptSpaced("__NEON__"); + #endif + #ifdef __ARM_FEATURE_SIMD32 + s2.Add_OptSpaced("SIMD32"); + #endif +#endif + + #ifdef __ARM_FEATURE_CRYPTO + s2.Add_OptSpaced("CRYPTO"); + #endif + + #ifdef __ARM_FEATURE_SHA2 + s2.Add_OptSpaced("SHA2"); + #endif + + #ifdef __ARM_FEATURE_AES + s2.Add_OptSpaced("AES"); + #endif + + #ifdef __ARM_FEATURE_CRC32 + s2.Add_OptSpaced("CRC32"); + #endif + + #ifdef __ARM_FEATURE_UNALIGNED + s2.Add_OptSpaced("UNALIGNED"); + #endif + + + #ifdef MY_CPU_BE + s2.Add_OptSpaced("BE"); + #endif + + #if defined(MY_CPU_LE_UNALIGN) \ + && !defined(MY_CPU_X86_OR_AMD64) \ + && !defined(MY_CPU_ARM64) + s2.Add_OptSpaced("LE-unaligned"); + #endif + + if (!s2.IsEmpty()) + { + s.Add_OptSpaced(": "); + s += s2; + } +} diff --git a/NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.h b/NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.h new file mode 100644 index 000000000..0443fd9ae --- /dev/null +++ b/NanaZip.Core/SevenZip/CPP/Windows/SystemInfo.h @@ -0,0 +1,19 @@ +// Windows/SystemInfo.h + +#ifndef ZIP7_INC_WINDOWS_SYSTEM_INFO_H +#define ZIP7_INC_WINDOWS_SYSTEM_INFO_H + +#include "../Common/MyString.h" + + +void GetCpuName_MultiLine(AString &s); + +void GetOsInfoText(AString &sRes); +void GetSystemInfoText(AString &s); +void PrintSize_KMGT_Or_Hex(AString &s, UInt64 v); +void Add_LargePages_String(AString &s); + +void GetCompiler(AString &s); +void GetVirtCpuid(AString &s); + +#endif diff --git a/NanaZip.sln b/NanaZip.sln index 5c2656c6d..bfb5ea270 100644 --- a/NanaZip.sln +++ b/NanaZip.sln @@ -47,6 +47,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NanaZip.Codecs", "NanaZip.C EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NanaZip.Frieren", "NanaZip.Frieren\NanaZip.Frieren.vcxproj", "{4E78AF72-D7EF-4769-B64B-95E7EDF05FA2}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NanaZip.Core.Console", "NanaZip.Core\NanaZip.Core.Console.vcxproj", "{86E818B3-D657-4E03-9336-48EE242A79D1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|ARM64 = Debug|ARM64 @@ -239,6 +241,18 @@ Global {4E78AF72-D7EF-4769-B64B-95E7EDF05FA2}.Release|x64.Build.0 = Release|x64 {4E78AF72-D7EF-4769-B64B-95E7EDF05FA2}.Release|x86.ActiveCfg = Release|Win32 {4E78AF72-D7EF-4769-B64B-95E7EDF05FA2}.Release|x86.Build.0 = Release|Win32 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Debug|ARM64.Build.0 = Debug|ARM64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Debug|x64.ActiveCfg = Debug|x64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Debug|x64.Build.0 = Debug|x64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Debug|x86.ActiveCfg = Debug|Win32 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Debug|x86.Build.0 = Debug|Win32 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Release|ARM64.ActiveCfg = Release|ARM64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Release|ARM64.Build.0 = Release|ARM64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Release|x64.ActiveCfg = Release|x64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Release|x64.Build.0 = Release|x64 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Release|x86.ActiveCfg = Release|Win32 + {86E818B3-D657-4E03-9336-48EE242A79D1}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -262,6 +276,7 @@ Global {6666CFC3-1986-469A-9266-0FD8C2674DF6} = {FB9122A4-25AA-405E-9EE1-886274614847} {89B81A5B-FF0D-4193-9CB1-738692775DD2} = {94A1E11C-B722-4BAE-9B12-1495F5EF3CC9} {4E78AF72-D7EF-4769-B64B-95E7EDF05FA2} = {94A1E11C-B722-4BAE-9B12-1495F5EF3CC9} + {86E818B3-D657-4E03-9336-48EE242A79D1} = {94A1E11C-B722-4BAE-9B12-1495F5EF3CC9} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {DE2C16C4-5306-4103-9C2A-749DC32B5CA6}