diff --git a/hifi_qt.py b/hifi_qt.py index d03987a6bf7..2df6ac7fa62 100644 --- a/hifi_qt.py +++ b/hifi_qt.py @@ -134,8 +134,10 @@ def __init__(self, args): if 'Windows' == system: self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_WIN_URLS').split(";") + self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_WIN_SHA512') elif 'Darwin' == system: self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_MAC_URLS').split(";") + self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_MAC_SHA512') elif 'Linux' == system: import distro cpu_architecture = platform.machine() @@ -149,6 +151,7 @@ def __init__(self, args): if distro.id() == 'ubuntu' or distro.id() == 'linuxmint': if (distro.id() == 'ubuntu' and u_major == 18) or distro.id() == 'linuxmint' and u_major == 19: self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_LINUX_URLS').split(";") + self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_LINUX_SHA512') elif (distro.id() == 'ubuntu' and u_major > 18) or (distro.id() == 'linuxmint' and u_major > 19): self.__no_qt_package_error() else: @@ -166,6 +169,7 @@ def __init__(self, args): if u_major == 18: self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_UBUNTU_AARCH64_URLS').split(";") + self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_UBUNTU_AARCH64_SHA512') elif u_major > 19: self.__no_qt_package_error() else: @@ -179,6 +183,7 @@ def __init__(self, args): if u_major == 10: self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_DEBIAN_AARCH64_URLS').split(";") + self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_DEBIAN_AARCH64_SHA512') elif u_major > 10: self.__no_qt_package_error() else: @@ -213,7 +218,7 @@ def writeConfig(self): def installQt(self): if not os.path.isdir(self.fullPath): print("Fetching Qt from {} to {}".format(self.qtUrl, self.path)) - hifi_utils.downloadAndExtract(self.qtUrl, self.path) + hifi_utils.downloadAndExtract(self.qtUrl, self.path, self.qtSha512) else: print ('Qt has already been downloaded') diff --git a/hifi_utils.py b/hifi_utils.py index d59087cc4a8..c1ef61112ce 100644 --- a/hifi_utils.py +++ b/hifi_utils.py @@ -80,7 +80,8 @@ def executeSubprocess(processArgs, folder=None, env=None): os.chdir(restoreDir) -def hashFile(file, hasher = hashlib.sha512()): +def hashFile(file, hasherType = hashlib.sha512): + hasher = hasherType() with open(file, "rb") as f: for chunk in iter(lambda: f.read(4096), b""): hasher.update(chunk) @@ -99,7 +100,7 @@ def hashFolder(folder): filenames = recursiveFileList(folder) return hashFiles(filenames) -def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3): +def downloadFile(urls, hash=None, hasher=hashlib.sha512, retries=3): for url in urls: for i in range(retries): tempFileName = None @@ -125,6 +126,7 @@ def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3): # Verify the hash if hash is not None and hash != downloadHash: print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url)) + print("File stats: ", os.stat(tempFileName)) os.remove(tempFileName) continue return tempFileName @@ -132,7 +134,7 @@ def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3): raise RuntimeError("Failed to download file from any of {}".format(urls)) -def downloadAndExtract(urls, destPath, hash=None, hasher=hashlib.sha512()): +def downloadAndExtract(urls, destPath, hash=None, hasher=hashlib.sha512): tempFileName = downloadFile(urls, hash, hasher) try: with zipfile.ZipFile(tempFileName) as zip: diff --git a/hifi_vcpkg.py b/hifi_vcpkg.py index 141e54f342a..df14f057992 100644 --- a/hifi_vcpkg.py +++ b/hifi_vcpkg.py @@ -94,27 +94,33 @@ def __init__(self, args): self.exe = os.path.join(self.path, 'vcpkg.exe') self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.bat'), '-disableMetrics' ] self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_WIN_CLIENT_URLS').split(';') + self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_WIN_CLIENT_SHA512') self.hostTriplet = 'x64-windows' if usePrebuilt: self.prebuiltArchive = self.readVar('EXTERNAL_VCPKG_WIN_URLS').split(';') + self.prebuiltArchiveSha512 = self.readVar('EXTERNAL_VCPKG_WIN_SHA512') elif 'Darwin' == system: self.exe = os.path.join(self.path, 'vcpkg') self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '--allowAppleClang', '-disableMetrics' ] self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_MAC_CLIENT_URLS').split(';') + self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_MAC_CLIENT_SHA512') self.hostTriplet = 'x64-osx' # Potential fix for a vcpkg build issue on OSX (see https://github.com/microsoft/vcpkg/issues/9029) self.bootstrapEnv['CXXFLAGS'] = '-D_CTERMID_H_' if usePrebuilt: self.prebuiltArchive = self.readVar('EXTERNAL_VCPKG_MAC_URLS').split(';') + self.prebuiltArchiveSha512 = self.readVar('EXTERNAL_VCPKG_MAC_SHA512') elif 'Linux' == system and 'aarch64' == machine: self.exe = os.path.join(self.path, 'vcpkg') self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ] self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_LINUX_AARCH64_URLS').split(';') + self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_LINUX_AARCH64_SHA512') self.hostTriplet = 'arm64-linux' else: self.exe = os.path.join(self.path, 'vcpkg') self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ] self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_LINUX_CLIENT_URLS').split(';') + self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_LINUX_CLIENT_SHA512') self.hostTriplet = 'x64-linux' if self.args.android: @@ -202,7 +208,7 @@ def bootstrap(self): hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv) else: print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path)) - hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path) + hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgSha512) print("Replacing port files") portsPath = os.path.join(self.path, 'ports') @@ -237,7 +243,7 @@ def setupDependencies(self, qt=None): if self.prebuiltArchive: if not os.path.isfile(self.prebuildTagFile): print('Extracting ' + self.prebuiltArchive + ' to ' + self.path) - hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path) + hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path, self.prebuiltArchiveSha512) self.writePrebuildTag() return @@ -283,9 +289,8 @@ def setupAndroidDependencies(self): dest = os.path.join(self.path, 'installed') url = self.readVar('EXTERNAL_VCPKG_ANDROID_URLS').split(';') # FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash - #hash = self.readVar(('EXTERNAL_VCPKG_ANDROID_SHA512') - #hifi_utils.downloadAndExtract(url, dest, hash) - hifi_utils.downloadAndExtract(url, dest) + hash = self.readVar('EXTERNAL_VCPKG_ANDROID_SHA512') + hifi_utils.downloadAndExtract(url, dest, hash) print("Installing additional android archives") androidPackages = hifi_android.getPlatformPackages() diff --git a/prebuild.py b/prebuild.py index 263f73d94a8..cfd9886fa91 100644 --- a/prebuild.py +++ b/prebuild.py @@ -100,6 +100,7 @@ def main(): args = parse_args() nsis_urls = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_URLS').split(';') + nsis_sha512 = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_SHA512') if args.ci_build: logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO) @@ -111,7 +112,7 @@ def main(): if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github": logger.info("Downloading NSIS") with timer('NSIS'): - hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)") + hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)", nsis_sha512) qtInstallPath = None # If not android, install our Qt build