Skip to content

Commit

Permalink
Merge pull request #1593 from namark/hash-checks-in-python-build-scripts
Browse files Browse the repository at this point in the history 
Added hash checks for downloaded items in python build scripts.
  • Loading branch information
@digisomni
digisomni authored Feb 26, 2022
2 parents 5dbf534 + 056dfb9 commit 7c3b9d1
Show file tree
Hide file tree
Showing 4 changed files with 23 additions and 10 deletions.
7 changes: 6 additions & 1 deletion hifi_qt.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,8 +134,10 @@ def __init__(self, args):

if 'Windows' == system:
self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_WIN_URLS').split(";")
self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_WIN_SHA512')
elif 'Darwin' == system:
self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_MAC_URLS').split(";")
self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_MAC_SHA512')
elif 'Linux' == system:
import distro
cpu_architecture = platform.machine()
Expand All @@ -149,6 +151,7 @@ def __init__(self, args):
if distro.id() == 'ubuntu' or distro.id() == 'linuxmint':
if (distro.id() == 'ubuntu' and u_major == 18) or distro.id() == 'linuxmint' and u_major == 19:
self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_LINUX_URLS').split(";")
self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_LINUX_SHA512')
elif (distro.id() == 'ubuntu' and u_major > 18) or (distro.id() == 'linuxmint' and u_major > 19):
self.__no_qt_package_error()
else:
Expand All @@ -166,6 +169,7 @@ def __init__(self, args):

if u_major == 18:
self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_UBUNTU_AARCH64_URLS').split(";")
self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_UBUNTU_AARCH64_SHA512')
elif u_major > 19:
self.__no_qt_package_error()
else:
Expand All @@ -179,6 +183,7 @@ def __init__(self, args):

if u_major == 10:
self.qtUrl = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_DEBIAN_AARCH64_URLS').split(";")
self.qtSha512 = hifi_utils.readEnviromentVariableFromFile(self.args.build_root, 'EXTERNAL_QT_DEBIAN_AARCH64_SHA512')
elif u_major > 10:
self.__no_qt_package_error()
else:
Expand Down Expand Up @@ -213,7 +218,7 @@ def writeConfig(self):
def installQt(self):
if not os.path.isdir(self.fullPath):
print("Fetching Qt from {} to {}".format(self.qtUrl, self.path))
hifi_utils.downloadAndExtract(self.qtUrl, self.path)
hifi_utils.downloadAndExtract(self.qtUrl, self.path, self.qtSha512)
else:
print ('Qt has already been downloaded')

Expand Down
8 changes: 5 additions & 3 deletions hifi_utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,8 @@ def executeSubprocess(processArgs, folder=None, env=None):
os.chdir(restoreDir)


def hashFile(file, hasher = hashlib.sha512()):
def hashFile(file, hasherType = hashlib.sha512):
hasher = hasherType()
with open(file, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hasher.update(chunk)
Expand All @@ -99,7 +100,7 @@ def hashFolder(folder):
filenames = recursiveFileList(folder)
return hashFiles(filenames)

def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3):
def downloadFile(urls, hash=None, hasher=hashlib.sha512, retries=3):
for url in urls:
for i in range(retries):
tempFileName = None
Expand All @@ -125,14 +126,15 @@ def downloadFile(urls, hash=None, hasher=hashlib.sha512(), retries=3):
# Verify the hash
if hash is not None and hash != downloadHash:
print("Try {}: Downloaded file {} hash {} does not match expected hash {} for url {}".format(i + 1, tempFileName, downloadHash, hash, url))
print("File stats: ", os.stat(tempFileName))
os.remove(tempFileName)
continue
return tempFileName

raise RuntimeError("Failed to download file from any of {}".format(urls))


def downloadAndExtract(urls, destPath, hash=None, hasher=hashlib.sha512()):
def downloadAndExtract(urls, destPath, hash=None, hasher=hashlib.sha512):
tempFileName = downloadFile(urls, hash, hasher)
try:
with zipfile.ZipFile(tempFileName) as zip:
Expand Down
15 changes: 10 additions & 5 deletions hifi_vcpkg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,27 +94,33 @@ def __init__(self, args):
self.exe = os.path.join(self.path, 'vcpkg.exe')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.bat'), '-disableMetrics' ]
self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_WIN_CLIENT_URLS').split(';')
self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_WIN_CLIENT_SHA512')
self.hostTriplet = 'x64-windows'
if usePrebuilt:
self.prebuiltArchive = self.readVar('EXTERNAL_VCPKG_WIN_URLS').split(';')
self.prebuiltArchiveSha512 = self.readVar('EXTERNAL_VCPKG_WIN_SHA512')
elif 'Darwin' == system:
self.exe = os.path.join(self.path, 'vcpkg')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '--allowAppleClang', '-disableMetrics' ]
self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_MAC_CLIENT_URLS').split(';')
self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_MAC_CLIENT_SHA512')
self.hostTriplet = 'x64-osx'
# Potential fix for a vcpkg build issue on OSX (see https://github.com/microsoft/vcpkg/issues/9029)
self.bootstrapEnv['CXXFLAGS'] = '-D_CTERMID_H_'
if usePrebuilt:
self.prebuiltArchive = self.readVar('EXTERNAL_VCPKG_MAC_URLS').split(';')
self.prebuiltArchiveSha512 = self.readVar('EXTERNAL_VCPKG_MAC_SHA512')
elif 'Linux' == system and 'aarch64' == machine:
self.exe = os.path.join(self.path, 'vcpkg')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_LINUX_AARCH64_URLS').split(';')
self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_LINUX_AARCH64_SHA512')
self.hostTriplet = 'arm64-linux'
else:
self.exe = os.path.join(self.path, 'vcpkg')
self.bootstrapCmds = [ os.path.join(self.path, 'bootstrap-vcpkg.sh'), '-disableMetrics' ]
self.vcpkgUrl = self.readVar('EXTERNAL_VCPKG_LINUX_CLIENT_URLS').split(';')
self.vcpkgSha512 = self.readVar('EXTERNAL_VCPKG_LINUX_CLIENT_SHA512')
self.hostTriplet = 'x64-linux'

if self.args.android:
Expand Down Expand Up @@ -202,7 +208,7 @@ def bootstrap(self):
hifi_utils.executeSubprocess(self.bootstrapCmds, folder=self.path, env=self.bootstrapEnv)
else:
print("Fetching vcpkg from {} to {}".format(self.vcpkgUrl, self.path))
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path)
hifi_utils.downloadAndExtract(self.vcpkgUrl, self.path, self.vcpkgSha512)

print("Replacing port files")
portsPath = os.path.join(self.path, 'ports')
Expand Down Expand Up @@ -237,7 +243,7 @@ def setupDependencies(self, qt=None):
if self.prebuiltArchive:
if not os.path.isfile(self.prebuildTagFile):
print('Extracting ' + self.prebuiltArchive + ' to ' + self.path)
hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path)
hifi_utils.downloadAndExtract(self.prebuiltArchive, self.path, self.prebuiltArchiveSha512)
self.writePrebuildTag()
return

Expand Down Expand Up @@ -283,9 +289,8 @@ def setupAndroidDependencies(self):
dest = os.path.join(self.path, 'installed')
url = self.readVar('EXTERNAL_VCPKG_ANDROID_URLS').split(';')
# FIXME I don't know why the hash check frequently fails here. If you examine the file later it has the right hash
#hash = self.readVar(('EXTERNAL_VCPKG_ANDROID_SHA512')
#hifi_utils.downloadAndExtract(url, dest, hash)
hifi_utils.downloadAndExtract(url, dest)
hash = self.readVar('EXTERNAL_VCPKG_ANDROID_SHA512')
hifi_utils.downloadAndExtract(url, dest, hash)

print("Installing additional android archives")
androidPackages = hifi_android.getPlatformPackages()
Expand Down
3 changes: 2 additions & 1 deletion prebuild.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ def main():

args = parse_args()
nsis_urls = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_URLS').split(';')
nsis_sha512 = hifi_utils.readEnviromentVariableFromFile(args.build_root, 'EXTERNAL_NSIS_HIFI_PLUGINS_SHA512')

if args.ci_build:
logging.basicConfig(datefmt='%H:%M:%S', format='%(asctime)s %(guid)s %(message)s', level=logging.INFO)
Expand All @@ -111,7 +112,7 @@ def main():
if 'Windows' == system and 'CI_BUILD' in os.environ and os.environ["CI_BUILD"] == "Github":
logger.info("Downloading NSIS")
with timer('NSIS'):
hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)")
hifi_utils.downloadAndExtract(nsis_urls, "C:/Program Files (x86)", nsis_sha512)

qtInstallPath = None
# If not android, install our Qt build
Expand Down

0 comments on commit 7c3b9d1

Please sign in to comment.