From 38c161f5ec15c9b6d6fdad0dfb01c5be24c33bc9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:06:48 +0200 Subject: [PATCH 01/13] Remove -no-ignition-auto option Signed-off-by: Jose Luis Rivero --- release.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/release.py b/release.py index 521289c3d..50f995c3d 100755 --- a/release.py +++ b/release.py @@ -143,8 +143,6 @@ def parse_args(argv): help='no-sanity-checks; i.e. skip sanity checks commands') parser.add_argument('--no-generate-source-file', dest='no_source_file', action='store_true', default=False, help='Do not generate source file when building') - parser.add_argument('--no-ignition-auto', dest='no_ignition_auto', action='store_true', default=False, - help='Use package name to create --package-alias if package name starts with ign-') parser.add_argument('--upload-to-repo', dest='upload_to_repository', default="stable", help='OSRF repo to upload: stable | prerelease | nightly') parser.add_argument('--extra-osrf-repo', dest='extra_repo', default="", @@ -162,9 +160,6 @@ def parse_args(argv): args.package = args.package.replace('ign-','gz-') args.package_alias = args.package - # If ignition auto is enabled, replace ign- with ignition- at the beginning - if not args.no_ignition_auto and args.package.startswith('ign-'): - args.package_alias = args.package.replace('ign-', 'ignition-') DRY_RUN = args.dry_run UPSTREAM = args.upstream From 4a22285ad45f06681b5fd6748088f702ab525916 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:09:47 +0200 Subject: [PATCH 02/13] Remove unused variables Signed-off-by: Jose Luis Rivero --- release.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/release.py b/release.py index 50f995c3d..3aaf736eb 100755 --- a/release.py +++ b/release.py @@ -164,10 +164,8 @@ def parse_args(argv): DRY_RUN = args.dry_run UPSTREAM = args.upstream NO_SRC_FILE = args.no_source_file - UPLOAD_REPO = args.upload_to_repository if args.upload_to_repository == 'nightly': NIGHTLY = True - NIGHTLY_BRANCH = args.nightly_branch if args.upload_to_repository == 'prerelease': PRERELEASE = True # Upstream and nightly do not generate a tar.bz2 file From 6db3917ab6f4bb289c86b3a8bc8c2f244803c6b9 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:09:59 +0200 Subject: [PATCH 03/13] 2 lines after methods Signed-off-by: Jose Luis Rivero --- release.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/release.py b/release.py index 3aaf736eb..16d2aa66e 100755 --- a/release.py +++ b/release.py @@ -157,7 +157,7 @@ def parse_args(argv): if args.package in GARDEN_IGN_PACKAGES: print(f"Garden packages start with gz- prefix, changing {args.package} to {args.package.replace('ign-','gz-')}",) - args.package = args.package.replace('ign-','gz-') + args.package = args.package.replace('ign-', 'gz-') args.package_alias = args.package @@ -175,6 +175,7 @@ def parse_args(argv): return args + def get_release_repository_info(package): # Do not use git@github method since it fails in non existant repositories # asking for stdin user/pass. Same happen if no user/pass is provided @@ -186,6 +187,7 @@ def get_release_repository_info(package): error("release repository not found in github.com/gazebo-release") + def download_release_repository(package, release_branch): vcs, url = get_release_repository_info(package) release_tmp_dir = tempfile.mkdtemp() @@ -203,6 +205,7 @@ def download_release_repository(package, release_branch): check_call(cmd, IGNORE_DRY_RUN) return release_tmp_dir, release_branch + def sanity_package_name_underscore(package, package_alias): # Alias is never empty. It hosts a exect copy of package if not provided if '_' in package_alias and package_alias != package: @@ -213,6 +216,7 @@ def sanity_package_name_underscore(package, package_alias): print_success("No underscore in package name") + def sanity_package_name(repo_dir, package, package_alias): expected_name = package @@ -264,6 +268,7 @@ def sanity_package_version(repo_dir, version, release_version): print_success("Package versions in changelog") print_success("Package release versions in changelog") + def sanity_check_gazebo_versions(package, version): if package == 'gazebo': if int(version[0]) > 1: @@ -276,6 +281,7 @@ def sanity_check_gazebo_versions(package, version): print_success("Gazebo version in proper gazebo package") + def sanity_check_sdformat_versions(package, version): if package == 'sdformat': if int(version[0]) > 1: @@ -285,12 +291,14 @@ def sanity_check_sdformat_versions(package, version): print_success("sdformat version in proper sdformat package") + def sanity_check_repo_name(repo_name): if repo_name in OSRF_REPOS_SUPPORTED: return error("Upload repo value: " + repo_name + " is not valid. stable | prerelease | nightly") + def sanity_project_package_in_stable(version, repo_name): if repo_name != 'stable': return @@ -303,12 +311,14 @@ def sanity_project_package_in_stable(version, repo_name): return + def sanity_use_prerelease_branch(release_branch): if release_branch == 'prerelease': error("The use of prerelease branch is now deprecated. Please check internal wiki instructions") return + def check_s3cmd_configuration(): # Need to check if s3cmd is installed try: @@ -323,6 +333,7 @@ def check_s3cmd_configuration(): return True + def sanity_checks(args, repo_dir): sanity_package_name_underscore(args.package, args.package_alias) sanity_package_name(repo_dir, args.package, args.package_alias) @@ -348,6 +359,7 @@ def get_exclusion_arches(files): return r + def discover_distros(repo_dir): if not os.path.isdir(repo_dir): return None @@ -379,6 +391,7 @@ def discover_distros(repo_dir): return distro_arch_list + def check_call(cmd, ignore_dry_run = False): if ignore_dry_run: # Commands that do not change anything in repo level @@ -443,6 +456,7 @@ def create_tarball_path(tarball_name, version, builddir, dry_run): return out.split(' ')[0], tarball_fname, tarball_path + def generate_upload_tarball(args): ################################################### # Platform-agnostic stuff. @@ -660,5 +674,6 @@ def go(argv): if not DRY_RUN: urllib.request.urlopen(url) + if __name__ == '__main__': go(sys.argv) From 89224f2c79959c6a7d65c55975f40c8fdfb86b77 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:13:32 +0200 Subject: [PATCH 04/13] Fix spacing and blank lines Signed-off-by: Jose Luis Rivero --- release.py | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/release.py b/release.py index 16d2aa66e..e17c20a95 100755 --- a/release.py +++ b/release.py @@ -16,9 +16,9 @@ JENKINS_URL = os.environ['JENKINS_URL'] except KeyError: JENKINS_URL = 'http://build.osrfoundation.org' -JOB_NAME_PATTERN = '%s-debbuilder' +JOB_NAME_PATTERN = ' %s-debbuilder' JOB_NAME_UPSTREAM_PATTERN = 'upstream-%s-debbuilder' -GENERIC_BREW_PULLREQUEST_JOB='generic-release-homebrew_pull_request_updater' +GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' UPLOAD_DEST_PATTERN = 's3://osrf-distributions/%s/releases/' DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' @@ -57,34 +57,43 @@ 'ign-transport12', 'ign-utils2'] + class ErrorNoPermsRepo(Exception): pass + class ErrorNoUsernameSupplied(Exception): pass + class ErrorURLNotFound404(Exception): pass + class ErrorNoOutput(Exception): pass + def error(msg): print("\n !! " + msg + "\n") sys.exit(1) + def print_success(msg): print(" + OK " + msg) + # Remove the last character if it is a number. # That should leave just the package name instead of packageVersion # I.E gazebo5 -> gazebo def get_canonical_package_name(pkg_name): return pkg_name.rstrip('1234567890') + def is_catkin_package(): return os.path.isfile("package.xml") + def github_repo_exists(url): try: check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN) @@ -94,6 +103,7 @@ def github_repo_exists(url): error("Unexpected problem checking for git repo: " + str(e)) return True + def generate_package_source(srcdir, builddir): cmake_cmd = ["cmake"] @@ -115,6 +125,7 @@ def exists_main_branch(github_url): except Exception as e: return False + def parse_args(argv): global DRY_RUN global NIGHTLY @@ -395,9 +406,9 @@ def discover_distros(repo_dir): def check_call(cmd, ignore_dry_run = False): if ignore_dry_run: # Commands that do not change anything in repo level - print('Dry-run running:\n %s\n'%(' '.join(cmd))) + print('Dry-run running:\n %s\n' % (' '.join(cmd))) else: - print('Running:\n %s'%(' '.join(cmd))) + print('Running:\n %s' % (' '.join(cmd))) if DRY_RUN and not ignore_dry_run: return '', '' else: @@ -416,9 +427,9 @@ def check_call(cmd, ignore_dry_run = False): raise ErrorNoOutput() # Unkown exception - print('Error running command (%s).'%(' '.join(cmd))) - print('stdout: %s'%(out.decode())) - print('stderr: %s'%(err.decode())) + print('Error running command (%s).' % (' '.join(cmd))) + print('stdout: %s' % (out.decode())) + print('stderr: %s' % (err.decode())) raise Exception('subprocess call failed') return out, err @@ -432,14 +443,14 @@ def create_tarball_name(args): # Returns: sha, tarball file name, tarball full path def create_tarball_path(tarball_name, version, builddir, dry_run): - tarball_fname = '%s-%s.tar.bz2'%(tarball_name, version) + tarball_fname = ' %s-%s.tar.bz2' % (tarball_name, version) # Try using the tarball_name as it is tarball_path = os.path.join(builddir, tarball_fname) if not os.path.isfile(tarball_path): # Try looking for special project names using underscores alt_tarball_name = "_".join(tarball_name.rsplit("-",1)) - alt_tarball_fname = '%s-%s.tar.bz2'%(alt_tarball_name, version) + alt_tarball_fname = ' %s-%s.tar.bz2' % (alt_tarball_name, version) alt_tarball_path = os.path.join(builddir, alt_tarball_fname) if (not dry_run): if not os.path.isfile(alt_tarball_path): @@ -500,7 +511,7 @@ def generate_upload_tarball(args): # If we're releasing under a different name, then rename the tarball (the # package itself doesn't know anything about this). if args.package != args.package_alias: - tarball_fname = '%s-%s.tar.bz2'%(args.package_alias, args.version) + tarball_fname = ' %s-%s.tar.bz2' % (args.package_alias, args.version) if (not args.dry_run): dest_file = os.path.join(builddir, tarball_fname) # Do not copy if files are the same @@ -526,7 +537,7 @@ def generate_upload_tarball(args): try: # tilde is not a valid character in git - tag = '%s_%s' % (args.package_alias, args.version.replace('~','-')) + tag = ' %s_%s' % (args.package_alias, args.version.replace('~','-')) check_call(['git', 'tag', '-f', tag]) check_call(['git', 'push', '--tags']) except ErrorNoPermsRepo as e: @@ -599,14 +610,14 @@ def go(argv): params['SOURCE_TARBALL_URI'] = args.nightly_branch if UPSTREAM: - job_name = JOB_NAME_UPSTREAM_PATTERN%(args.package) + job_name = JOB_NAME_UPSTREAM_PATTERN% (args.package) else: - job_name = JOB_NAME_PATTERN%(args.package) + job_name = JOB_NAME_PATTERN% (args.package) params_query = urllib.parse.urlencode(params) # RELEASING FOR BREW - brew_url = '%s/job/%s/buildWithParameters?%s'%(JENKINS_URL, + brew_url = ' %s/job/%s/buildWithParameters?%s' % (JENKINS_URL, GENERIC_BREW_PULLREQUEST_JOB, params_query) if not NIGHTLY and not args.bump_rev_linux_only: @@ -668,8 +679,8 @@ def go(argv): linux_platform_params_query = urllib.parse.urlencode(linux_platform_params) - url = '%s/job/%s/buildWithParameters?%s'%(JENKINS_URL, job_name, linux_platform_params_query) - print('- Linux: %s'%(url)) + url = ' %s/job/%s/buildWithParameters?%s' % (JENKINS_URL, job_name, linux_platform_params_query) + print('- Linux: %s' % (url)) if not DRY_RUN: urllib.request.urlopen(url) From 71ab6708303162712e56fe7e2622cf3953f158c2 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:20:53 +0200 Subject: [PATCH 05/13] More spacing fixes Signed-off-by: Jose Luis Rivero --- release.py | 54 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 28 insertions(+), 26 deletions(-) diff --git a/release.py b/release.py index e17c20a95..0a7a0d706 100755 --- a/release.py +++ b/release.py @@ -22,7 +22,7 @@ UPLOAD_DEST_PATTERN = 's3://osrf-distributions/%s/releases/' DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' -LINUX_DISTROS = [ 'ubuntu', 'debian' ] +LINUX_DISTROS = ['ubuntu', 'debian'] SUPPORTED_ARCHS = ['amd64', 'i386', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' @@ -87,7 +87,7 @@ def print_success(msg): # That should leave just the package name instead of packageVersion # I.E gazebo5 -> gazebo def get_canonical_package_name(pkg_name): - return pkg_name.rstrip('1234567890') + return pkg_name.rstrip('1234567890') def is_catkin_package(): @@ -97,7 +97,7 @@ def is_catkin_package(): def github_repo_exists(url): try: check_call(['git', 'ls-remote', '-q', '--exit-code', url], IGNORE_DRY_RUN) - except (ErrorURLNotFound404, ErrorNoOutput) as e: + except (ErrorURLNotFound404, ErrorNoOutput): return False except Exception as e: error("Unexpected problem checking for git repo: " + str(e)) @@ -122,7 +122,7 @@ def exists_main_branch(github_url): try: if (check_call(check_main_cmd, IGNORE_DRY_RUN)): return True - except Exception as e: + except Exception: return False @@ -220,10 +220,10 @@ def download_release_repository(package, release_branch): def sanity_package_name_underscore(package, package_alias): # Alias is never empty. It hosts a exect copy of package if not provided if '_' in package_alias and package_alias != package: - error("Found an underscore in package_alias. It will conflict with debian package names. May be fixed changing the underscore for a dash.") + error("Found an underscore in package_alias. It will conflict with debian package names. May be fixed changing the underscore for a dash.") if '_' in package and package_alias == package: - error("Found an underscore in package name without providing a package alias (-a ). You probably want to match the package name in the debian changelog") + error("Found an underscore in package name without providing a package alias (-a ). You probably want to match the package name in the debian changelog") print_success("No underscore in package name") @@ -235,10 +235,10 @@ def sanity_package_name(repo_dir, package, package_alias): expected_name = package_alias # Use igntiion for Citadel and Fortress, gz for Garden and beyond - gz_name = expected_name.replace("ignition", "gz"); - gz_name = gz_name.replace("gazebo", "sim"); + gz_name = expected_name.replace("ignition", "gz") + gz_name = gz_name.replace("gazebo", "sim") - cmd = ["find", repo_dir, "-name", "changelog","-exec","head","-n","1","{}",";"] + cmd = ["find", repo_dir, "-name", "changelog", "-exec", "head", "-n", "1", "{}", ";"] out, err = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: @@ -247,7 +247,7 @@ def sanity_package_name(repo_dir, package, package_alias): if line.partition(' ')[0] != expected_name and line.partition(' ')[0] != gz_name: error("Error in changelog package name or alias: " + line) - cmd = ["find", repo_dir, "-name", "control","-exec","grep","-H","Source:","{}",";"] + cmd = ["find", repo_dir, "-name", "control", "-exec", "grep", "-H", "Source:", "{}", ";"] out, err = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: @@ -258,17 +258,18 @@ def sanity_package_name(repo_dir, package, package_alias): print_success("Package names in changelog and control") + def sanity_package_version(repo_dir, version, release_version): - cmd = ["find", repo_dir, "-name", "changelog","-exec","head","-n","1","{}",";"] + cmd = ["find", repo_dir, "-name", "changelog", "-exec", "head", "-n", "1", "{}", ";"] out, err = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue # return full version in brackets - full_version=line.split(' ')[1] + full_version = line.split(' ')[1] # get only version (not release) in brackets - c_version=full_version[full_version.find("(")+1:full_version.find("-")] - c_revision=full_version[full_version.find("-")+1:full_version.rfind("~")] + c_version = full_version[full_version.find("(")+1:full_version.find("-")] + c_revision = full_version[full_version.find("-")+1:full_version.rfind("~")] if c_version != version: error("Error in package version. Repo version: " + c_version + " Provided version: " + version) @@ -398,12 +399,12 @@ def discover_distros(repo_dir): print('Releasing for distributions: ') for k in distro_arch_list: - print( "- " + k + " (" + ', '.join(distro_arch_list[k]) +")") + print("- " + k + " (" + ', '.join(distro_arch_list[k]) + ")") return distro_arch_list -def check_call(cmd, ignore_dry_run = False): +def check_call(cmd, ignore_dry_run=False): if ignore_dry_run: # Commands that do not change anything in repo level print('Dry-run running:\n %s\n' % (' '.join(cmd))) @@ -437,8 +438,7 @@ def check_call(cmd, ignore_dry_run = False): # Returns tarball name: package name/alias without versions def create_tarball_name(args): # For ignition, we use the package_alias instead of package - return re.sub(r'[0-9]+$', '', - args.package if not IGN_REPO else args.package_alias) + return re.sub(r'[0-9]+$', '', args.package if not IGN_REPO else args.package_alias) # Returns: sha, tarball file name, tarball full path @@ -449,7 +449,7 @@ def create_tarball_path(tarball_name, version, builddir, dry_run): if not os.path.isfile(tarball_path): # Try looking for special project names using underscores - alt_tarball_name = "_".join(tarball_name.rsplit("-",1)) + alt_tarball_name = "_".join(tarball_name.rsplit("-", 1)) alt_tarball_fname = ' %s-%s.tar.bz2' % (alt_tarball_name, version) alt_tarball_path = os.path.join(builddir, alt_tarball_fname) if (not dry_run): @@ -572,8 +572,8 @@ def go(argv): if not UPSTREAM: repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) # The supported distros are the ones in the top level of -release repo - ubuntu_distros = discover_distros(repo_dir) # top level, ubuntu - debian_distros = discover_distros(repo_dir + '/debian/') # debian dir top level, Debian + ubuntu_distros = discover_distros(repo_dir) # top level, ubuntu + debian_distros = discover_distros(repo_dir + '/debian/') # debian dir top level, Debian if not args.no_sanity_checks: sanity_checks(args, repo_dir) @@ -610,16 +610,18 @@ def go(argv): params['SOURCE_TARBALL_URI'] = args.nightly_branch if UPSTREAM: - job_name = JOB_NAME_UPSTREAM_PATTERN% (args.package) + job_name = JOB_NAME_UPSTREAM_PATTERN % (args.package) else: - job_name = JOB_NAME_PATTERN% (args.package) + job_name = JOB_NAME_PATTERN % (args.package) params_query = urllib.parse.urlencode(params) # RELEASING FOR BREW - brew_url = ' %s/job/%s/buildWithParameters?%s' % (JENKINS_URL, - GENERIC_BREW_PULLREQUEST_JOB, - params_query) + brew_url = ' %s/job/%s/buildWithParameters?%s' % ( + JENKINS_URL, + GENERIC_BREW_PULLREQUEST_JOB, + params_query) + if not NIGHTLY and not args.bump_rev_linux_only: print('- Brew: %s' % (brew_url)) if not DRY_RUN: From 5492f87e0a2210930f14e2d123849c3c83d4dd3e Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:22:05 +0200 Subject: [PATCH 06/13] For create_tarball_name remove the option of package_alias Signed-off-by: Jose Luis Rivero --- release.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/release.py b/release.py index 0a7a0d706..122e8f5ce 100755 --- a/release.py +++ b/release.py @@ -437,8 +437,7 @@ def check_call(cmd, ignore_dry_run=False): # Returns tarball name: package name/alias without versions def create_tarball_name(args): - # For ignition, we use the package_alias instead of package - return re.sub(r'[0-9]+$', '', args.package if not IGN_REPO else args.package_alias) + return re.sub(r'[0-9]+$', '', args.package) # Returns: sha, tarball file name, tarball full path From 1b03461f10af136fa87d284d4fe3f1e58691106c Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:35:35 +0200 Subject: [PATCH 07/13] Remove sanity_check_gazebo_versions (outdated) Signed-off-by: Jose Luis Rivero --- release.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/release.py b/release.py index 122e8f5ce..3e2ae9401 100755 --- a/release.py +++ b/release.py @@ -281,19 +281,6 @@ def sanity_package_version(repo_dir, version, release_version): print_success("Package release versions in changelog") -def sanity_check_gazebo_versions(package, version): - if package == 'gazebo': - if int(version[0]) > 1: - error("Error in gazebo version. Please use 'gazebo-current' package for gazebo 2") - elif package == 'gazebo-current': - if int(version[0]) < 2: - error("Error in gazebo-current version. Please use 'gazebo' package for gazebo 1.x") - else: - return - - print_success("Gazebo version in proper gazebo package") - - def sanity_check_sdformat_versions(package, version): if package == 'sdformat': if int(version[0]) > 1: @@ -335,7 +322,7 @@ def check_s3cmd_configuration(): # Need to check if s3cmd is installed try: subprocess.call(["s3cmd", "--version"]) - except OSError as e: + except OSError: error("s3cmd command for uploading is not available. Install it using: apt-get install s3cmd") # Need to check if configuration for s3 exists @@ -355,7 +342,6 @@ def sanity_checks(args, repo_dir): if not NIGHTLY: check_s3cmd_configuration() sanity_package_version(repo_dir, args.version, str(args.release_version)) - sanity_check_gazebo_versions(args.package, args.version) sanity_check_sdformat_versions(args.package, args.version) sanity_project_package_in_stable(args.version, args.upload_to_repository) From 7aa5bcb1101a0f002b2833f31b144b57b371e0ba Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:53:57 +0200 Subject: [PATCH 08/13] Use byte stream for empty return type in check_call Signed-off-by: Jose Luis Rivero --- release.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/release.py b/release.py index 3e2ae9401..7ae919fc1 100755 --- a/release.py +++ b/release.py @@ -365,12 +365,16 @@ def discover_distros(repo_dir): _, subdirs, files = os.walk(repo_dir).__next__() repo_arch_exclusion = get_exclusion_arches(files) - if '.git' in subdirs: subdirs.remove('.git') - # remove ubuntu (common stuff) and debian (new supported distro at top level) - if 'ubuntu' in subdirs: subdirs.remove('ubuntu') - if 'debian' in subdirs: subdirs.remove('debian') + if '.git' in subdirs: + subdirs.remove('.git') + # remove ubuntu (common stuff) and debian (supported distro at top level) + if 'ubuntu' in subdirs: + subdirs.remove('ubuntu') + if 'debian' in subdirs: + subdirs.remove('debian') # Some releasing methods use patches/ in root - if 'patches' in subdirs: subdirs.remove('patches') + if 'patches' in subdirs: + subdirs.remove('patches') if not subdirs: error('Can not find distributions directories in the -release repo') @@ -397,7 +401,7 @@ def check_call(cmd, ignore_dry_run=False): else: print('Running:\n %s' % (' '.join(cmd))) if DRY_RUN and not ignore_dry_run: - return '', '' + return b'', b'' else: po = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = po.communicate() From f808a2844c0c66de13f74110afeb90d454874877 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 19:56:43 +0200 Subject: [PATCH 09/13] Change method not to end up in exit Signed-off-by: Jose Luis Rivero --- release.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/release.py b/release.py index 7ae919fc1..27ba055a5 100755 --- a/release.py +++ b/release.py @@ -192,11 +192,12 @@ def get_release_repository_info(package): # asking for stdin user/pass. Same happen if no user/pass is provided # using the fake foo:foo here seems to work github_test_url = "https://foo:foo@github.com/gazebo-release/" + package + "-release" - if (github_repo_exists(github_test_url)): - github_url = "https://github.com/gazebo-release/" + package + "-release" - return 'git', github_url + if (not github_repo_exists(github_test_url)): + error("release repository not found in github.com/gazebo-release") + + github_url = "https://github.com/gazebo-release/" + package + "-release" + return 'git', github_url - error("release repository not found in github.com/gazebo-release") def download_release_repository(package, release_branch): @@ -239,7 +240,7 @@ def sanity_package_name(repo_dir, package, package_alias): gz_name = gz_name.replace("gazebo", "sim") cmd = ["find", repo_dir, "-name", "changelog", "-exec", "head", "-n", "1", "{}", ";"] - out, err = check_call(cmd, IGNORE_DRY_RUN) + out, _ = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue @@ -248,7 +249,7 @@ def sanity_package_name(repo_dir, package, package_alias): error("Error in changelog package name or alias: " + line) cmd = ["find", repo_dir, "-name", "control", "-exec", "grep", "-H", "Source:", "{}", ";"] - out, err = check_call(cmd, IGNORE_DRY_RUN) + out, _ = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue @@ -261,7 +262,7 @@ def sanity_package_name(repo_dir, package, package_alias): def sanity_package_version(repo_dir, version, release_version): cmd = ["find", repo_dir, "-name", "changelog", "-exec", "head", "-n", "1", "{}", ";"] - out, err = check_call(cmd, IGNORE_DRY_RUN) + out, _ = check_call(cmd, IGNORE_DRY_RUN) for line in out.decode().split('\n'): if not line: continue @@ -529,16 +530,16 @@ def generate_upload_tarball(args): tag = ' %s_%s' % (args.package_alias, args.version.replace('~','-')) check_call(['git', 'tag', '-f', tag]) check_call(['git', 'push', '--tags']) - except ErrorNoPermsRepo as e: + except ErrorNoPermsRepo: print('The Git server reports problems with permissions') print('The branch could be blocked by configuration if you do not have') print('rights to push code in default branch.') sys.exit(1) - except ErrorNoUsernameSupplied as e: + except ErrorNoUsernameSupplied: print('git tag could not be committed because you have not configured') print('your username. Use "git config --username" to set your username.') sys.exit(1) - except Exception as e: + except Exception: print('There was a problem with pushing tags to the git repository') print('Do you have write perms in the repository?') sys.exit(1) From 13b4936850580f5552689f4d2bc1cb8070394773 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 20:22:50 +0200 Subject: [PATCH 10/13] Remove upstream option Signed-off-by: Jose Luis Rivero --- release.py | 30 +++++++++--------------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/release.py b/release.py index 27ba055a5..5e5007611 100755 --- a/release.py +++ b/release.py @@ -17,7 +17,6 @@ except KeyError: JENKINS_URL = 'http://build.osrfoundation.org' JOB_NAME_PATTERN = ' %s-debbuilder' -JOB_NAME_UPSTREAM_PATTERN = 'upstream-%s-debbuilder' GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' UPLOAD_DEST_PATTERN = 's3://osrf-distributions/%s/releases/' DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' @@ -36,7 +35,6 @@ DRY_RUN = False NIGHTLY = False PRERELEASE = False -UPSTREAM = False NO_SRC_FILE = False IGNORE_DRY_RUN = True @@ -130,7 +128,6 @@ def parse_args(argv): global DRY_RUN global NIGHTLY global PRERELEASE - global UPSTREAM global NO_SRC_FILE parser = argparse.ArgumentParser(description='Make releases.') @@ -139,8 +136,6 @@ def parse_args(argv): parser.add_argument('jenkins_token', help='secret token to allow access to Jenkins to start builds') parser.add_argument('--dry-run', dest='dry_run', action='store_true', default=False, help='dry-run; i.e., do actually run any of the commands') - parser.add_argument('-u', '--upstream', dest='upstream', action='store_true', default=False, - help='release non OSRF software (do not generate and upload source tar.bz)') parser.add_argument('-a', '--package-alias', dest='package_alias', default=None, help='different name that we are releasing under') @@ -173,14 +168,13 @@ def parse_args(argv): args.package_alias = args.package DRY_RUN = args.dry_run - UPSTREAM = args.upstream NO_SRC_FILE = args.no_source_file if args.upload_to_repository == 'nightly': NIGHTLY = True if args.upload_to_repository == 'prerelease': PRERELEASE = True - # Upstream and nightly do not generate a tar.bz2 file - if args.upstream or NIGHTLY: + # Nightly do not generate a tar.bz2 file + if NIGHTLY: NO_SRC_FILE = True args.no_source_file = True @@ -558,14 +552,12 @@ def go(argv): args.release_version = 1 # Sanity checks and dicover supported distributions before proceed. - # UPSTREAM repository is not known in release-tools script - if not UPSTREAM: - repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) - # The supported distros are the ones in the top level of -release repo - ubuntu_distros = discover_distros(repo_dir) # top level, ubuntu - debian_distros = discover_distros(repo_dir + '/debian/') # debian dir top level, Debian - if not args.no_sanity_checks: - sanity_checks(args, repo_dir) + repo_dir, args.release_repo_branch = download_release_repository(args.package, args.release_repo_branch) + # The supported distros are the ones in the top level of -release repo + ubuntu_distros = discover_distros(repo_dir) # top level, ubuntu + debian_distros = discover_distros(repo_dir + '/debian/') # debian dir top level, Debian + if not args.no_sanity_checks: + sanity_checks(args, repo_dir) source_tarball_uri = '' source_tarball_sha = '' @@ -599,11 +591,7 @@ def go(argv): # name must be modified in the future params['SOURCE_TARBALL_URI'] = args.nightly_branch - if UPSTREAM: - job_name = JOB_NAME_UPSTREAM_PATTERN % (args.package) - else: - job_name = JOB_NAME_PATTERN % (args.package) - + job_name = JOB_NAME_PATTERN % (args.package) params_query = urllib.parse.urlencode(params) # RELEASING FOR BREW From 2c3717a0c36895e49df22f89a1b495b8f05a2ffe Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 15 Sep 2023 20:29:11 +0200 Subject: [PATCH 11/13] Remove i386 from release.py Signed-off-by: Jose Luis Rivero --- release.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/release.py b/release.py index 5e5007611..b7737258f 100755 --- a/release.py +++ b/release.py @@ -22,7 +22,7 @@ DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' LINUX_DISTROS = ['ubuntu', 'debian'] -SUPPORTED_ARCHS = ['amd64', 'i386', 'armhf', 'arm64'] +SUPPORTED_ARCHS = ['amd64', 'armhf', 'arm64'] RELEASEPY_NO_ARCH_PREFIX = '.releasepy_NO_ARCH_' # Ubuntu distributions are automatically taken from the top directory of @@ -624,9 +624,6 @@ def go(argv): if (PRERELEASE or NIGHTLY): if (a == 'armhf' or a == 'arm64'): continue - # Only i386 for Ubuntu in Bionic - if (a == 'i386' and l != 'debian' and d != 'bionic'): - continue linux_platform_params = params.copy() linux_platform_params['ARCH'] = a @@ -645,9 +642,6 @@ def go(argv): ('gz-physics' in args.package_alias): linux_platform_params['JENKINS_NODE_TAG'] = 'large-memory' - if (NIGHTLY and a == 'i386'): - continue - # control nightly generation using a single machine to process # all distribution builds to avoid race conditions. Note: this # assumes that large-memory nodes are beind used for nightly From 38f04ab0dcd479b13edd1906c311cc7418a86909 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Sep 2023 17:26:31 +0200 Subject: [PATCH 12/13] Fix tarball name and tag name Signed-off-by: Jose Luis Rivero --- release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release.py b/release.py index b7737258f..001ecc6db 100755 --- a/release.py +++ b/release.py @@ -427,7 +427,7 @@ def create_tarball_name(args): # Returns: sha, tarball file name, tarball full path def create_tarball_path(tarball_name, version, builddir, dry_run): - tarball_fname = ' %s-%s.tar.bz2' % (tarball_name, version) + tarball_fname = '%s-%s.tar.bz2' % (tarball_name, version) # Try using the tarball_name as it is tarball_path = os.path.join(builddir, tarball_fname) @@ -521,7 +521,7 @@ def generate_upload_tarball(args): try: # tilde is not a valid character in git - tag = ' %s_%s' % (args.package_alias, args.version.replace('~','-')) + tag = '%s_%s' % (args.package_alias, args.version.replace('~', '-')) check_call(['git', 'tag', '-f', tag]) check_call(['git', 'push', '--tags']) except ErrorNoPermsRepo: From f82e937ec0b9920f81fcaeb691b764dc271d3b86 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Thu, 28 Sep 2023 17:35:48 +0200 Subject: [PATCH 13/13] More fix to pattern '%s' Signed-off-by: Jose Luis Rivero --- release.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/release.py b/release.py index 001ecc6db..3149e76ef 100755 --- a/release.py +++ b/release.py @@ -16,7 +16,7 @@ JENKINS_URL = os.environ['JENKINS_URL'] except KeyError: JENKINS_URL = 'http://build.osrfoundation.org' -JOB_NAME_PATTERN = ' %s-debbuilder' +JOB_NAME_PATTERN = '%s-debbuilder' GENERIC_BREW_PULLREQUEST_JOB = 'generic-release-homebrew_pull_request_updater' UPLOAD_DEST_PATTERN = 's3://osrf-distributions/%s/releases/' DOWNLOAD_URI_PATTERN = 'https://osrf-distributions.s3.amazonaws.com/%s/releases/' @@ -434,7 +434,7 @@ def create_tarball_path(tarball_name, version, builddir, dry_run): if not os.path.isfile(tarball_path): # Try looking for special project names using underscores alt_tarball_name = "_".join(tarball_name.rsplit("-", 1)) - alt_tarball_fname = ' %s-%s.tar.bz2' % (alt_tarball_name, version) + alt_tarball_fname = '%s-%s.tar.bz2' % (alt_tarball_name, version) alt_tarball_path = os.path.join(builddir, alt_tarball_fname) if (not dry_run): if not os.path.isfile(alt_tarball_path): @@ -495,7 +495,7 @@ def generate_upload_tarball(args): # If we're releasing under a different name, then rename the tarball (the # package itself doesn't know anything about this). if args.package != args.package_alias: - tarball_fname = ' %s-%s.tar.bz2' % (args.package_alias, args.version) + tarball_fname = '%s-%s.tar.bz2' % (args.package_alias, args.version) if (not args.dry_run): dest_file = os.path.join(builddir, tarball_fname) # Do not copy if files are the same @@ -595,7 +595,7 @@ def go(argv): params_query = urllib.parse.urlencode(params) # RELEASING FOR BREW - brew_url = ' %s/job/%s/buildWithParameters?%s' % ( + brew_url = '%s/job/%s/buildWithParameters?%s' % ( JENKINS_URL, GENERIC_BREW_PULLREQUEST_JOB, params_query) @@ -653,7 +653,7 @@ def go(argv): linux_platform_params_query = urllib.parse.urlencode(linux_platform_params) - url = ' %s/job/%s/buildWithParameters?%s' % (JENKINS_URL, job_name, linux_platform_params_query) + url = '%s/job/%s/buildWithParameters?%s' % (JENKINS_URL, job_name, linux_platform_params_query) print('- Linux: %s' % (url)) if not DRY_RUN: